Fixes to checkpoint.nextXid accounting:

- Advance next_xid only when incoming xlog record has one. - Advance next_xid by one. Remove XID_CHECKPOINT_INTERVAL.
Add batch_wraparound/test_xid_wraparound.py
2026-07-07 14:10:43 +00:00 · 2021-09-01 12:26:57 +03:00 · 2021-09-01 12:26:57 +03:00 · 2021-09-01 09:42:18 +03:00 · 2021-08-31 13:09:16 -07:00 · 2021-08-31 14:10:50 +03:00
109 changed files with 9086 additions and 5996 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -7,9 +7,19 @@ executors:
  zenith-build-executor:
    resource_class: xlarge
    docker:
-      - image: cimg/rust:1.52.0
+      - image: cimg/rust:1.52.1

 jobs:
+  check-codestyle:
+    executor: zenith-build-executor
+    steps:
+      - checkout
+
+      - run:
+          name: rustfmt
+          when: always
+          command: |
+            cargo fmt --all -- --check

  # A job to build postgres
  build-postgres:
@@ -229,7 +239,7 @@ jobs:
          when: always
          command: |
            du -sh /tmp/test_output/*
-            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "wal_acceptor.log" -delete
+            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "wal_acceptor.log" ! -name "regression.diffs" ! -name "junit.xml" -delete
            du -sh /tmp/test_output/*
      - store_artifacts:
          path: /tmp/test_output
@@ -237,9 +247,27 @@ jobs:
      - store_test_results:
          path: /tmp/test_output

+  # Build zenithdb/zenith:latest image and push it to Docker hub
+  docker-image:
+    docker:
+      - image: cimg/base:2021.04
+    steps:
+      - checkout
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - run:
+          name: Init postgres submodule
+          command: git submodule update --init --depth 1
+      - run:
+          name: Build and push Docker image
+          command: |
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
+            docker build -t zenithdb/zenith:latest . && docker push zenithdb/zenith:latest
+
 workflows:
  build_and_test:
    jobs:
+      - check-codestyle
      - build-postgres
      - build-zenith:
          name: build-zenith-<< matrix.build_type >>
@@ -257,6 +285,14 @@ workflows:
          needs_postgres_source: true
          requires:
            - build-zenith-<< matrix.build_type >>
+      - run-pytest:
+          name: xid wraparound tests << matrix.build_type >>
+          matrix:
+            parameters:
+              build_type: ["debug", "release"]
+          test_selection: batch_wraparound
+          requires:
+            - build-zenith-<< matrix.build_type >>
      - run-pytest:
          name: other tests << matrix.build_type >>
          matrix:
@@ -265,3 +301,20 @@ workflows:
          test_selection: batch_others
          requires:
            - build-zenith-<< matrix.build_type >>
+      - run-pytest:
+          name: benchmarks
+          build_type: release
+          test_selection: performance
+          requires:
+            - build-zenith-release
+      - docker-image:
+          # Context gives an ability to login
+          context: Docker Hub
+          # Build image only for commits to main
+          filters:
+            branches:
+              only:
+                - main
+          requires:
+            - pg_regress tests release
+            - other tests release
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,4 +1,4 @@
 [submodule "vendor/postgres"]
 	path = vendor/postgres
-	url = https://github.com/libzenith/postgres
+	url = https://github.com/zenithdb/postgres
 	branch = main
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1,5 +1,7 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+version = 3
+
 [[package]]
 name = "ahash"
 version = "0.4.7"
@@ -82,9 +84,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"

 [[package]]
 name = "aversion"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25b49482974b90e9f36c5adcc50acde2e27e806ac269ff32758d700432782bc0"
+checksum = "41992ab8cfcc3026ef9abceffe0c2b0479c043183fc23825e30d22baab6df334"
 dependencies = [
 "aversion-macros",
 "byteorder",
@@ -95,9 +97,9 @@ dependencies = [

 [[package]]
 name = "aversion-macros"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed3009cf133dbd82459e96cf46bb24c8e6ad5c02c387ddb21d0f2c4c781a5394"
+checksum = "5ba5785f953985aa0caca927ba4005880f3b4f53de87f134e810ae3549f744d2"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -129,6 +131,12 @@ dependencies = [
 "anyhow",
 ]

+[[package]]
+name = "base64"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
+
 [[package]]
 name = "base64"
 version = "0.13.0"
@@ -146,9 +154,9 @@ dependencies = [

 [[package]]
 name = "bindgen"
-version = "0.57.0"
+version = "0.59.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd4865004a46a0aafb2a0a5eb19d3c9fc46ee5f063a6cfc605c69ac9ecf5263d"
+checksum = "453c49e5950bb0eb63bb3df640e31618846c89d5b7faa54040d76e98e0134375"
 dependencies = [
 "bitflags",
 "cexpr",
@@ -173,6 +181,18 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"

+[[package]]
+name = "bitvec"
+version = "0.19.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8942c8d352ae1838c9dda0b0ca2ab657696ef2232a20147cf1b30ae1a9cb4321"
+dependencies = [
+ "funty",
+ "radium",
+ "tap",
+ "wyz",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.9.0"
@@ -184,9 +204,9 @@ dependencies = [

 [[package]]
 name = "bookfile"
-version = "0.2.0"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7db391acd99b8bdce5d5a66ca28530761affec9a407df91aee668fc318e3db71"
+checksum = "efa3e2086414e1bbecbc10730f265e5b079ab4ea0b830e7219a70dab6471e753"
 dependencies = [
 "aversion",
 "byteorder",
@@ -226,15 +246,12 @@ name = "cc"
 version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2"
-dependencies = [
- "jobserver",
-]

 [[package]]
 name = "cexpr"
-version = "0.4.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27"
+checksum = "db507a7679252d2276ed0dd8113c6875ec56d3089f9225b2b42c30cc1f8e5c89"
 dependencies = [
 "nom",
 ]
@@ -304,6 +321,7 @@ dependencies = [
 "postgres_ffi",
 "rand",
 "regex",
+ "reqwest",
 "serde",
 "serde_json",
 "tar",
@@ -529,10 +547,10 @@ dependencies = [
 ]

 [[package]]
-name = "fs_extra"
-version = "1.2.0"
+name = "funty"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394"
+checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7"

 [[package]]
 name = "futures"
@@ -848,15 +866,6 @@ version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"

-[[package]]
-name = "jobserver"
-version = "0.1.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "972f5ae5d1cb9c6ae417789196c803205313edde988685da5e3aae0827b9e7fd"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "js-sys"
 version = "0.3.51"
@@ -866,6 +875,20 @@ dependencies = [
 "wasm-bindgen",
 ]

+[[package]]
+name = "jsonwebtoken"
+version = "7.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "afabcc15e437a6484fc4f12d0fd63068fe457bf93f1c148d3d9649c60b103f32"
+dependencies = [
+ "base64 0.12.3",
+ "pem",
+ "ring",
+ "serde",
+ "serde_json",
+ "simple_asn1",
+]
+
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -894,18 +917,6 @@ dependencies = [
 "winapi",
 ]

-[[package]]
-name = "librocksdb-sys"
-version = "6.17.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5da125e1c0f22c7cae785982115523a0738728498547f415c9054cb17c7e89f9"
-dependencies = [
- "bindgen",
- "cc",
- "glob",
- "libc",
-]
-
 [[package]]
 name = "lock_api"
 version = "0.4.4"
@@ -1042,10 +1053,12 @@ dependencies = [

 [[package]]
 name = "nom"
-version = "5.1.2"
+version = "6.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af"
+checksum = "e7413f999671bd4745a7b624bd370a569fb6bc574b23c83a3c5ed2e453f3d5e2"
 dependencies = [
+ "bitvec",
+ "funty",
 "memchr",
 "version_check",
 ]
@@ -1059,6 +1072,17 @@ dependencies = [
 "winapi",
 ]

+[[package]]
+name = "num-bigint"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-integer"
 version = "0.1.44"
@@ -1155,10 +1179,10 @@ dependencies = [
 "clap",
 "crc32c",
 "daemonize",
- "fs_extra",
 "futures",
 "hex",
 "humantime",
+ "hyper",
 "lazy_static",
 "log",
 "postgres",
@@ -1167,22 +1191,20 @@ dependencies = [
 "postgres_ffi",
 "rand",
 "regex",
- "rocksdb",
- "rust-s3",
+ "routerify",
+ "scopeguard",
 "serde",
 "serde_json",
 "slog",
- "slog-async",
 "slog-scope",
 "slog-stdlog",
 "slog-term",
 "tar",
 "thiserror",
 "tokio",
- "tokio-stream",
 "toml",
- "walkdir",
 "workspace_hack",
+ "zenith_metrics",
 "zenith_utils",
 ]

@@ -1217,6 +1239,17 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"

+[[package]]
+name = "pem"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd56cbd21fea48d0c440b41cd69c589faacade08c992d9a54e471b79d0fd13eb"
+dependencies = [
+ "base64 0.13.0",
+ "once_cell",
+ "regex",
+]
+
 [[package]]
 name = "percent-encoding"
 version = "2.1.0"
@@ -1273,12 +1306,30 @@ dependencies = [
 "tokio-postgres 0.7.1",
 ]

+[[package]]
+name = "postgres-protocol"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff3e0f70d32e20923cabf2df02913be7c1842d4c772db8065c00fcfdd1d1bff3"
+dependencies = [
+ "base64 0.13.0",
+ "byteorder",
+ "bytes",
+ "fallible-iterator",
+ "hmac",
+ "md-5",
+ "memchr",
+ "rand",
+ "sha2",
+ "stringprep",
+]
+
 [[package]]
 name = "postgres-protocol"
 version = "0.6.1"
 source = "git+https://github.com/zenithdb/rust-postgres.git?rev=9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858#9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858"
 dependencies = [
- "base64",
+ "base64 0.13.0",
 "byteorder",
 "bytes",
 "fallible-iterator",
@@ -1292,21 +1343,14 @@ dependencies = [
 ]

 [[package]]
-name = "postgres-protocol"
-version = "0.6.1"
+name = "postgres-types"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff3e0f70d32e20923cabf2df02913be7c1842d4c772db8065c00fcfdd1d1bff3"
+checksum = "430f4131e1b7657b0cd9a2b0c3408d77c9a43a042d300b8c77f981dffcc43a2f"
 dependencies = [
- "base64",
- "byteorder",
 "bytes",
 "fallible-iterator",
- "hmac",
- "md-5",
- "memchr",
- "rand",
- "sha2",
- "stringprep",
+ "postgres-protocol 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

 [[package]]
@@ -1319,17 +1363,6 @@ dependencies = [
 "postgres-protocol 0.6.1 (git+https://github.com/zenithdb/rust-postgres.git?rev=9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858)",
 ]

-[[package]]
-name = "postgres-types"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "430f4131e1b7657b0cd9a2b0c3408d77c9a43a042d300b8c77f981dffcc43a2f"
-dependencies = [
- "bytes",
- "fallible-iterator",
- "postgres-protocol 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "postgres_ffi"
 version = "0.1.0"
@@ -1346,8 +1379,10 @@ dependencies = [
 "memoffset",
 "rand",
 "regex",
+ "serde",
 "thiserror",
 "workspace_hack",
+ "zenith_utils",
 ]

 [[package]]
@@ -1377,6 +1412,27 @@ dependencies = [
 "unicode-xid",
 ]

+[[package]]
+name = "prometheus"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5986aa8d62380092d2f50f8b1cdba9cb9b6731ffd4b25b51fd126b6c3e05b99c"
+dependencies = [
+ "cfg-if 1.0.0",
+ "fnv",
+ "lazy_static",
+ "memchr",
+ "parking_lot",
+ "protobuf",
+ "thiserror",
+]
+
+[[package]]
+name = "protobuf"
+version = "2.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db50e77ae196458ccd3dc58a31ea1a90b0698ab1b7928d89f644c25d72070267"
+
 [[package]]
 name = "proxy"
 version = "0.1.0"
@@ -1412,6 +1468,12 @@ dependencies = [
 "proc-macro2",
 ]

+[[package]]
+name = "radium"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8"
+
 [[package]]
 name = "rand"
 version = "0.8.4"
@@ -1503,7 +1565,7 @@ version = "0.11.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22"
 dependencies = [
- "base64",
+ "base64 0.13.0",
 "bytes",
 "encoding_rs",
 "futures-core",
@@ -1533,13 +1595,31 @@ dependencies = [
 ]

 [[package]]
-name = "rocksdb"
-version = "0.16.0"
+name = "ring"
+version = "0.16.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c749134fda8bfc90d0de643d59bfc841dcb3ac8a1062e12b6754bd60235c48b3"
+checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
 dependencies = [
+ "cc",
 "libc",
- "librocksdb-sys",
+ "once_cell",
+ "spin",
+ "untrusted",
+ "web-sys",
+ "winapi",
+]
+
+[[package]]
+name = "routerify"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c6bb49594c791cadb5ccfa5f36d41b498d40482595c199d10cd318800280bd9"
+dependencies = [
+ "http",
+ "hyper",
+ "lazy_static",
+ "percent-encoding",
+ "regex",
 ]

 [[package]]
@@ -1562,7 +1642,7 @@ dependencies = [
 "async-trait",
 "aws-creds",
 "aws-region",
- "base64",
+ "base64 0.13.0",
 "cfg-if 1.0.0",
 "chrono",
 "futures",
@@ -1754,9 +1834,9 @@ dependencies = [

 [[package]]
 name = "shlex"
-version = "0.1.1"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
+checksum = "42a568c8f2cd051a4d283bd6eb0343ac214c1b0f1ac19f93e1175b2dee38c73d"

 [[package]]
 name = "signal-hook-registry"
@@ -1767,6 +1847,17 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "simple_asn1"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "692ca13de57ce0613a363c8c2f1de925adebc81b04c923ac60c5488bb44abe4b"
+dependencies = [
+ "chrono",
+ "num-bigint",
+ "num-traits",
+]
+
 [[package]]
 name = "siphasher"
 version = "0.3.5"
@@ -1848,6 +1939,12 @@ dependencies = [
 "winapi",
 ]

+[[package]]
+name = "spin"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
+
 [[package]]
 name = "stringprep"
 version = "0.1.2"
@@ -1887,6 +1984,12 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60"

+[[package]]
+name = "tap"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
+
 [[package]]
 name = "tar"
 version = "0.4.35"
@@ -2184,6 +2287,12 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"

+[[package]]
+name = "untrusted"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
+
 [[package]]
 name = "url"
 version = "2.2.2"
@@ -2230,6 +2339,7 @@ name = "walkeeper"
 version = "0.1.0"
 dependencies = [
 "anyhow",
+ "bincode",
 "byteorder",
 "bytes",
 "clap",
@@ -2424,6 +2534,12 @@ dependencies = [
 "syn",
 ]

+[[package]]
+name = "wyz"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214"
+
 [[package]]
 name = "xattr"
 version = "0.2.2"
@@ -2454,6 +2570,14 @@ dependencies = [
 "zenith_utils",
 ]

+[[package]]
+name = "zenith_metrics"
+version = "0.1.0"
+dependencies = [
+ "lazy_static",
+ "prometheus",
+]
+
 [[package]]
 name = "zenith_utils"
 version = "0.1.0"
@@ -2462,11 +2586,19 @@ dependencies = [
 "bincode",
 "byteorder",
 "bytes",
+ "hex",
 "hex-literal",
+ "hyper",
+ "jsonwebtoken",
+ "lazy_static",
 "log",
 "postgres",
 "rand",
+ "routerify",
 "serde",
+ "serde_json",
 "thiserror",
+ "tokio",
 "workspace_hack",
+ "zenith_metrics",
 ]
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,13 +1,14 @@
 [workspace]
 members = [
-    "pageserver",
-    "walkeeper",
-    "zenith",
    "control_plane",
+    "pageserver",
    "postgres_ffi",
-    "zenith_utils",
+    "proxy",
+    "walkeeper",
    "workspace_hack",
-    "proxy"
+    "zenith",
+    "zenith_metrics",
+    "zenith_utils",
 ]

 [profile.release]
--- a/83
+++ b/83
@@ -1,94 +1,77 @@
 #
 # Docker image for console integration testing.
 #
-# We may also reuse it in CI to unify installation process and as a general binaries building
-# tool for production servers.
-#
-# Dynamic linking is used for librocksdb and libstdc++ bacause librocksdb-sys calls
-# bindgen with "dynamic" feature flag. This also prevents usage of dockerhub alpine-rust
-# images which are statically linked and have guards against any dlopen. I would rather
-# prefer all static binaries so we may change the way librocksdb-sys builds or wait until
-# we will have our own storage and drop rockdb dependency.
-#
-# Cargo-chef is used to separate dependencies building from main binaries building. This
-# way `docker build` will download and install dependencies only of there are changes to
-# out Cargo.toml files.
-#
-

 #
-# build postgres separately -- this layer will be rebuilt only if one of
-# mentioned paths will get any changes
+# Build Postgres separately --- this layer will be rebuilt only if one of
+# mentioned paths will get any changes.
 #
-FROM alpine:3.13 as pg-build
-RUN apk add --update clang llvm compiler-rt compiler-rt-static lld musl-dev binutils \
-                     make bison flex readline-dev zlib-dev perl linux-headers libseccomp-dev
-WORKDIR zenith
+FROM zenithdb/build:buster AS pg-build
+WORKDIR /zenith
 COPY ./vendor/postgres vendor/postgres
 COPY ./Makefile Makefile
-# Build using clang and lld
-RUN CC='clang' LD='lld' CFLAGS='-fuse-ld=lld --rtlib=compiler-rt' make postgres -j4
+RUN make -j $(getconf _NPROCESSORS_ONLN) -s postgres

 #
 # Calculate cargo dependencies.
 # This will always run, but only generate recipe.json with list of dependencies without
 # installing them.
 #
-FROM alpine:20210212 as cargo-deps-inspect
-RUN apk add --update rust cargo
-RUN cargo install cargo-chef
-WORKDIR zenith
+FROM zenithdb/build:buster AS cargo-deps-inspect
+WORKDIR /zenith
 COPY . .
-RUN cargo chef prepare --recipe-path recipe.json
+RUN cargo chef prepare --recipe-path /zenith/recipe.json

 #
 # Build cargo dependencies.
-# This temp cantainner would be build only if recipe.json was changed.
+# This temp cantainner should be rebuilt only if recipe.json was changed.
 #
-FROM alpine:20210212 as deps-build
-RUN apk add --update rust cargo openssl-dev clang build-base
-# rust-rocksdb can be built against system-wide rocksdb -- that saves about
-# 10 minutes during build. Rocksdb apk package is in testing now, but use it
-# anyway. In case of any troubles we can download and build rocksdb here manually
-# (to cache it as a docker layer).
-RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
-WORKDIR zenith
+FROM zenithdb/build:buster AS deps-build
+WORKDIR /zenith
 COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
-COPY --from=cargo-deps-inspect /root/.cargo/bin/cargo-chef /root/.cargo/bin/
+COPY --from=cargo-deps-inspect /usr/local/cargo/bin/cargo-chef /usr/local/cargo/bin/
 COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
 RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json

 #
 # Build zenith binaries
 #
-FROM alpine:20210212 as build
-RUN apk add --update rust cargo openssl-dev clang build-base
-RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
-WORKDIR zenith
+FROM zenithdb/build:buster AS build
+WORKDIR /zenith
 COPY . .
 # Copy cached dependencies
 COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
 COPY --from=deps-build /zenith/target target
-COPY --from=deps-build /root/.cargo /root/.cargo
+COPY --from=deps-build /usr/local/cargo/ /usr/local/cargo/
 RUN cargo build --release

 #
 # Copy binaries to resulting image.
-# build-base hare to provide libstdc++ (it will also bring gcc, but leave it this way until we figure
-# out how to statically link rocksdb or avoid it at all).
 #
-FROM alpine:3.13
-RUN apk add --update openssl build-base libseccomp-dev
-RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb
+FROM debian:buster-slim
+WORKDIR /data
+
+RUN apt-get update && apt-get -yq install librocksdb-dev libseccomp-dev openssl && \
+    mkdir zenith_install
+
 COPY --from=build /zenith/target/release/pageserver /usr/local/bin
 COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin
 COPY --from=build /zenith/target/release/proxy /usr/local/bin
-COPY --from=pg-build /zenith/tmp_install /usr/local
+COPY --from=pg-build /zenith/tmp_install postgres_install
 COPY docker-entrypoint.sh /docker-entrypoint.sh

-RUN addgroup zenith && adduser -h /data -D -G zenith zenith
+# Remove build artifacts (~ 500 MB)
+RUN rm -rf postgres_install/build && \
+    # 'Install' Postgres binaries locally
+    cp -r postgres_install/* /usr/local/ && \
+    # Prepare an archive of Postgres binaries (should be around 11 MB)
+    # and keep it inside container for an ease of deploy pipeline.
+    cd postgres_install && tar -czf /data/postgres_install.tar.gz . && cd .. && \
+    rm -rf postgres_install
+
+RUN useradd -d /data zenith && chown -R zenith:zenith /data
+
 VOLUME ["/data"]
-WORKDIR /data
 USER zenith
 EXPOSE 6400
 ENTRYPOINT ["/docker-entrypoint.sh"]
--- a/Dockerfile.alpine
+++ b/Dockerfile.alpine
@@ -0,0 +1,95 @@
+#
+# Docker image for console integration testing.
+#
+# We may also reuse it in CI to unify installation process and as a general binaries building
+# tool for production servers.
+#
+# Dynamic linking is used for librocksdb and libstdc++ bacause librocksdb-sys calls
+# bindgen with "dynamic" feature flag. This also prevents usage of dockerhub alpine-rust
+# images which are statically linked and have guards against any dlopen. I would rather
+# prefer all static binaries so we may change the way librocksdb-sys builds or wait until
+# we will have our own storage and drop rockdb dependency.
+#
+# Cargo-chef is used to separate dependencies building from main binaries building. This
+# way `docker build` will download and install dependencies only of there are changes to
+# out Cargo.toml files.
+#
+
+
+#
+# build postgres separately -- this layer will be rebuilt only if one of
+# mentioned paths will get any changes
+#
+FROM alpine:3.13 as pg-build
+RUN apk add --update clang llvm compiler-rt compiler-rt-static lld musl-dev binutils \
+                     make bison flex readline-dev zlib-dev perl linux-headers libseccomp-dev
+WORKDIR zenith
+COPY ./vendor/postgres vendor/postgres
+COPY ./Makefile Makefile
+# Build using clang and lld
+RUN CC='clang' LD='lld' CFLAGS='-fuse-ld=lld --rtlib=compiler-rt' make postgres -j4
+
+#
+# Calculate cargo dependencies.
+# This will always run, but only generate recipe.json with list of dependencies without
+# installing them.
+#
+FROM alpine:20210212 as cargo-deps-inspect
+RUN apk add --update rust cargo
+RUN cargo install cargo-chef
+WORKDIR zenith
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+#
+# Build cargo dependencies.
+# This temp cantainner would be build only if recipe.json was changed.
+#
+FROM alpine:20210212 as deps-build
+RUN apk add --update rust cargo openssl-dev clang build-base
+# rust-rocksdb can be built against system-wide rocksdb -- that saves about
+# 10 minutes during build. Rocksdb apk package is in testing now, but use it
+# anyway. In case of any troubles we can download and build rocksdb here manually
+# (to cache it as a docker layer).
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
+WORKDIR zenith
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=cargo-deps-inspect /root/.cargo/bin/cargo-chef /root/.cargo/bin/
+COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
+RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json
+
+#
+# Build zenith binaries
+#
+FROM alpine:20210212 as build
+RUN apk add --update rust cargo openssl-dev clang build-base
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
+WORKDIR zenith
+COPY . .
+# Copy cached dependencies
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=deps-build /zenith/target target
+COPY --from=deps-build /root/.cargo /root/.cargo
+RUN cargo build --release
+
+#
+# Copy binaries to resulting image.
+# build-base hare to provide libstdc++ (it will also bring gcc, but leave it this way until we figure
+# out how to statically link rocksdb or avoid it at all).
+#
+FROM alpine:3.13
+RUN apk add --update openssl build-base libseccomp-dev
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb
+COPY --from=build /zenith/target/release/pageserver /usr/local/bin
+COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin
+COPY --from=build /zenith/target/release/proxy /usr/local/bin
+COPY --from=pg-build /zenith/tmp_install /usr/local
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+
+RUN addgroup zenith && adduser -h /data -D -G zenith zenith
+VOLUME ["/data"]
+WORKDIR /data
+USER zenith
+EXPOSE 6400
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["pageserver"]
--- a/Dockerfile.build
+++ b/Dockerfile.build
@@ -0,0 +1,15 @@
+#
+# Image with all the required dependencies to build https://github.com/zenithdb/zenith
+# and Postgres from https://github.com/zenithdb/postgres
+# Also includes some rust development and build tools.
+#
+FROM rust:slim-buster
+WORKDIR /zenith
+
+# Install postgres and zenith build dependencies
+# clang is for rocksdb
+RUN apt-get update && apt-get -yq install automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \
+                                          libseccomp-dev pkg-config libssl-dev librocksdb-dev clang
+
+# Install rust tools
+RUN rustup component add clippy && cargo install cargo-chef cargo-audit
--- a/14
+++ b/14
@@ -21,6 +21,7 @@ all: zenith postgres
 ### Zenith Rust bits
 #
 # The 'postgres_ffi' depends on the Postgres headers.
+.PHONY: zenith
 zenith: postgres-headers
 	cargo build

@@ -37,20 +38,25 @@ tmp_install/build/config.status:
 		--prefix=$(abspath tmp_install) > configure.log)

 # nicer alias for running 'configure'
+.PHONY: postgres-configure
 postgres-configure: tmp_install/build/config.status

 # Install the PostgreSQL header files into tmp_install/include
+.PHONY: postgres-headers
 postgres-headers: postgres-configure
 	+@echo "Installing PostgreSQL headers"
 	$(MAKE) -C tmp_install/build/src/include MAKELEVEL=0 install


 # Compile and install PostgreSQL and contrib/zenith
+.PHONY: postgres
 postgres: postgres-configure
 	+@echo "Compiling PostgreSQL"
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 install
 	+@echo "Compiling contrib/zenith"
 	$(MAKE) -C tmp_install/build/contrib/zenith install
+	+@echo "Compiling contrib/zenith_test_utils"
+	$(MAKE) -C tmp_install/build/contrib/zenith_test_utils install

 postgres-clean:
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 clean
@@ -65,4 +71,10 @@ distclean:
 	rm -rf tmp_install
 	cargo clean

-.PHONY: postgres-configure postgres postgres-headers zenith
+.PHONY: fmt
+fmt:
+	./pre-commit.py --fix-inplace
+
+.PHONY: setup-pre-commit-hook
+setup-pre-commit-hook:
+	ln -s -f ../../pre-commit.py .git/hooks/pre-commit
--- a/README.md
+++ b/README.md
@@ -2,6 +2,22 @@

 Zenith substitutes PostgreSQL storage layer and redistributes data across a cluster of nodes

+## Architecture overview
+
+A Zenith installation consists of Compute nodes and Storage engine.
+
+Compute nodes are stateles PostgreSQL nodes, backed by zenith storage.
+
+Zenith storage engine consists of two major components:
+- Pageserver. Scalable storage backend for compute nodes.
+- WAL service. The service that receives WAL from compute node and ensures that it is stored durably.
+
+Pageserver consists of:
+- Repository - Zenith storage implementation.
+- WAL receiver - service that receives WAL from WAL service and stores it in the repository.
+- Page service - service that communicates with compute nodes and responds with pages from the repository.
+- WAL redo - service that builds pages from base images and WAL records on Page service request.
+
 ## Running local installation

 1. Install build dependencies and other useful packages
@@ -12,15 +28,12 @@ apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libsec
 libssl-dev clang
 ```

-[Rust] 1.48 or later is also required.
+[Rust] 1.52 or later is also required.

 To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively.

 To run the integration tests (not required to use the code), install
-Python (3.6 or higher), and install python3 packages with `pip` (called `pip3` on some systems):
-```
-pip install pytest psycopg2
-```
+Python (3.6 or higher), and install python3 packages with `pipenv` using `pipenv install` in the project directory.

 2. Build zenith and patched postgres
 ```sh
@@ -54,7 +67,7 @@ main	127.0.0.1:55432	0/1609610	running

 4. Now it is possible to connect to postgres and run some queries:
 ```text
-> psql -p55432 -h 127.0.0.1 postgres
+> psql -p55432 -h 127.0.0.1 -U zenith_admin postgres
 postgres=# CREATE TABLE t(key int primary key, value text);
 CREATE TABLE
 postgres=# insert into t values(1,1);
@@ -84,7 +97,7 @@ waiting for server to start.... done

 # this new postgres instance will have all the data from 'main' postgres,
 # but all modifications would not affect data in original postgres
-> psql -p55433 -h 127.0.0.1 postgres
+> psql -p55433 -h 127.0.0.1 -U zenith_admin postgres
 postgres=# select * from t;
 key | value
 -----+-------
@@ -98,7 +111,7 @@ INSERT 0 1
 ## Running tests

 ```sh
-git clone --recursive https://github.com/libzenith/zenith.git
+git clone --recursive https://github.com/zenithdb/zenith.git
 make # builds also postgres and installs it to ./tmp_install
 cd test_runner
 pytest
@@ -106,55 +119,15 @@ pytest

 ## Documentation

-Now we use README files to cover design ideas and overall architecture for each module.
-And rustdoc style documentation comments.
+Now we use README files to cover design ideas and overall architecture for each module and `rustdoc` style documentation comments. See also [/docs/](/docs/) a top-level overview of all available markdown documentation.

-To view your documentation in a browser, try running `cargo doc --no-deps --open`
+- [/docs/sourcetree.md](/docs/sourcetree.md) contains overview of source tree layout.

-## Source tree layout
+To view your `rustdoc` documentation in a browser, try running `cargo doc --no-deps --open`

-`/control_plane`:
+## Join the development

-Local control plane.
-Functions to start, configure and stop pageserver and postgres instances running as a local processes.
-Intended to be used in integration tests and in CLI tools for local installations.
-
-`/zenith`
-
-Main entry point for the 'zenith' CLI utility.
-TODO: Doesn't it belong to control_plane?
-
-`/postgres_ffi`:
-
-Utility functions for interacting with PostgreSQL file formats.
-Misc constants, copied from PostgreSQL headers.
-
-`/zenith_utils`:
-
-Helpers that are shared between other crates in this repository.
-
-`/walkeeper`:
-
-WAL safekeeper (also known as WAL acceptor). Written in Rust.
-
-`/pageserver`:
-
-Page Server. Written in Rust.
-
-Depends on the modified 'postgres' binary for WAL redo.
-
-`/vendor/postgres`:
-
-PostgreSQL source tree, with the modifications needed for Zenith.
-
-`/vendor/postgres/contrib/zenith`:
-
-PostgreSQL extension that implements storage manager API and network communications with remote page server.
-
-`/test_runner`:
-
-Integration tests, written in Python using the `pytest` framework.
-
-`test_runner/zenith_regress`:
-
-Quick way to add new SQL regression test to integration tests set.
+- Read `CONTRIBUTING.md` to learn about project code style and practices.
+- Use glossary in [/docs/glossary.md](/docs/glossary.md)
+- To get familiar with a source tree layout, use [/docs/sourcetree.md](/docs/sourcetree.md).
+- To learn more about PostgreSQL internals, check http://www.interdb.jp/pg/index.html
--- a/control_plane/Cargo.toml
+++ b/control_plane/Cargo.toml
@@ -20,6 +20,7 @@ bytes = "1.0.1"
 nix = "0.20"
 url = "2.2.2"
 hex = { version = "0.4.3", features = ["serde"] }
+reqwest = { version = "0.11", features = ["blocking", "json"] }

 pageserver = { path = "../pageserver" }
 walkeeper = { path = "../walkeeper" }
--- a/control_plane/src/compute.rs
+++ b/control_plane/src/compute.rs
@@ -1,3 +1,4 @@
+use std::fs::{self, File, OpenOptions};
 use std::io::Write;
 use std::net::SocketAddr;
 use std::net::TcpStream;
@@ -6,19 +7,16 @@ use std::process::Command;
 use std::sync::Arc;
 use std::time::Duration;
 use std::{collections::BTreeMap, path::PathBuf};
-use std::{
-    fs::{self, OpenOptions},
-    io::Read,
-};

 use anyhow::{Context, Result};
 use lazy_static::lazy_static;
 use regex::Regex;
 use zenith_utils::connstring::connection_host_port;
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::ZTenantId;
+use zenith_utils::zid::ZTimelineId;

 use crate::local_env::LocalEnv;
-use pageserver::{ZTenantId, ZTimelineId};
-
 use crate::storage::PageServerNode;

 //
@@ -84,48 +82,36 @@ impl ComputeControlPlane {
        }
    }

-    /// Connect to a page server, get base backup, and untar it to initialize a
-    /// new data directory
-    pub fn new_from_page_server(
-        &mut self,
-        is_test: bool,
-        timelineid: ZTimelineId,
-        name: &str,
-        tenantid: ZTenantId,
-    ) -> Result<Arc<PostgresNode>> {
-        let node = Arc::new(PostgresNode {
-            name: name.to_owned(),
-            address: SocketAddr::new("127.0.0.1".parse().unwrap(), self.get_port()),
-            env: self.env.clone(),
-            pageserver: Arc::clone(&self.pageserver),
-            is_test,
-            timelineid,
-            tenantid,
-        });
-
-        node.init_from_page_server()?;
-        self.nodes
-            .insert((tenantid, node.name.clone()), Arc::clone(&node));
-
-        Ok(node)
-    }
-
    pub fn new_node(
        &mut self,
        tenantid: ZTenantId,
        branch_name: &str,
+        config_only: bool,
    ) -> Result<Arc<PostgresNode>> {
        let timeline_id = self
            .pageserver
            .branch_get_by_name(&tenantid, branch_name)?
            .timeline_id;
-        let node = self.new_from_page_server(false, timeline_id, branch_name, tenantid)?;
+
+        let node = Arc::new(PostgresNode {
+            name: branch_name.to_owned(),
+            address: SocketAddr::new("127.0.0.1".parse().unwrap(), self.get_port()),
+            env: self.env.clone(),
+            pageserver: Arc::clone(&self.pageserver),
+            is_test: false,
+            timelineid: timeline_id,
+            tenantid,
+        });
+
+        node.init_from_page_server(self.env.auth_type, config_only)?;
+        self.nodes
+            .insert((tenantid, node.name.clone()), Arc::clone(&node));
+
        // Configure the node to stream WAL directly to the pageserver
        node.append_conf(
            "postgresql.conf",
            format!(
                concat!(
-                    "shared_preload_libraries = zenith\n",
                    "synchronous_standby_names = 'pageserver'\n", // TODO: add a new function arg?
                    "zenith.callmemaybe_connstring = '{}'\n",     // FIXME escaping
                ),
@@ -245,9 +231,32 @@ impl PostgresNode {
        })
    }

-    // Connect to a page server, get base backup, and untar it to initialize a
-    // new data directory
-    pub fn init_from_page_server(&self) -> Result<()> {
+    /// Get basebackup from the pageserver as a tar archive and extract it
+    /// to the `self.pgdata()` directory.
+    pub fn do_basebackup(&self) -> Result<()> {
+        let pgdata = self.pgdata();
+
+        let sql = format!("basebackup {} {}", self.tenantid, self.timelineid);
+        let mut client = self
+            .pageserver
+            .page_server_psql_client()
+            .with_context(|| "connecting to page server failed")?;
+
+        let copyreader = client
+            .copy_out(sql.as_str())
+            .with_context(|| "page server 'basebackup' command failed")?;
+
+        // Read the archive directly from the `CopyOutReader`
+        tar::Archive::new(copyreader)
+            .unpack(&pgdata)
+            .with_context(|| "extracting page backup failed")?;
+
+        Ok(())
+    }
+
+    /// Connect to a pageserver, get basebackup, and untar it to initialize a
+    /// new data directory
+    pub fn init_from_page_server(&self, auth_type: AuthType, config_only: bool) -> Result<()> {
        let pgdata = self.pgdata();

        println!(
@@ -261,12 +270,6 @@ impl PostgresNode {
            fs::remove_dir_all(&pgdata).ok();
        }

-        let sql = format!("basebackup {} {}", self.tenantid, self.timelineid);
-        let mut client = self
-            .pageserver
-            .page_server_psql_client()
-            .with_context(|| "connecting to page server failed")?;
-
        fs::create_dir_all(&pgdata)
            .with_context(|| format!("could not create data directory {}", pgdata.display()))?;
        fs::set_permissions(pgdata.as_path(), fs::Permissions::from_mode(0o700)).with_context(
@@ -278,20 +281,14 @@ impl PostgresNode {
            },
        )?;

-        let mut copyreader = client
-            .copy_out(sql.as_str())
-            .with_context(|| "page server 'basebackup' command failed")?;
-
-        // FIXME: Currently, we slurp the whole tarball into memory, and then extract it,
-        // but we really should do this:
-        //let mut ar = tar::Archive::new(copyreader);
-        let mut buf = vec![];
-        copyreader
-            .read_to_end(&mut buf)
-            .with_context(|| "reading base backup from page server failed")?;
-        let mut ar = tar::Archive::new(buf.as_slice());
-        ar.unpack(&pgdata)
-            .with_context(|| "extracting page backup failed")?;
+        if config_only {
+            //Just create an empty config file
+            File::create(self.pgdata().join("postgresql.conf").to_str().unwrap())?;
+        } else {
+            self.do_basebackup()?;
+            fs::create_dir_all(self.pgdata().join("pg_wal"))?;
+            fs::create_dir_all(self.pgdata().join("pg_wal").join("archive_status"))?;
+        }

        // wal_log_hints is mandatory when running against pageserver (see gh issue#192)
        // TODO: is it possible to check wal_log_hints at pageserver side via XLOG_PARAMETER_CHANGE?
@@ -316,30 +313,35 @@ impl PostgresNode {

        // Never clean up old WAL. TODO: We should use a replication
        // slot or something proper, to prevent the compute node
-        // from removing WAL that hasn't been streamed to the safekeepr or
+        // from removing WAL that hasn't been streamed to the safekeeper or
        // page server yet. (gh issue #349)
        self.append_conf("postgresql.conf", "wal_keep_size='10TB'\n")?;

-        // Connect it to the page server.
+        // set up authentication
+        let password = if let AuthType::ZenithJWT = auth_type {
+            "$ZENITH_AUTH_TOKEN"
+        } else {
+            ""
+        };

        // Configure that node to take pages from pageserver
-        let (host, port) = connection_host_port(&self.pageserver.connection_config());
+        let (host, port) = connection_host_port(&self.pageserver.pg_connection_config);
        self.append_conf(
            "postgresql.conf",
            format!(
                concat!(
                    "shared_preload_libraries = zenith\n",
-                    "zenith.page_server_connstring = 'host={} port={}'\n",
+                    // $ZENITH_AUTH_TOKEN will be replaced with value from environment variable during compute pg startup
+                    // it is done this way because otherwise user will be able to retrieve the value using SHOW command or pg_settings
+                    "zenith.page_server_connstring = 'host={} port={} password={}'\n",
                    "zenith.zenith_timeline='{}'\n",
                    "zenith.zenith_tenant='{}'\n",
                ),
-                host, port, self.timelineid, self.tenantid,
+                host, port, password, self.timelineid, self.tenantid,
            )
            .as_str(),
        )?;

-        fs::create_dir_all(self.pgdata().join("pg_wal"))?;
-        fs::create_dir_all(self.pgdata().join("pg_wal").join("archive_status"))?;
        Ok(())
    }

@@ -368,45 +370,88 @@ impl PostgresNode {
        Ok(())
    }

-    fn pg_ctl(&self, args: &[&str]) -> Result<()> {
+    fn pg_ctl(&self, args: &[&str], auth_token: &Option<String>) -> Result<()> {
        let pg_ctl_path = self.env.pg_bin_dir().join("pg_ctl");
+        let mut cmd = Command::new(pg_ctl_path);
+        cmd.args(
+            [
+                &[
+                    "-D",
+                    self.pgdata().to_str().unwrap(),
+                    "-l",
+                    self.pgdata().join("pg.log").to_str().unwrap(),
+                    "-w", //wait till pg_ctl actually does what was asked
+                ],
+                args,
+            ]
+            .concat(),
+        )
+        .env_clear()
+        .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
+        .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap());
+
+        if let Some(token) = auth_token {
+            cmd.env("ZENITH_AUTH_TOKEN", token);
+        }
+        let pg_ctl = cmd.status().with_context(|| "pg_ctl failed")?;

-        let pg_ctl = Command::new(pg_ctl_path)
-            .args(
-                [
-                    &[
-                        "-D",
-                        self.pgdata().to_str().unwrap(),
-                        "-l",
-                        self.pgdata().join("pg.log").to_str().unwrap(),
-                        "-w", //wait till pg_ctl actually does what was asked
-                    ],
-                    args,
-                ]
-                .concat(),
-            )
-            .env_clear()
-            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .status()
-            .with_context(|| "pg_ctl failed")?;
        if !pg_ctl.success() {
            anyhow::bail!("pg_ctl failed");
        }
        Ok(())
    }

-    pub fn start(&self) -> Result<()> {
+    pub fn start(&self, auth_token: &Option<String>) -> Result<()> {
+        // Bail if the node already running.
+        if self.status() == "running" {
+            anyhow::bail!("The node is already running");
+        }
+
+        // 1. We always start compute node from scratch, so
+        // if old dir exists, preserve config files and drop the directory
+
+        // XXX Now we only use 'postgresql.conf'.
+        // If we will need 'pg_hba.conf', support it here too
+
+        let postgresql_conf_path = self.pgdata().join("postgresql.conf");
+        let postgresql_conf = fs::read(postgresql_conf_path.clone()).with_context(|| {
+            format!(
+                "failed to read config file in {}",
+                postgresql_conf_path.to_str().unwrap()
+            )
+        })?;
+
+        println!(
+            "Destroying postgres data directory '{}'",
+            self.pgdata().to_str().unwrap()
+        );
+        fs::remove_dir_all(&self.pgdata())?;
+
+        // 2. Create new node
+        self.init_from_page_server(self.env.auth_type, false)?;
+
+        // 3. Bring back config files
+
+        if let Ok(mut file) = OpenOptions::new()
+            .append(false)
+            .write(true)
+            .open(&postgresql_conf_path)
+        {
+            file.write_all(&postgresql_conf)?;
+            file.sync_all()?;
+        }
+
+        // 4. Finally start the compute node postgres
        println!("Starting postgres node at '{}'", self.connstr());
-        self.pg_ctl(&["start"])
+        self.pg_ctl(&["start"], auth_token)
    }

-    pub fn restart(&self) -> Result<()> {
-        self.pg_ctl(&["restart"])
+    pub fn restart(&self, auth_token: &Option<String>) -> Result<()> {
+        self.pg_ctl(&["restart"], auth_token)
    }

    pub fn stop(&self, destroy: bool) -> Result<()> {
-        self.pg_ctl(&["-m", "immediate", "stop"])?;
+        self.pg_ctl(&["-m", "immediate", "stop"], &None)?;
        if destroy {
            println!(
                "Destroying postgres data directory '{}'",
--- a/control_plane/src/local_env.rs
+++ b/control_plane/src/local_env.rs
@@ -4,14 +4,17 @@
 // Now it also provides init method which acts like a stub for proper installation
 // script which will use local paths.
 //
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, Context, Result};
 use hex;
-use pageserver::ZTenantId;
 use serde::{Deserialize, Serialize};
 use std::fs;
 use std::path::PathBuf;
+use std::process::{Command, Stdio};
 use std::{collections::BTreeMap, env};
 use url::Url;
+use zenith_utils::auth::{encode_from_key_path, Claims, Scope};
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::ZTenantId;

 pub type Remotes = BTreeMap<String, String>;

@@ -39,6 +42,15 @@ pub struct LocalEnv {
    #[serde(with = "hex")]
    pub tenantid: ZTenantId,

+    // jwt auth token used for communication with pageserver
+    pub auth_token: String,
+
+    // used to determine which auth type is used
+    pub auth_type: AuthType,
+
+    // used to issue tokens during e.g pg start
+    pub private_key_path: PathBuf,
+
    pub remotes: Remotes,
 }

@@ -85,7 +97,11 @@ fn base_path() -> PathBuf {
 //
 // Initialize a new Zenith repository
 //
-pub fn init(remote_pageserver: Option<&str>, tenantid: ZTenantId) -> Result<()> {
+pub fn init(
+    remote_pageserver: Option<&str>,
+    tenantid: ZTenantId,
+    auth_type: AuthType,
+) -> Result<()> {
    // check if config already exists
    let base_path = base_path();
    if base_path.exists() {
@@ -94,6 +110,7 @@ pub fn init(remote_pageserver: Option<&str>, tenantid: ZTenantId) -> Result<()>
            base_path.to_str().unwrap()
        );
    }
+    fs::create_dir(&base_path)?;

    // ok, now check that expected binaries are present

@@ -110,6 +127,44 @@ pub fn init(remote_pageserver: Option<&str>, tenantid: ZTenantId) -> Result<()>
        anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
    }

+    // generate keys for jwt
+    // openssl genrsa -out private_key.pem 2048
+    let private_key_path = base_path.join("auth_private_key.pem");
+    let keygen_output = Command::new("openssl")
+        .arg("genrsa")
+        .args(&["-out", private_key_path.to_str().unwrap()])
+        .arg("2048")
+        .stdout(Stdio::null())
+        .output()
+        .with_context(|| "failed to generate auth private key")?;
+    if !keygen_output.status.success() {
+        anyhow::bail!(
+            "openssl failed: '{}'",
+            String::from_utf8_lossy(&keygen_output.stderr)
+        );
+    }
+
+    let public_key_path = base_path.join("auth_public_key.pem");
+    // openssl rsa -in private_key.pem -pubout -outform PEM -out public_key.pem
+    let keygen_output = Command::new("openssl")
+        .arg("rsa")
+        .args(&["-in", private_key_path.to_str().unwrap()])
+        .arg("-pubout")
+        .args(&["-outform", "PEM"])
+        .args(&["-out", public_key_path.to_str().unwrap()])
+        .stdout(Stdio::null())
+        .output()
+        .with_context(|| "failed to generate auth private key")?;
+    if !keygen_output.status.success() {
+        anyhow::bail!(
+            "openssl failed: '{}'",
+            String::from_utf8_lossy(&keygen_output.stderr)
+        );
+    }
+
+    let auth_token =
+        encode_from_key_path(&Claims::new(None, Scope::PageServerApi), &private_key_path)?;
+
    let conf = if let Some(addr) = remote_pageserver {
        // check that addr is parsable
        let _uri = Url::parse(addr).map_err(|e| anyhow!("{}: {}", addr, e))?;
@@ -121,6 +176,9 @@ pub fn init(remote_pageserver: Option<&str>, tenantid: ZTenantId) -> Result<()>
            base_data_dir: base_path,
            remotes: BTreeMap::default(),
            tenantid,
+            auth_token,
+            auth_type,
+            private_key_path,
        }
    } else {
        // Find zenith binaries.
@@ -136,6 +194,9 @@ pub fn init(remote_pageserver: Option<&str>, tenantid: ZTenantId) -> Result<()>
            base_data_dir: base_path,
            remotes: BTreeMap::default(),
            tenantid,
+            auth_token,
+            auth_type,
+            private_key_path,
        }
    };

--- a/control_plane/src/storage.rs
+++ b/control_plane/src/storage.rs
@@ -5,17 +5,23 @@ use std::process::Command;
 use std::thread;
 use std::time::Duration;

-use anyhow::{anyhow, bail, Result};
+use anyhow::{anyhow, bail, ensure, Result};
 use nix::sys::signal::{kill, Signal};
 use nix::unistd::Pid;
-use pageserver::ZTenantId;
+use pageserver::http::models::{BranchCreateRequest, TenantCreateRequest};
 use postgres::{Config, NoTls};
+use reqwest::blocking::{Client, RequestBuilder};
+use reqwest::{IntoUrl, Method, StatusCode};
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::ZTenantId;

 use crate::local_env::LocalEnv;
 use crate::read_pidfile;
 use pageserver::branches::BranchInfo;
 use zenith_utils::connstring::connection_address;

+const HTTP_BASE_URL: &str = "http://127.0.0.1:9898/v1";
+
 //
 // Control routines for pageserver.
 //
@@ -24,33 +30,36 @@ use zenith_utils::connstring::connection_address;
 #[derive(Debug)]
 pub struct PageServerNode {
    pub kill_on_exit: bool,
-    pub connection_config: Option<Config>,
+    pub pg_connection_config: Config,
    pub env: LocalEnv,
+    pub http_client: Client,
+    pub http_base_url: String,
 }

 impl PageServerNode {
    pub fn from_env(env: &LocalEnv) -> PageServerNode {
+        let password = if env.auth_type == AuthType::ZenithJWT {
+            &env.auth_token
+        } else {
+            ""
+        };
+
        PageServerNode {
            kill_on_exit: false,
-            connection_config: None, // default
+            pg_connection_config: Self::default_config(password), // default
            env: env.clone(),
+            http_client: Client::new(),
+            http_base_url: HTTP_BASE_URL.to_owned(),
        }
    }

-    fn default_config() -> Config {
-        "postgresql://no_user@localhost:64000/no_db"
+    fn default_config(password: &str) -> Config {
+        format!("postgresql://no_user:{}@localhost:64000/no_db", password)
            .parse()
            .unwrap()
    }

-    pub fn connection_config(&self) -> Config {
-        match &self.connection_config {
-            Some(config) => config.clone(),
-            None => Self::default_config(),
-        }
-    }
-
-    pub fn init(&self, create_tenant: Option<&str>) -> Result<()> {
+    pub fn init(&self, create_tenant: Option<&str>, enable_auth: bool) -> Result<()> {
        let mut cmd = Command::new(self.env.pageserver_bin()?);
        let mut args = vec![
            "--init",
@@ -59,6 +68,12 @@ impl PageServerNode {
            "--postgres-distrib",
            self.env.pg_distrib_dir.to_str().unwrap(),
        ];
+
+        if enable_auth {
+            args.extend(&["--auth-validation-public-key-path", "auth_public_key.pem"]);
+            args.extend(&["--auth-type", "ZenithJWT"]);
+        }
+
        create_tenant.map(|tenantid| args.extend(&["--create-tenant", tenantid]));
        let status = cmd
            .args(args)
@@ -85,7 +100,7 @@ impl PageServerNode {
    pub fn start(&self) -> Result<()> {
        println!(
            "Starting pageserver at '{}' in {}",
-            connection_address(&self.connection_config()),
+            connection_address(&self.pg_connection_config),
            self.repo_path().display()
        );

@@ -105,18 +120,21 @@ impl PageServerNode {
        // It takes a while for the page server to start up. Wait until it is
        // open for business.
        for retries in 1..15 {
-            let client = self.page_server_psql_client();
-            if client.is_ok() {
-                break;
-            } else {
-                println!("Pageserver not responding yet, retrying ({})...", retries);
-                thread::sleep(Duration::from_secs(1));
+            match self.check_status() {
+                Ok(_) => {
+                    println!("Pageserver started");
+                    return Ok(());
+                }
+                Err(err) => {
+                    println!(
+                        "Pageserver not responding yet, err {} retrying ({})...",
+                        err, retries
+                    );
+                    thread::sleep(Duration::from_secs(1));
+                }
            }
        }
-
-        println!("Pageserver started");
-
-        Ok(())
+        bail!("pageserver failed to start");
    }

    pub fn stop(&self) -> Result<()> {
@@ -127,7 +145,7 @@ impl PageServerNode {
        }

        // wait for pageserver stop
-        let address = connection_address(&self.connection_config());
+        let address = connection_address(&self.pg_connection_config);
        for _ in 0..5 {
            let stream = TcpStream::connect(&address);
            thread::sleep(Duration::from_secs(1));
@@ -142,50 +160,64 @@ impl PageServerNode {
    }

    pub fn page_server_psql(&self, sql: &str) -> Vec<postgres::SimpleQueryMessage> {
-        let mut client = self.connection_config().connect(NoTls).unwrap();
+        let mut client = self.pg_connection_config.connect(NoTls).unwrap();

        println!("Pageserver query: '{}'", sql);
        client.simple_query(sql).unwrap()
    }

    pub fn page_server_psql_client(&self) -> Result<postgres::Client, postgres::Error> {
-        self.connection_config().connect(NoTls)
+        self.pg_connection_config.connect(NoTls)
    }

-    pub fn tenants_list(&self) -> Result<Vec<String>> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query("tenant_list")?;
-        let tenants_json = query_result
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
-            })
-            .flatten()
-            .ok_or_else(|| anyhow!("missing tenants"))?;
-
-        Ok(serde_json::from_str(tenants_json)?)
+    fn http_request<U: IntoUrl>(&self, method: Method, url: U) -> RequestBuilder {
+        let mut builder = self.http_client.request(method, url);
+        if self.env.auth_type == AuthType::ZenithJWT {
+            builder = builder.bearer_auth(&self.env.auth_token)
+        }
+        builder
    }

-    pub fn tenant_create(&self, tenantid: &ZTenantId) -> Result<()> {
-        let mut client = self.page_server_psql_client()?;
-        client.simple_query(format!("tenant_create {}", tenantid).as_str())?;
+    pub fn check_status(&self) -> Result<()> {
+        let status = self
+            .http_request(Method::GET, format!("{}/{}", self.http_base_url, "status"))
+            .send()?
+            .status();
+        ensure!(
+            status == StatusCode::OK,
+            format!("got unexpected response status {}", status)
+        );
        Ok(())
    }

-    pub fn branches_list(&self, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query(&format!("branch_list {}", tenantid))?;
-        let branches_json = query_result
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
-            })
-            .flatten()
-            .ok_or_else(|| anyhow!("missing branches"))?;
+    pub fn tenant_list(&self) -> Result<Vec<String>> {
+        Ok(self
+            .http_request(Method::GET, format!("{}/{}", self.http_base_url, "tenant"))
+            .send()?
+            .error_for_status()?
+            .json()?)
+    }

-        Ok(serde_json::from_str(branches_json)?)
+    pub fn tenant_create(&self, tenantid: ZTenantId) -> Result<()> {
+        Ok(self
+            .http_request(Method::POST, format!("{}/{}", self.http_base_url, "tenant"))
+            .json(&TenantCreateRequest {
+                tenant_id: tenantid,
+            })
+            .send()?
+            .error_for_status()?
+            .json()?)
+    }
+
+    pub fn branch_list(&self, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
+        Ok(self
+            .http_request(
+                Method::GET,
+                format!("{}/branch/{}", self.http_base_url, tenantid),
+            )
+            .send()?
+            .error_for_status()?
+            .json()?)
    }

    pub fn branch_create(
@@ -194,29 +226,16 @@ impl PageServerNode {
        startpoint: &str,
        tenantid: &ZTenantId,
    ) -> Result<BranchInfo> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query(
-            format!("branch_create {} {} {}", tenantid, branch_name, startpoint).as_str(),
-        )?;
-
-        let branch_json = query_result
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
+        Ok(self
+            .http_request(Method::POST, format!("{}/{}", self.http_base_url, "branch"))
+            .json(&BranchCreateRequest {
+                tenant_id: tenantid.to_owned(),
+                name: branch_name.to_owned(),
+                start_point: startpoint.to_owned(),
            })
-            .flatten()
-            .ok_or_else(|| anyhow!("missing branch"))?;
-
-        let res: BranchInfo = serde_json::from_str(branch_json).map_err(|e| {
-            anyhow!(
-                "failed to parse branch_create response: {}: {}",
-                branch_json,
-                e
-            )
-        })?;
-
-        Ok(res)
+            .send()?
+            .error_for_status()?
+            .json()?)
    }

    // TODO: make this a separate request type and avoid loading all the branches
@@ -225,14 +244,14 @@ impl PageServerNode {
        tenantid: &ZTenantId,
        branch_name: &str,
    ) -> Result<BranchInfo> {
-        let branch_infos = self.branches_list(tenantid)?;
-        let branche_by_name: Result<HashMap<String, BranchInfo>> = branch_infos
+        let branch_infos = self.branch_list(tenantid)?;
+        let branch_by_name: Result<HashMap<String, BranchInfo>> = branch_infos
            .into_iter()
            .map(|branch_info| Ok((branch_info.name.clone(), branch_info)))
            .collect();
-        let branche_by_name = branche_by_name?;
+        let branch_by_name = branch_by_name?;

-        let branch = branche_by_name
+        let branch = branch_by_name
            .get(branch_name)
            .ok_or_else(|| anyhow!("Branch {} not found", branch_name))?;

--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -1,6 +1,8 @@
 #!/bin/sh
+set -eux
+
 if [ "$1" = 'pageserver' ]; then
-    if [ ! -d "/data/timelines" ]; then
+    if [ ! -d "/data/tenants" ]; then
        echo "Initializing pageserver data directory"
        pageserver --init -D /data --postgres-distrib /usr/local
    fi
--- a/docs/README.md
+++ b/docs/README.md
@@ -0,0 +1,13 @@
+# Zenith documentation
+
+## Table of contents
+
+- [authentication.md](authentication.md) — pageserver JWT authentication.
+- [docker.md](docker.md) — Docker images and building pipeline.
+- [glossary.md](glossary.md) — Glossary of all the terms used in codebase.
+- [multitenancy.md](multitenancy.md) — how multitenancy is organized in the pageserver and Zenith CLI.
+- [sourcetree.md](sourcetree.md) — Overview of the source tree layeout.
+- [pageserver/README](/pageserver/README) — pageserver overview.
+- [postgres_ffi/README](/postgres_ffi/README) — Postgres FFI overview.
+- [test_runner/README.md](/test_runner/README.md) — tests infrastructure overview.
+- [walkeeper/README](/walkeeper/README.md) — WAL service overview.
--- a/docs/authentication.md
+++ b/docs/authentication.md
@@ -0,0 +1,30 @@
+## Authentication
+
+### Overview
+
+Current state of authentication includes usage of JWT tokens in communication between compute and pageserver and between CLI and pageserver. JWT token is signed using RSA keys. CLI generates a key pair during call to `zenith init`. Using following openssl commands:
+
+```bash
+openssl genrsa -out private_key.pem 2048
+openssl rsa -in private_key.pem -pubout -outform PEM -out public_key.pem
+```
+
+CLI also generates signed token and saves it in the config for later access to pageserver. Now authentication is optional. Pageserver has two variables in config: `auth_validation_public_key_path` and `auth_type`, so when auth type present and set to `ZenithJWT` pageserver will require authentication for connections. Actual JWT is passed in password field of connection string. There is a caveat for psql, it silently truncates passwords to 100 symbols, so to correctly pass JWT via psql you have to either use PGPASSWORD environment variable, or store password in psql config file.
+
+Currently there is no authentication between compute and safekeepers, because this communication layer is under heavy refactoring. After this refactoring support for authentication will be added there too. Now safekeeper supports "hardcoded" token passed via environment variable to be able to use callmemaybe command in pageserver.
+
+Compute uses token passed via environment variable to communicate to pageserver and in the future to the safekeeper too.
+
+JWT authentication now supports two scopes: tenant and pageserverapi. Tenant scope is intended for use in tenant related api calls, e.g. create_branch. Compute launched for particular tenant also uses this scope. Scope pageserver api is intended to be used by console to manage pageserver. For now we have only one management operation - create tenant.
+
+Examples for token generation in python:
+
+```python
+# generate pageserverapi token
+management_token = jwt.encode({"scope": "pageserverapi"}, auth_keys.priv, algorithm="RS256")
+
+# generate tenant token
+tenant_token = jwt.encode({"scope": "tenant", "tenant_id": ps.initial_tenant}, auth_keys.priv, algorithm="RS256")
+```
+
+Utility functions to work with jwts in rust are located in zenith_utils/src/auth.rs
--- a/docs/docker.md
+++ b/docs/docker.md
@@ -0,0 +1,38 @@
+# Docker images of Zenith
+
+## Images
+
+Currently we build two main images:
+
+- [zenithdb/zenith](https://hub.docker.com/repository/docker/zenithdb/zenith) — image with pre-built `pageserver`, `wal_acceptor` and `proxy` binaries and all the required runtime dependencies. Built from [/Dockerfile](/Dockerfile).
+- [zenithdb/compute-node](https://hub.docker.com/repository/docker/zenithdb/compute-node) — compute node image with pre-built Postgres binaries from [zenithdb/postgres](https://github.com/zenithdb/postgres).
+
+And two intermediate images used either to reduce build time or to deliver some additional binary tools from other repos:
+
+- [zenithdb/build](https://hub.docker.com/repository/docker/zenithdb/build) — image with all the dependencies required to build Zenith and compute node images. This image is based on `rust:slim-buster`, so it also has a proper `rust` environment. Built from [/Dockerfile.build](/Dockerfile.build).
+- [zenithdb/compute-tools](https://hub.docker.com/repository/docker/zenithdb/compute-tools) — compute node configuration management tools.
+
+## Building pipeline
+
+1. Image `zenithdb/compute-tools` is re-built automatically.
+
+2. Image `zenithdb/build` is built manually. If you want to introduce any new compile time dependencies to Zenith or compute node you have to update this image as well, build it and push to Docker Hub.
+
+Build:
+```sh
+docker build -t zenithdb/build:buster -f Dockerfile.build .
+```
+
+Login:
+```sh
+docker login
+```
+
+Push to Docker Hub:
+```sh
+docker push zenithdb/build:buster
+```
+
+3. Image `zenithdb/compute-node` is built independently in the [zenithdb/postgres](https://github.com/zenithdb/postgres) repo.
+
+4. Image `zenithdb/zenith` is built in this repo after a successful `release` tests run and pushed to Docker Hub automatically.
--- a/docs/glossary.md
+++ b/docs/glossary.md
@@ -0,0 +1,185 @@
+# Glossary
+
+### Authentication
+
+### Base image (page image)
+
+### Basebackup
+
+A tarball with files needed to bootstrap a compute node[] and a corresponding command to create it.
+NOTE:It has nothing to do with PostgreSQL pg_basebackup.
+
+### Branch
+
+We can create branch at certain LSN using `zenith branch` command.
+Each Branch lives in a corresponding timeline[] and has an ancestor[].
+
+
+### Checkpoint (PostgreSQL)
+
+NOTE: This is an overloaded term.
+
+A checkpoint record in the WAL marks a point in the WAL sequence at which it is guaranteed that all data files have been updated with all information from shared memory modified before that checkpoint; 
+
+### Checkpoint (Layered repository)
+
+NOTE: This is an overloaded term.
+
+Whenever enough WAL has been accumulated in memory, the page server []
+writes out the changes in memory into new layer files[]. This process
+is called "checkpointing". The page server only creates layer files for
+relations that have been modified since the last checkpoint. 
+
+### Compute node
+
+Stateless Postgres node that stores data in pageserver.
+
+### Garbage collection
+
+### Fork
+
+Each of the separate segmented file sets in which a relation is stored. The main fork is where the actual data resides. There also exist two secondary forks for metadata: the free space map and the visibility map.
+Each PostgreSQL fork is considered a separate relish.
+
+### Layer file
+
+Layered repository on-disk format is based on immutable files.  The
+files are called "layer files". Each file corresponds to one 10 MB
+segment of a PostgreSQL relation fork. There are two kinds of layer
+files: image files and delta files. An image file contains a
+"snapshot" of the segment at a particular LSN, and a delta file
+contains WAL records applicable to the segment, in a range of LSNs.
+
+### Layered repository
+
+### LSN
+
+
+### Page (block)
+
+The basic structure used to store relation data. All pages are of the same size.
+This is the unit of data exchange between compute node and pageserver.
+
+### Pageserver
+
+Zenith storage engine: repositories + wal receiver + page service + wal redo.
+
+### Page service
+
+The Page Service listens for GetPage@LSN requests from the Compute Nodes,
+and responds with pages from the repository.
+
+
+### PITR (Point-in-time-recovery)
+
+PostgreSQL's ability to restore up to a specified LSN.
+
+### Primary node
+
+
+### Proxy
+
+Postgres protocol proxy/router.
+This service listens psql port, can check auth via external service
+and create new databases and accounts (control plane API in our case).
+
+### Relation
+
+The generic term in PostgreSQL for all objects in a database that have a name and a list of attributes defined in a specific order.
+
+### Relish
+
+We call each relation and other file that is stored in the
+repository a "relish". It comes from "rel"-ish, as in "kind of a
+rel", because it covers relations as well as other things that are
+not relations, but are treated similarly for the purposes of the
+storage layer.
+
+### Replication slot
+
+
+### Replica node
+
+
+### Repository
+
+Repository stores multiple timelines, forked off from the same initial call to 'initdb'
+and has associated WAL redo service.
+One repository corresponds to one Tenant.
+
+### Retention policy
+
+How much history do we need to keep around for PITR and read-only nodes?
+
+### SLRU
+
+SLRUs include pg_clog, pg_multixact/members, and
+pg_multixact/offsets. There are other SLRUs in PostgreSQL, but
+they don't need to be stored permanently (e.g. pg_subtrans),
+or we do not support them in zenith yet (pg_commit_ts).
+Each SLRU segment is considered a separate relish[].
+
+### Tenant (Multitenancy)
+Tenant represents a single customer, interacting with Zenith.
+Wal redo[] activity, timelines[], snapshots[] are managed for each tenant independently.
+One pageserver[] can serve multiple tenants at once.
+One safekeeper 
+
+See `docs/multitenancy.md` for more.
+
+### Timeline
+
+Timeline accepts page changes and serves get_page_at_lsn() and
+get_rel_size() requests. The term "timeline" is used internally
+in the system, but to users they are exposed as "branches", with
+human-friendly names.
+
+NOTE: this has nothing to do with PostgreSQL WAL timelines.
+
+### XLOG
+
+PostgreSQL alias for WAL[].
+
+### WAL (Write-ahead log)
+
+The journal that keeps track of the changes in the database cluster as user- and system-invoked operations take place. It comprises many individual WAL records[] written sequentially to WAL files[].
+
+### WAL acceptor, WAL proposer
+
+In the context of the consensus algorithm, the Postgres
+compute node is also known as the WAL proposer, and the safekeeper is also known
+as the acceptor. Those are the standard terms in the Paxos algorithm.
+
+### WAL receiver (WAL decoder)
+
+The WAL receiver connects to the external WAL safekeeping service (or
+directly to the primary) using PostgreSQL physical streaming
+replication, and continuously receives WAL. It decodes the WAL records,
+and stores them to the repository.
+
+We keep one WAL receiver active per timeline.
+
+### WAL record
+
+A low-level description of an individual data change.
+
+### WAL redo
+
+A service that runs PostgreSQL in a special wal_redo mode
+to apply given WAL records over an old page image and return new page image.
+
+### WAL safekeeper
+
+One node that participates in the quorum. All the safekeepers
+together form the WAL service.
+
+### WAL segment (WAL file)
+
+Also known as WAL segment or WAL segment file. Each of the sequentially-numbered files that provide storage space for WAL. The files are all of the same predefined size and are written in sequential order, interspersing changes as they occur in multiple simultaneous sessions.
+
+### WAL service
+
+The service as whole that ensures that WAL is stored durably.
+
+### Web console
+
--- a/docs/sourcetree.md
+++ b/docs/sourcetree.md
@@ -0,0 +1,81 @@
+## Source tree layout
+
+Below you will find a brief overview of each subdir in the source tree in alphabetical order.
+
+`/control_plane`:
+
+Local control plane.
+Functions to start, configure and stop pageserver and postgres instances running as a local processes.
+Intended to be used in integration tests and in CLI tools for local installations.
+
+`/docs`:
+
+Documentaion of the Zenith features and concepts.
+Now it is mostly dev documentation.
+
+`/monitoring`:
+
+TODO
+
+`/pageserver`:
+
+Zenith storage service.
+The pageserver has a few different duties:
+
+- Store and manage the data.
+- Generate a tarball with files needed to bootstrap ComputeNode.
+- Respond to GetPage@LSN requests from the Compute Nodes.
+- Receive WAL from the WAL service and decode it.
+- Replay WAL that's applicable to the chunks that the Page Server maintains
+
+For more detailed info, see `/pageserver/README`
+
+`/postgres_ffi`:
+
+Utility functions for interacting with PostgreSQL file formats.
+Misc constants, copied from PostgreSQL headers.
+
+`/proxy`:
+
+Postgres protocol proxy/router.
+This service listens psql port, can check auth via external service
+and create new databases and accounts (control plane API in our case).
+
+`/test_runner`:
+
+Integration tests, written in Python using the `pytest` framework.
+
+`/vendor/postgres`:
+
+PostgreSQL source tree, with the modifications needed for Zenith.
+
+`/vendor/postgres/contrib/zenith`:
+
+PostgreSQL extension that implements storage manager API and network communications with remote page server.
+
+`/vendor/postgres/contrib/zenith_test_utils`:
+
+PostgreSQL extension that contains functions needed for testing and debugging.
+
+`/walkeeper`:
+
+The zenith WAL service that receives WAL from a primary compute nodes and streams it to the pageserver.
+It acts as a holding area and redistribution center for recently generated WAL.
+
+For more detailed info, see `/walkeeper/README`
+
+`/workspace_hack`:
+The workspace_hack crate exists only to pin down some dependencies.
+
+`/zenith`
+
+Main entry point for the 'zenith' CLI utility.
+TODO: Doesn't it belong to control_plane?
+
+`/zenith_metrics`:
+
+Helpers for exposing Prometheus metrics from the server.
+
+`/zenith_utils`:
+
+Helpers that are shared between other crates in this repository.
--- a/monitoring/docker-compose.yml
+++ b/monitoring/docker-compose.yml
@@ -0,0 +1,25 @@
+version: "3"
+services:
+
+  prometheus:
+    container_name: prometheus
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
+    # ports:
+    #   - "9090:9090"
+    # TODO: find a proper portable solution
+    network_mode: "host"
+
+  grafana:
+    image: grafana/grafana:latest
+    volumes:
+      - ./grafana.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+    # ports:
+    #   - "3000:3000"
+    # TODO: find a proper portable solution
+    network_mode: "host"
--- a/monitoring/grafana.yaml
+++ b/monitoring/grafana.yaml
@@ -0,0 +1,12 @@
+apiVersion: 1
+
+datasources:
+- name: Prometheus
+  type: prometheus
+  access: proxy
+  orgId: 1
+  url: http://localhost:9090
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: false
--- a/monitoring/prometheus.yaml
+++ b/monitoring/prometheus.yaml
@@ -0,0 +1,5 @@
+scrape_configs:
+  - job_name: 'default'
+    scrape_interval: 10s
+    static_configs:
+      - targets: ['localhost:9898']
--- a/pageserver/Cargo.toml
+++ b/pageserver/Cargo.toml
@@ -7,42 +7,39 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

 [dependencies]
-bookfile = "^0.2"
+bookfile = "^0.3"
 chrono = "0.4.19"
 rand = "0.8.3"
 regex = "1.4.5"
 bytes = { version = "1.0.1", features = ['serde'] }
 byteorder = "1.4.3"
 futures = "0.3.13"
+hyper = "0.14"
 lazy_static = "1.4.0"
 slog-stdlog = "4.1.0"
-slog-async = "2.6.0"
 slog-scope = "4.4.0"
 slog-term = "2.8.0"
 slog = "2.7.0"
 log = "0.4.14"
 clap = "2.33.0"
 daemonize = "0.4.1"
-rust-s3 = { version = "0.27.0-rc4", features = ["no-verify-ssl"] }
-tokio = { version = "1.3.0", features = ["full"] }
-tokio-stream = { version = "0.1.4" }
+tokio = { version = "1.5.0", features = ["full"] }
 postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-# by default rust-rocksdb tries to build a lot of compression algos. Use lz4 only for now as it is simplest dependency.
-rocksdb = { version = "0.16.0", features = ["lz4"], default-features = false }
+routerify = "2"
 anyhow = "1.0"
 crc32c = "0.6.0"
-walkdir = "2"
 thiserror = "1.0"
-hex = "0.4.3"
+hex = { version = "0.4.3", features = ["serde"] }
 tar = "0.4.33"
 humantime = "2.1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
-fs_extra = "1.2.0"
 toml = "0.5"
+scopeguard = "1.1.0"

 postgres_ffi = { path = "../postgres_ffi" }
+zenith_metrics = { path = "../zenith_metrics" }
 zenith_utils = { path = "../zenith_utils" }
 workspace_hack = { path = "../workspace_hack" }
--- a/pageserver/README
+++ b/pageserver/README
@@ -1,16 +1,5 @@
 ## Page server architecture

-The Page Server is responsible for all operations on a number of
-"chunks" of relation data. A chunk corresponds to a PostgreSQL
-relation segment (i.e. one max. 1 GB file in the data directory), but
-it holds all the different versions of every page in the segment that
-are still needed by the system.
-
-Currently we do not specifically organize data in chunks.
-All page images and corresponding WAL records are stored as entries in a key-value storage,
-where StorageKey is a zenith_timeline_id + BufferTag + LSN.
-
-
 The Page Server has a few different duties:

 - Respond to GetPage@LSN requests from the Compute Nodes
@@ -19,10 +8,11 @@ The Page Server has a few different duties:
 - Backup to S3


-The Page Server consists of multiple threads that operate on a shared
-cache of page versions:


+The Page Server consists of multiple threads that operate on a shared
+repository of page versions:
+
                                           | WAL
                                           V
                                   +--------------+
@@ -34,16 +24,14 @@ cache of page versions:
                  +---------+                              ..........            |    |
                  |         |                              .        .            |    |
 GetPage@LSN      |         |                              . backup .  ------->  | S3 |
------------->    |  Page   |         page cache           .        .            |    |
+------------->    |  Page   |         repository           .        .            |    |
                  | Service |                              ..........            |    |
   page           |         |                                                    +----+
 <-------------    |         |
-                  +---------+
-
-                             ...................................
-                             .                                 .
-                             . Garbage Collection / Compaction .
-                             ...................................
+                  +---------+      +--------------------+
+		                   |   Checkpointing /  |
+				   | Garbage collection |
+                                   +--------------------+

 Legend:

@@ -63,7 +51,7 @@ Page Service
 ------------

 The Page Service listens for GetPage@LSN requests from the Compute Nodes,
-and responds with pages from the page cache.
+and responds with pages from the repository.


 WAL Receiver
@@ -72,47 +60,59 @@ WAL Receiver
 The WAL receiver connects to the external WAL safekeeping service (or
 directly to the primary) using PostgreSQL physical streaming
 replication, and continuously receives WAL. It decodes the WAL records,
-and stores them to the page cache.
+and stores them to the repository.


-Page Cache
+Repository
 ----------

-The Page Cache is a switchboard to access different Repositories.
+The repository stores all the page versions, or WAL records needed to
+reconstruct them. Each tenant has a separate Repository, which is
+stored in the .zenith/tenants/<tenantid> directory.

-#### Repository
-Repository corresponds to one .zenith directory.
-Repository is needed to manage Timelines.
+Repository is an abstract trait, defined in `repository.rs`. It is
+implemented by the LayeredRepository object in
+`layered_repository.rs`. There is only that one implementation of the
+Repository trait, but it's still a useful abstraction that keeps the
+interface for the low-level storage functionality clean. The layered
+storage format is described in layered_repository/README.md.

-#### Timeline
-Timeline is a page cache workhorse that accepts page changes
-and serves get_page_at_lsn() and get_rel_size() requests.
-Note: this has nothing to do with PostgreSQL WAL timeline.
+Each repository consists of multiple Timelines. Timeline is a
+workhorse that accepts page changes from the WAL, and serves
+get_page_at_lsn() and get_rel_size() requests. Note: this has nothing
+to do with PostgreSQL WAL timeline. The term "timeline" is mostly
+interchangeable with "branch", there is a one-to-one mapping from
+branch to timeline. A timeline has a unique ID within the tenant,
+represented as 16-byte hex string that never changes, whereas a
+branch is a user-given name for a timeline.

-#### Branch
-We can create branch at certain LSN.
-Each Branch lives in a corresponding timeline and has an ancestor.
-
-To get full snapshot of data at certain moment we need to traverse timeline and its ancestors.
-
-#### ObjectRepository
-ObjectRepository implements Repository and has associated ObjectStore and WAL redo service.
-
-#### ObjectStore
-ObjectStore is an interface for key-value store for page images and wal records.
-Currently it has one implementation - RocksDB.
-
-#### WAL redo service
-WAL redo service - service that runs PostgreSQL in a special wal_redo mode
-to apply given WAL records over an old page image and return new page image.
+Each repository also has a WAL redo manager associated with it, see
+`walredo.rs`. The WAL redo manager is used to replay PostgreSQL WAL
+records, whenever we need to reconstruct a page version from WAL to
+satisfy a GetPage@LSN request, or to avoid accumulating too much WAL
+for a page. The WAL redo manager uses a Postgres process running in
+special zenith wal-redo mode to do the actual WAL redo, and
+communicates with the process using a pipe.


-TODO: Garbage Collection / Compaction
-------------------------------------
+Checkpointing / Garbage Collection
+----------------------------------

-Periodically, the Garbage Collection / Compaction thread runs
-and applies pending WAL records, and removes old page versions that
-are no longer needed.
+Periodically, the checkpointer thread wakes up and performs housekeeping
+duties on the repository. It has two duties:
+
+### Checkpointing
+
+Flush WAL that has accumulated in memory to disk, so that the old WAL
+can be truncated away in the WAL safekeepers. Also, to free up memory
+for receiving new WAL. This process is called "checkpointing". It's
+similar to checkpointing in PostgreSQL or other DBMSs, but in the page
+server, checkpointing happens on a per-segment basis.
+
+### Garbage collection
+
+Remove old on-disk layer files that are no longer needed according to the
+PITR retention policy


 TODO: Backup service
--- a/pageserver/src/basebackup.rs
+++ b/pageserver/src/basebackup.rs
@@ -50,7 +50,6 @@ impl<'a> Basebackup<'a> {
    }

    pub fn send_tarball(&mut self) -> anyhow::Result<()> {
-
        // Create pgdata subdirs structure
        for dir in pg_constants::PGDATA_SUBDIRS.iter() {
            info!("send subdir {:?}", *dir);
@@ -71,9 +70,6 @@ impl<'a> Basebackup<'a> {
        }

        // Gather non-relational files from object storage pages.
-        // Iteration is sorted order: all objects of the same type are grouped and traversed
-        // in key ascending order. For example all pg_xact records precede pg_multixact records and are sorted by block number.
-        // It allows to easily construct SLRU segments.
        for obj in self.timeline.list_nonrels(self.lsn)? {
            match obj {
                RelishTag::Slru { slru, segno } => {
@@ -97,12 +93,23 @@ impl<'a> Basebackup<'a> {
    }

    //
-    // Generate SLRU segment files from repository. Path identifies SLRU kind (pg_xact, pg_multixact/members, ...).
+    // Generate SLRU segment files from repository.
    //
    fn add_slru_segment(&mut self, slru: SlruKind, segno: u32) -> anyhow::Result<()> {
-        let nblocks = self
+        let seg_size = self
            .timeline
-            .get_rel_size(RelishTag::Slru { slru, segno }, self.lsn)?;
+            .get_relish_size(RelishTag::Slru { slru, segno }, self.lsn)?;
+
+        if seg_size == None {
+            trace!(
+                "SLRU segment {}/{:>04X} was truncated",
+                slru.to_str(),
+                segno
+            );
+            return Ok(());
+        }
+
+        let nblocks = seg_size.unwrap();

        let mut slru_buf: Vec<u8> =
            Vec::with_capacity(nblocks as usize * pg_constants::BLCKSZ as usize);
@@ -117,15 +124,11 @@ impl<'a> Basebackup<'a> {
            slru_buf.extend_from_slice(&img);
        }

-        let dir = match slru {
-            SlruKind::Clog => "pg_xact",
-            SlruKind::MultiXactMembers => "pg_multixact/members",
-            SlruKind::MultiXactOffsets => "pg_multixact/offsets",
-        };
-
-        let segname = format!("{}/{:>04X}", dir, segno);
+        let segname = format!("{}/{:>04X}", slru.to_str(), segno);
        let header = new_tar_header(&segname, slru_buf.len() as u64)?;
        self.ar.append(&header, slru_buf.as_slice())?;
+
+        trace!("Added to basebackup slru {} relsize {}", segname, nblocks);
        Ok(())
    }

@@ -176,21 +179,18 @@ impl<'a> Basebackup<'a> {
    // Extract twophase state files
    //
    fn add_twophase_file(&mut self, xid: TransactionId) -> anyhow::Result<()> {
-        // Include in tarball two-phase files only of in-progress transactions
-        if self.timeline.get_tx_status(xid, self.lsn)?
-            == pg_constants::TRANSACTION_STATUS_IN_PROGRESS
-        {
-            let img =
-                self.timeline
-                    .get_page_at_lsn_nowait(RelishTag::TwoPhase { xid }, 0, self.lsn)?;
-            let mut buf = BytesMut::new();
-            buf.extend_from_slice(&img[..]);
-            let crc = crc32c::crc32c(&img[..]);
-            buf.put_u32_le(crc);
-            let path = format!("pg_twophase/{:>08X}", xid);
-            let header = new_tar_header(&path, buf.len() as u64)?;
-            self.ar.append(&header, &buf[..])?;
-        }
+        let img = self
+            .timeline
+            .get_page_at_lsn_nowait(RelishTag::TwoPhase { xid }, 0, self.lsn)?;
+
+        let mut buf = BytesMut::new();
+        buf.extend_from_slice(&img[..]);
+        let crc = crc32c::crc32c(&img[..]);
+        buf.put_u32_le(crc);
+        let path = format!("pg_twophase/{:>08X}", xid);
+        let header = new_tar_header(&path, buf.len() as u64)?;
+        self.ar.append(&header, &buf[..])?;
+
        Ok(())
    }

--- a/pageserver/src/bin/pageserver.rs
+++ b/pageserver/src/bin/pageserver.rs
@@ -9,21 +9,24 @@ use std::{
    net::TcpListener,
    path::{Path, PathBuf},
    process::exit,
+    str::FromStr,
    thread,
    time::Duration,
 };
+use zenith_utils::{auth::JwtAuth, postgres_backend::AuthType};

-use anyhow::Result;
+use anyhow::{ensure, Result};
 use clap::{App, Arg, ArgMatches};
 use daemonize::Daemonize;

-use pageserver::{branches, logger, page_cache, page_service};
-use pageserver::{PageServerConf, RepositoryFormat};
+use pageserver::{branches, http, logger, page_service, tenant_mgr, PageServerConf};
+use zenith_utils::http::endpoint;

 const DEFAULT_LISTEN_ADDR: &str = "127.0.0.1:64000";
+const DEFAULT_HTTP_ENDPOINT_ADDR: &str = "127.0.0.1:9898";

 const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
-const DEFAULT_GC_PERIOD: Duration = Duration::from_secs(100);
+const DEFAULT_GC_PERIOD: Duration = Duration::from_secs(10);

 const DEFAULT_SUPERUSER: &str = "zenith_admin";

@@ -31,10 +34,12 @@ const DEFAULT_SUPERUSER: &str = "zenith_admin";
 #[derive(Serialize, Deserialize)]
 struct CfgFileParams {
    listen_addr: Option<String>,
+    http_endpoint_addr: Option<String>,
    gc_horizon: Option<String>,
    gc_period: Option<String>,
    pg_distrib_dir: Option<String>,
-    repository_format: Option<String>,
+    auth_validation_public_key_path: Option<String>,
+    auth_type: Option<String>,
 }

 impl CfgFileParams {
@@ -46,10 +51,12 @@ impl CfgFileParams {

        Self {
            listen_addr: get_arg("listen"),
+            http_endpoint_addr: get_arg("http_endpoint"),
            gc_horizon: get_arg("gc_horizon"),
            gc_period: get_arg("gc_period"),
            pg_distrib_dir: get_arg("postgres-distrib"),
-            repository_format: get_arg("repository-format"),
+            auth_validation_public_key_path: get_arg("auth-validation-public-key-path"),
+            auth_type: get_arg("auth-type"),
        }
    }

@@ -58,20 +65,31 @@ impl CfgFileParams {
        // TODO cleaner way to do this
        Self {
            listen_addr: self.listen_addr.or(other.listen_addr),
+            http_endpoint_addr: self.http_endpoint_addr.or(other.http_endpoint_addr),
            gc_horizon: self.gc_horizon.or(other.gc_horizon),
            gc_period: self.gc_period.or(other.gc_period),
            pg_distrib_dir: self.pg_distrib_dir.or(other.pg_distrib_dir),
-            repository_format: self.repository_format.or(other.repository_format),
+            auth_validation_public_key_path: self
+                .auth_validation_public_key_path
+                .or(other.auth_validation_public_key_path),
+            auth_type: self.auth_type.or(other.auth_type),
        }
    }

    /// Create a PageServerConf from these string parameters
    fn try_into_config(&self) -> Result<PageServerConf> {
+        let workdir = PathBuf::from(".");
+
        let listen_addr = match self.listen_addr.as_ref() {
            Some(addr) => addr.clone(),
            None => DEFAULT_LISTEN_ADDR.to_owned(),
        };

+        let http_endpoint_addr = match self.http_endpoint_addr.as_ref() {
+            Some(addr) => addr.clone(),
+            None => DEFAULT_HTTP_ENDPOINT_ADDR.to_owned(),
+        };
+
        let gc_horizon: u64 = match self.gc_horizon.as_ref() {
            Some(horizon_str) => horizon_str.parse()?,
            None => DEFAULT_GC_HORIZON,
@@ -86,38 +104,50 @@ impl CfgFileParams {
            None => env::current_dir()?.join("tmp_install"),
        };

+        let auth_validation_public_key_path = self
+            .auth_validation_public_key_path
+            .as_ref()
+            .map(PathBuf::from);
+
+        let auth_type = self
+            .auth_type
+            .as_ref()
+            .map_or(Ok(AuthType::Trust), |auth_type| {
+                AuthType::from_str(&auth_type)
+            })?;
+
        if !pg_distrib_dir.join("bin/postgres").exists() {
            anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
        }

-        //
-        // FIXME: This pageserver --repository-format option is pretty useless as it
-        // isn't exposed as an option to "zenith init". But you can change the default
-        // here if you want to test the rocksdb implementation:
-        //
-        let repository_format = match self.repository_format.as_ref() {
-            Some(repo_format_str) if repo_format_str == "rocksdb" => RepositoryFormat::RocksDb,
-            Some(repo_format_str) if repo_format_str == "layered" => RepositoryFormat::Layered,
-            Some(repo_format_str) => anyhow::bail!(
-                "invalid --repository-format '{}', must be 'rocksdb' or 'layered'",
-                repo_format_str
-            ),
-            None => RepositoryFormat::Layered, // default
-        };
+        if auth_type == AuthType::ZenithJWT {
+            ensure!(
+                auth_validation_public_key_path.is_some(),
+                "Missing auth_validation_public_key_path when auth_type is ZenithJWT"
+            );
+            let path_ref = auth_validation_public_key_path.as_ref().unwrap();
+            ensure!(
+                path_ref.exists(),
+                format!("Can't find auth_validation_public_key at {:?}", path_ref)
+            );
+        }

        Ok(PageServerConf {
            daemonize: false,

            listen_addr,
+            http_endpoint_addr,
            gc_horizon,
            gc_period,

            superuser: String::from(DEFAULT_SUPERUSER),

-            workdir: PathBuf::from("."),
+            workdir,

            pg_distrib_dir,
-            repository_format,
+
+            auth_validation_public_key_path,
+            auth_type,
        })
    }
 }
@@ -178,10 +208,16 @@ fn main() -> Result<()> {
                .requires("init"),
        )
        .arg(
-            Arg::with_name("repository-format")
-                .long("repository-format")
+            Arg::with_name("auth-validation-public-key-path")
+                .long("auth-validation-public-key-path")
                .takes_value(true)
-                .help("Which repository implementation to use, 'rocksdb' or 'layered'"),
+                .help("Path to public key used to validate jwt signature"),
+        )
+        .arg(
+            Arg::with_name("auth-type")
+                .long("auth-type")
+                .takes_value(true)
+                .help("Authentication scheme type. One of: Trust, MD5, ZenithJWT"),
        )
        .get_matches();

@@ -203,6 +239,9 @@ fn main() -> Result<()> {
        args_params.or(file_params)
    };

+    // Set CWD to workdir for non-daemon modes
+    env::set_current_dir(&workdir)?;
+
    // Ensure the config is valid, even if just init-ing
    let mut conf = params.try_into_config()?;

@@ -220,17 +259,15 @@ fn main() -> Result<()> {

    // Create repo and exit if init was requested
    if init {
-        branches::init_pageserver(conf, workdir, create_tenant)?;
+        branches::init_pageserver(conf, create_tenant)?;
        // write the config file
        let cfg_file_contents = toml::to_string_pretty(&params)?;
+        // TODO support enable-auth flag
        std::fs::write(&cfg_file_path, cfg_file_contents)?;

        return Ok(());
    }

-    // Set CWD to workdir for non-daemon modes
-    env::set_current_dir(&workdir)?;
-
    start_pageserver(conf)
 }

@@ -264,19 +301,41 @@ fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
        }
    }

+    // initialize authentication for incoming connections
+    let auth = match &conf.auth_type {
+        AuthType::Trust | AuthType::MD5 => None,
+        AuthType::ZenithJWT => {
+            // unwrap is ok because check is performed when creating config, so path is set and file exists
+            let key_path = conf.auth_validation_public_key_path.as_ref().unwrap();
+            Some(JwtAuth::from_key_path(key_path)?.into())
+        }
+    };
+    info!("Using auth: {:#?}", conf.auth_type);
+
+    // Spawn a new thread for the http endpoint
+    let cloned = auth.clone();
+    thread::Builder::new()
+        .name("http_endpoint_thread".into())
+        .spawn(move || {
+            let router = http::make_router(conf, cloned);
+            endpoint::serve_thread_main(router, conf.http_endpoint_addr.clone())
+        })?;
+
    // Check that we can bind to address before starting threads to simplify shutdown
    // sequence if port is occupied.
    info!("Starting pageserver on {}", conf.listen_addr);
    let pageserver_listener = TcpListener::bind(conf.listen_addr.clone())?;

-    // Initialize page cache, this will spawn walredo_thread
-    page_cache::init(conf);
+    // Initialize tenant manager.
+    tenant_mgr::init(conf);

    // Spawn a thread to listen for connections. It will spawn further threads
    // for each connection.
    let page_service_thread = thread::Builder::new()
        .name("Page Service thread".into())
-        .spawn(move || page_service::thread_main(conf, pageserver_listener))?;
+        .spawn(move || {
+            page_service::thread_main(conf, auth, pageserver_listener, conf.auth_type)
+        })?;

    page_service_thread
        .join()
--- a/pageserver/src/branches.rs
+++ b/pageserver/src/branches.rs
@@ -7,7 +7,6 @@
 use anyhow::{bail, ensure, Context, Result};
 use postgres_ffi::ControlFileData;
 use serde::{Deserialize, Serialize};
-use std::env;
 use std::{
    fs,
    path::Path,
@@ -15,21 +14,21 @@ use std::{
    str::FromStr,
    sync::Arc,
 };
+use zenith_utils::zid::{ZTenantId, ZTimelineId};

 use log::*;
 use zenith_utils::lsn::Lsn;

 use crate::logger;
-use crate::object_repository::ObjectRepository;
-use crate::page_cache;
 use crate::restore_local_repo;
+use crate::tenant_mgr;
 use crate::walredo::WalRedoManager;
-use crate::ZTenantId;
-use crate::{repository::Repository, PageServerConf, RepositoryFormat, ZTimelineId};
+use crate::{repository::Repository, PageServerConf};

 #[derive(Serialize, Deserialize, Clone)]
 pub struct BranchInfo {
    pub name: String,
+    #[serde(with = "hex")]
    pub timeline_id: ZTimelineId,
    pub latest_valid_lsn: Option<Lsn>,
    pub ancestor_id: Option<String>,
@@ -42,27 +41,27 @@ pub struct PointInTime {
    pub lsn: Lsn,
 }

-pub fn init_pageserver(
-    conf: &'static PageServerConf,
-    workdir: &Path,
-    create_tenant: Option<&str>,
-) -> Result<()> {
-
-    env::set_current_dir(workdir)?;
-
+pub fn init_pageserver(conf: &'static PageServerConf, create_tenant: Option<&str>) -> Result<()> {
    // Initialize logger
    let (_scope_guard, _log_file) = logger::init_logging(&conf, "pageserver.log")?;
    let _log_guard = slog_stdlog::init()?;

+    // We don't use the real WAL redo manager, because we don't want to spawn the WAL redo
+    // process during repository initialization.
+    //
+    // FIXME: That caused trouble, because the WAL redo manager spawned a thread that launched
+    // initdb in the background, and it kept running even after the "zenith init" had exited.
+    // In tests, we started the  page server immediately after that, so that initdb was still
+    // running in the background, and we failed to run initdb again in the same directory. This
+    // has been solved for the rapid init+start case now, but the general race condition remains
+    // if you restart the server quickly. The WAL redo manager doesn't use a separate thread
+    // anymore, but I think that could still happen.
+    let dummy_redo_mgr = Arc::new(crate::walredo::DummyRedoManager {});
+
    if let Some(tenantid) = create_tenant {
        let tenantid = ZTenantId::from_str(tenantid)?;
        println!("initializing tenantid {}", tenantid);
-        create_repo(
-            conf,
-            tenantid,
-            Arc::new(crate::walredo::DummyRedoManager {}),
-        )
-        .with_context(|| "failed to create repo")?;
+        create_repo(conf, tenantid, dummy_redo_mgr).with_context(|| "failed to create repo")?;
    }
    fs::create_dir_all(conf.tenants_path())?;

@@ -95,32 +94,11 @@ pub fn create_repo(

    let tli = create_timeline(conf, None, &tenantid)?;

-    // We don't use page_cache here, because we don't want to spawn the WAL redo thread during
-    // repository initialization.
-    //
-    // FIXME: That caused trouble, because the WAL redo thread launched initdb in the background,
-    // and it kept running even after the "zenith init" had exited. In tests, we started the
-    // page server immediately after that, so that initdb was still running in the background,
-    // and we failed to run initdb again in the same directory. This has been solved for the
-    // rapid init+start case now, but the general race condition remains if you restart the
-    // server quickly.
-    let repo: Arc<dyn Repository + Sync + Send> = match conf.repository_format {
-        RepositoryFormat::Layered => Arc::new(
-            crate::layered_repository::LayeredRepository::new(conf,
-                                                              wal_redo_manager,
-                                                              tenantid
-        )),
-        RepositoryFormat::RocksDb => {
-            let obj_store = crate::rocksdb_storage::RocksObjectStore::create(conf, &tenantid)?;
-
-            Arc::new(ObjectRepository::new(
-                conf,
-                Arc::new(obj_store),
-                wal_redo_manager,
-                tenantid
-            ))
-        }
-    };
+    let repo = Arc::new(crate::layered_repository::LayeredRepository::new(
+        conf,
+        wal_redo_manager,
+        tenantid,
+    ));

    // Load data into pageserver
    // TODO To implement zenith import we need to
@@ -147,7 +125,7 @@ fn run_initdb(conf: &'static PageServerConf, initdbpath: &Path) -> Result<()> {
    info!("running initdb... ");

    let initdb_path = conf.pg_bin_dir().join("initdb");
-    let initdb_otput = Command::new(initdb_path)
+    let initdb_output = Command::new(initdb_path)
        .args(&["-D", initdbpath.to_str().unwrap()])
        .args(&["-U", &conf.superuser])
        .arg("--no-instructions")
@@ -157,10 +135,10 @@ fn run_initdb(conf: &'static PageServerConf, initdbpath: &Path) -> Result<()> {
        .stdout(Stdio::null())
        .output()
        .with_context(|| "failed to execute initdb")?;
-    if !initdb_otput.status.success() {
+    if !initdb_output.status.success() {
        anyhow::bail!(
            "initdb failed: '{}'",
-            String::from_utf8_lossy(&initdb_otput.stderr)
+            String::from_utf8_lossy(&initdb_output.stderr)
        );
    }
    info!("initdb succeeded");
@@ -190,7 +168,6 @@ fn bootstrap_timeline(

    let timeline = repo.create_empty_timeline(tli, lsn)?;
    restore_local_repo::import_timeline_from_postgres_datadir(&pgdata_path, &*timeline, lsn)?;
-    timeline.checkpoint()?;

    let wal_dir = pgdata_path.join("pg_wal");
    restore_local_repo::import_timeline_wal(&wal_dir, &*timeline, timeline.get_last_record_lsn())?;
@@ -224,7 +201,7 @@ pub(crate) fn get_tenants(conf: &PageServerConf) -> Result<Vec<String>> {
 }

 pub(crate) fn get_branches(conf: &PageServerConf, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
-    let repo = page_cache::get_repository_for_tenant(tenantid)?;
+    let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;

    // Each branch has a corresponding record (text file) in the refs/branches
    // with timeline_id.
@@ -280,7 +257,7 @@ pub(crate) fn create_branch(
    startpoint_str: &str,
    tenantid: &ZTenantId,
 ) -> Result<BranchInfo> {
-    let repo = page_cache::get_repository_for_tenant(tenantid)?;
+    let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;

    if conf.branch_path(branchname, tenantid).exists() {
        anyhow::bail!("branch {} already exists", branchname);
@@ -293,7 +270,7 @@ pub(crate) fn create_branch(
        let end_of_wal = repo
            .get_timeline(startpoint.timelineid)?
            .get_last_record_lsn();
-        println!("branching at end of WAL: {}", end_of_wal);
+        info!("branching at end of WAL: {}", end_of_wal);
        startpoint.lsn = end_of_wal;
    }

--- a/pageserver/src/http/mod.rs
+++ b/pageserver/src/http/mod.rs
@@ -0,0 +1,3 @@
+pub mod models;
+pub mod routes;
+pub use routes::make_router;
--- a/pageserver/src/http/models.rs
+++ b/pageserver/src/http/models.rs
@@ -0,0 +1,17 @@
+use serde::{Deserialize, Serialize};
+
+use crate::ZTenantId;
+
+#[derive(Serialize, Deserialize)]
+pub struct BranchCreateRequest {
+    #[serde(with = "hex")]
+    pub tenant_id: ZTenantId,
+    pub name: String,
+    pub start_point: String,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct TenantCreateRequest {
+    #[serde(with = "hex")]
+    pub tenant_id: ZTenantId,
+}
--- a/pageserver/src/http/openapi_spec.yml
+++ b/pageserver/src/http/openapi_spec.yml
@@ -0,0 +1,239 @@
+openapi: "3.0.2"
+info:
+  title: Page Server API
+  version: "1.0"
+servers:
+  - url: ""
+paths:
+  /v1/status:
+    description: Healthcheck endpoint
+    get:
+      description: Healthcheck
+      security: []
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+  /v1/branch/{tenant_id}:
+    parameters:
+      - name: tenant_id
+        in: path
+        required: true
+        schema:
+          type: string
+          format: hex
+    get:
+      description: Get branches for tenant
+      responses:
+        "200":
+          description: BranchInfo
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/BranchInfo"
+        "400":
+          description: Error when no tenant id found in path
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+  /v1/branch/:
+    post:
+      description: Create branch
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - "tenant_id"
+                - "name"
+                - "start_point"
+              properties:
+                tenant_id:
+                  type: string
+                  format: hex
+                name:
+                  type: string
+                start_point:
+                  type: string
+      responses:
+        "201":
+          description: BranchInfo
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/BranchInfo"
+        "400":
+          description: Malformed branch create request
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+  /v1/tenant/:
+    get:
+      description: Get tenants list
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+    post:
+      description: Create tenant
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - "tenant_id"
+              properties:
+                tenant_id:
+                  type: string
+                  format: hex
+      responses:
+        "201":
+          description: CREATED
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+        "400":
+          description: Malformed tenant create request
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+
+components:
+  securitySchemes:
+    JWT:
+      type: http
+      scheme: bearer
+      bearerFormat: JWT
+  schemas:
+    BranchInfo:
+      type: object
+      required:
+        - name
+        - timeline_id
+      properties:
+        name:
+          type: string
+        timeline_id:
+          type: string
+          format: hex
+        ancestor_id:
+          type: string
+        ancestor_lsn:
+          type: string
+    Error:
+      type: object
+      required:
+        - msg
+      properties:
+        msg:
+          type: string
+    UnauthorizedError:
+      type: object
+      required:
+        - msg
+      properties:
+        msg:
+          type: string
+    ForbiddenError:
+      type: object
+      required:
+        - msg
+      properties:
+        msg:
+          type: string
+
+security:
+  - JWT: []
--- a/pageserver/src/http/routes.rs
+++ b/pageserver/src/http/routes.rs
@@ -0,0 +1,166 @@
+use std::sync::Arc;
+
+use anyhow::Result;
+use hyper::header;
+use hyper::StatusCode;
+use hyper::{Body, Request, Response, Uri};
+use routerify::{ext::RequestExt, RouterBuilder};
+use zenith_utils::auth::JwtAuth;
+use zenith_utils::http::endpoint::attach_openapi_ui;
+use zenith_utils::http::endpoint::auth_middleware;
+use zenith_utils::http::endpoint::check_permission;
+use zenith_utils::http::error::ApiError;
+use zenith_utils::http::{
+    endpoint,
+    error::HttpErrorBody,
+    json::{json_request, json_response},
+};
+
+use super::models::BranchCreateRequest;
+use super::models::TenantCreateRequest;
+use crate::{
+    branches::{self},
+    tenant_mgr, PageServerConf, ZTenantId,
+};
+
+#[derive(Debug)]
+struct State {
+    conf: &'static PageServerConf,
+    auth: Option<Arc<JwtAuth>>,
+    allowlist_routes: Vec<Uri>,
+}
+
+impl State {
+    fn new(conf: &'static PageServerConf, auth: Option<Arc<JwtAuth>>) -> Self {
+        let allowlist_routes = ["/v1/status", "/v1/doc", "/swagger.yml"]
+            .iter()
+            .map(|v| v.parse().unwrap())
+            .collect::<Vec<_>>();
+        Self {
+            conf,
+            auth,
+            allowlist_routes,
+        }
+    }
+}
+
+#[inline(always)]
+fn get_state(request: &Request<Body>) -> &State {
+    request
+        .data::<Arc<State>>()
+        .expect("unknown state type")
+        .as_ref()
+}
+
+#[inline(always)]
+fn get_config(request: &Request<Body>) -> &'static PageServerConf {
+    get_state(request).conf
+}
+
+// healthcheck handler
+async fn status_handler(_: Request<Body>) -> Result<Response<Body>, ApiError> {
+    Ok(Response::builder()
+        .status(StatusCode::OK)
+        .header(header::CONTENT_TYPE, "application/json")
+        .body(Body::from("{}"))
+        .map_err(ApiError::from_err)?)
+}
+
+async fn branch_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let request_data: BranchCreateRequest = json_request(&mut request).await?;
+
+    check_permission(&request, Some(request_data.tenant_id))?;
+
+    let response_data = tokio::task::spawn_blocking(move || {
+        branches::create_branch(
+            get_config(&request),
+            &request_data.name,
+            &request_data.start_point,
+            &request_data.tenant_id,
+        )
+    })
+    .await
+    .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::CREATED, response_data)?)
+}
+
+async fn branch_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let tenantid: ZTenantId = match request.param("tenant_id") {
+        Some(arg) => arg
+            .parse()
+            .map_err(|_| ApiError::BadRequest("failed to parse tenant id".to_string()))?,
+        None => {
+            return Err(ApiError::BadRequest(
+                "no tenant id specified in path param".to_string(),
+            ))
+        }
+    };
+
+    check_permission(&request, Some(tenantid))?;
+
+    let response_data = tokio::task::spawn_blocking(move || {
+        crate::branches::get_branches(get_config(&request), &tenantid)
+    })
+    .await
+    .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::OK, response_data)?)
+}
+
+async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    // check for management permission
+    check_permission(&request, None)?;
+
+    let response_data =
+        tokio::task::spawn_blocking(move || crate::branches::get_tenants(get_config(&request)))
+            .await
+            .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::OK, response_data)?)
+}
+
+async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    // check for management permission
+    check_permission(&request, None)?;
+
+    let request_data: TenantCreateRequest = json_request(&mut request).await?;
+
+    let response_data = tokio::task::spawn_blocking(move || {
+        tenant_mgr::create_repository_for_tenant(get_config(&request), request_data.tenant_id)
+    })
+    .await
+    .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::CREATED, response_data)?)
+}
+
+async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
+    json_response(
+        StatusCode::NOT_FOUND,
+        HttpErrorBody::from_msg("page not found".to_owned()),
+    )
+}
+
+pub fn make_router(
+    conf: &'static PageServerConf,
+    auth: Option<Arc<JwtAuth>>,
+) -> RouterBuilder<hyper::Body, ApiError> {
+    let spec = include_bytes!("openapi_spec.yml");
+    let mut router = attach_openapi_ui(endpoint::make_router(), spec, "/swagger.yml", "/v1/doc");
+    if auth.is_some() {
+        router = router.middleware(auth_middleware(|request| {
+            let state = get_state(request);
+            if state.allowlist_routes.contains(request.uri()) {
+                None
+            } else {
+                state.auth.as_deref()
+            }
+        }))
+    }
+
+    router
+        .data(Arc::new(State::new(conf, auth)))
+        .get("/v1/status", status_handler)
+        .get("/v1/branch/:tenant_id", branch_list_handler)
+        .post("/v1/branch", branch_create_handler)
+        .get("/v1/tenant", tenant_list_handler)
+        .post("/v1/tenant", tenant_create_handler)
+        .any(handler_404)
+}
--- a/pageserver/src/layered_repository.rs
+++ b/pageserver/src/layered_repository.rs
--- a/pageserver/src/layered_repository/README.md
+++ b/pageserver/src/layered_repository/README.md
@@ -7,47 +7,63 @@ memory. Every now and then, the accumulated changes are written out to
 new files.

 The files are called "snapshot files". Each snapshot file corresponds
-to one PostgreSQL relation fork. The snapshot files for each timeline
-are stored in the timeline's subdirectory under .zenith/timelines.
+to one 10 MB slice of a PostgreSQL relation fork. The snapshot files
+for each timeline are stored in the timeline's subdirectory under
+.zenith/tenants/<tenantid>/timelines.

-The files are named like this:
+There are two kind of snapshot file: base images, and deltas. A base
+image file contains a snapshot of a segment as it was at one LSN,
+whereas a delta file contains modifications to a segment - mostly in
+the form of WAL records - in a range of LSN

-    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<start LSN>_<end LSN>
+base image file:
+
+    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>
+
+delta file:
+
+    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>_<end LSN>

 For example:

-    rel_1663_13990_2609_0_000000000169C348_0000000001702000
+    rel_1663_13990_2609_0_10_000000000169C348
+    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000

-Each snapshot file contains a full snapshot, that is, full copy of all
-pages in the relation, as of the "start LSN". It also contains all WAL
-records applicable to the relation between the start and end
-LSNs. With this information, the page server can reconstruct any page
-version of the relation in the LSN range.
+In addition to the relations, with "rel_*" prefix, we use the same
+format for storing various smaller files from the PostgreSQL data
+directory. They will use different suffixes and the naming scheme up
+to the LSNs vary. The Zenith source code uses the term "relish" to
+mean "a relation, or other file that's treated like a relation in the
+storage" For example, a base image of a CLOG segment would be named
+like this:
+
+    pg_xact_0000_0_00000000198B06B0
+
+There is no difference in how the relation and non-relation files are
+managed, except that the first part of file names is different.
+Internally, the relations and non-relation files that are managed in
+the versioned store are together called "relishes".

 If a file has been dropped, the last snapshot file for it is created
 with the _DROPPED suffix, e.g.

-    rel_1663_13990_2609_0_000000000169C348_0000000001702000_DROPPED
-	
-In addition to the relations, with "rel_*" prefix, we use the same
-format for storing various smaller files from the PostgreSQL data
-directory. They will use different suffixes and the naming scheme
-up to the LSN range varies. The Zenith source code uses the term
-"relish" to mean "a relation, or other file that's treated like a
-relation in the storage"
+    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000_DROPPED
+

 ## Notation used in this document

-The full path of a snapshot file looks like this:
+The full path of a delta file looks like this:

-    .zenith/timelines/4af489b06af8eed9e27a841775616962/rel_1663_13990_2609_0_000000000169C348_0000000001702000
+    .zenith/tenants/941ddc8604413b88b3d208bddf90396c/timelines/4af489b06af8eed9e27a841775616962/rel_1663_13990_2609_0_10_000000000169C348_0000000001702000

 For simplicity, the examples below use a simplified notation for the
-paths.  The timeline ID is replaced with the human-readable branch
-name, and spcnode+dbnode+relnode+forkum with a human-readable table
-name. The LSNs are also shorter. For example, a snapshot file for
-'orders' table on 'main' branch, with LSN range 100-200 would be:
+paths.  The tenant ID is left out, the timeline ID is replaced with
+the human-readable branch name, and spcnode+dbnode+relnode+forkum+segno
+with a human-readable table name. The LSNs are also shorter. For
+example, a base image file at LSN 100 and a delta file between 100-200
+for 'orders' table on 'main' branch is represented like this:

+    main/orders_100
    main/orders_100_200


@@ -56,10 +72,14 @@ name. The LSNs are also shorter. For example, a snapshot file for
 Let's start with a simple example with a system that contains one
 branch called 'main' and two tables, 'orders' and 'customers'. The end
 of WAL is currently at LSN 250. In this starting situation, you would
-have two files on disk:
+have these files on disk:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
+	main/customers_100
 	main/customers_100_200
+	main/customers_200

 In addition to those files, the recent changes between LSN 200 and the
 end of WAL at 250 are kept in memory. If the page server crashes, the
@@ -75,20 +95,33 @@ the last checkpoint happened at LSN 400 but there hasn't been any
 recent changes to 'customers' table, you would have these files on
 disk:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
+	main/orders_300
 	main/orders_300_400
+	main/orders_400
+	main/customers_100
 	main/customers_100_200
+	main/customers_200

 If the customers table is modified later, a new file is created for it
 at the next checkpoint. The new file will cover the "gap" from the
 last snapshot file, so the LSN ranges are always contiguous:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
+	main/orders_300
 	main/orders_300_400
+	main/orders_400
+	main/customers_100
 	main/customers_100_200
+	main/customers_200
 	main/customers_200_500
+	main/customers_500

 ## Reading page versions

@@ -108,7 +141,6 @@ relation at the start LSN and the WAL, reconstructing the page
 involves replaying any WAL records applicable to the page between LSNs
 200-250, starting from the base image at LSN 200.

-
 # Multiple branches

 Imagine that a child branch is created at LSN 250:
@@ -122,12 +154,20 @@ Imagine that a child branch is created at LSN 250:
 Then, the 'orders' table is updated differently on the 'main' and
 'child' branches. You now have this situation on disk:

+    main/orders_100
    main/orders_100_200
+    main/orders_200
    main/orders_200_300
+    main/orders_300
    main/orders_300_400
+    main/orders_400
+    main/customers_100
    main/customers_100_200
+    main/customers_200
    child/orders_250_300
+    child/orders_300
    child/orders_300_400
+    child/orders_400

 Because the 'customers' table hasn't been modified on the child
 branch, there is no file for it there. If you request a page for it on
@@ -140,24 +180,34 @@ is linear, and the request's LSN identifies unambiguously which file
 you need to look at. For example, the history for the 'orders' table
 on the 'main' branch consists of these files:

+    main/orders_100
    main/orders_100_200
+    main/orders_200
    main/orders_200_300
+    main/orders_300
    main/orders_300_400
+    main/orders_400

 And from the 'child' branch's point of view, it consists of these
 files:

+    main/orders_100
    main/orders_100_200
+    main/orders_200
    main/orders_200_300
    child/orders_250_300
+    child/orders_300
    child/orders_300_400
+    child/orders_400

 The branch metadata includes the point where the child branch was
 created, LSN 250. If a page request comes with LSN 275, we read the
-page version from the 'child/orders_250_300' file. If the request LSN
-is 225, we read it from the 'main/orders_200_300' file instead.  The
-page versions between 250-300 in the 'main/orders_200_300' file are
-ignored when operating on the child branch.
+page version from the 'child/orders_250_300' file. We might also
+need to reconstruct the page version as it was at LSN 250, in order
+to replay the WAL up to LSN 275, using 'main/orders_200_300' and
+'main/orders_200'. The page versions between 250-300 in the
+'main/orders_200_300' file are ignored when operating on the child
+branch.

 Note: It doesn't make any difference if the child branch is created
 when the end of the main branch was at LSN 250, or later when the tip of
@@ -181,17 +231,37 @@ Let's look at the single branch scenario again. Imagine that the end
 of the branch is LSN 525, so that the GC horizon is currently at
 525-150 = 375

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
+	main/orders_300
 	main/orders_300_400
+	main/orders_400
 	main/orders_400_500
+	main/orders_500
+	main/customers_100
 	main/customers_100_200
+	main/customers_200

-We can remove files 'main/orders_100_200' and 'main/orders_200_300',
-because the end LSNs of those files are older than GC horizon 375, and
-there are more recent snapshot files for the table. 'main/orders_300_400'
-and 'main/orders_400_500' are still within the horizon, so they must be
-retained. 'main/customers_100_200' is old enough, but it cannot be
+We can remove the following files because the end LSNs of those files are
+older than GC horizon 375, and there are more recent snapshot files for the
+table:
+
+	main/orders_100       DELETE
+	main/orders_100_200   DELETE
+	main/orders_200       DELETE
+	main/orders_200_300   DELETE
+	main/orders_300       STILL NEEDED BY orders_300_400
+	main/orders_300_400   KEEP, NEWER THAN GC HORIZON
+	main/orders_400       .. 
+	main/orders_400_500   .. 
+	main/orders_500       .. 
+	main/customers_100      DELETE
+	main/customers_100_200  DELETE
+	main/customers_200      KEEP, NO NEWER VERSION
+
+'main/customers_100_200' is old enough, but it cannot be
 removed because there is no newer snapshot file for the table.

 Things get slightly more complicated with multiple branches. All of
@@ -200,63 +270,81 @@ retain older shapshot files that are still needed by child branches.
 For example, if child branch is created at LSN 150, and the 'customers'
 table is updated on the branch, you would have these files:

-	main/orders_100_200
-	main/orders_200_300
-	main/orders_300_400
-	main/orders_400_500
-	main/customers_100_200
-	child/customers_150_300
+	main/orders_100        KEEP, NEEDED BY child BRANCH
+	main/orders_100_200    KEEP, NEEDED BY child BRANCH
+	main/orders_200        DELETE
+	main/orders_200_300    DELETE
+	main/orders_300        KEEP, NEWER THAN GC HORIZON
+	main/orders_300_400    KEEP, NEWER THAN GC HORIZON
+	main/orders_400        KEEP, NEWER THAN GC HORIZON
+	main/orders_400_500    KEEP, NEWER THAN GC HORIZON
+	main/orders_500        KEEP, NEWER THAN GC HORIZON
+	main/customers_100       DELETE
+	main/customers_100_200   DELETE
+	main/customers_200       KEEP, NO NEWER VERSION
+	child/customers_150_300  DELETE
+	child/customers_300      KEEP, NO NEWER VERSION

-In this situation, the 'main/orders_100_200' file cannot be removed,
-even though it is older than the GC horizon, because it is still
-needed by the child branch.  'main/orders_200_300' can still be
-removed. So after garbage collection, these files would remain:
-
-	main/orders_100_200
-
-	main/orders_300_400
-	main/orders_400_500
-	main/customers_100_200
-	child/customers_150_300
+In this situation, 'main/orders_100' and 'main/orders_100_200' cannot
+be removed, even though they are older than the GC horizon, because
+they are still needed by the child branch. 'main/orders_200'
+and 'main/orders_200_300' can still be removed.

 If 'orders' is modified later on the 'child' branch, we will create a
-snapshot file for it on the child:
+new base image and delta file for it on the child:

+	main/orders_100
 	main/orders_100_200

+	main/orders_300
 	main/orders_300_400
+	main/orders_400
 	main/orders_400_500
-	main/customers_100_200
-	child/customers_150_300
+	main/orders_500
+	main/customers_200
+	child/customers_300
 	child/orders_150_400
+	child/orders_400

-After this, the 'main/orders_100_200' file can be removed. It is no
-longer needed by the child branch, because there is a newer snapshot
-file there. TODO: This optimization hasn't been implemented! The GC
-algorithm will curently keep the file on the 'main' branch anyway, for
-as long as the child branch exists.
+After this, the 'main/orders_100' and 'main/orders_100_200' file could
+be removed. It is no longer needed by the child branch, because there
+is a newer snapshot file there. TODO: This optimization hasn't been
+implemented! The GC algorithm will currently keep the file on the
+'main' branch anyway, for as long as the child branch exists.


-# On LSN ranges
+# TODO: On LSN ranges

 In principle, each relation can be checkpointed separately, i.e. the
 LSN ranges of the files don't need to line up. So this would be legal:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
+	main/orders_300
 	main/orders_300_400
+	main/orders_400
+	main/customers_150
 	main/customers_150_250
+	main/customers_250
 	main/customers_250_500
+	main/customers_500

 However, the code currently always checkpoints all relations together.
 So that situation doesn't arise in practice.

 It would also be OK to have overlapping LSN ranges for the same relation:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
+	main/orders_300
 	main/orders_250_350
+	main/orders_350
 	main/orders_300_400
+	main/orders_400

 The code that reads the snapshot files should cope with this, but this
 situation doesn't arise either, because the checkpointing code never
@@ -264,11 +352,14 @@ does that.  It could be useful, however, as a transient state when
 garbage collecting around branch points, or explicit recovery
 points. For example, if we start with this:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
-	main/orders_300_400
+	main/orders_300

 And there is a branch or explicit recovery point at LSN 150, we could
-replace 'main/orders_100_200' with 'main/orders_150_150' to keep a
+replace 'main/orders_100_200' with 'main/orders_150' to keep a
 snapshot only at that exact point that's still needed, removing the
-other page versions around it.
+other page versions around it. But such compaction has not been
+implemented yet.
--- a/pageserver/src/layered_repository/blob.rs
+++ b/pageserver/src/layered_repository/blob.rs
@@ -0,0 +1,45 @@
+use std::{fs::File, io::Write};
+
+use anyhow::Result;
+use bookfile::{BookWriter, BoundedReader, ChapterId, ChapterWriter};
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+pub struct BlobRange {
+    offset: u64,
+    size: usize,
+}
+
+pub fn read_blob(reader: &BoundedReader<&'_ File>, range: &BlobRange) -> Result<Vec<u8>> {
+    let mut buf = vec![0u8; range.size];
+    reader.read_exact_at(&mut buf, range.offset)?;
+    Ok(buf)
+}
+
+pub struct BlobWriter {
+    writer: ChapterWriter<File>,
+    offset: u64,
+}
+
+impl BlobWriter {
+    // This function takes a BookWriter and creates a new chapter to ensure offset is 0.
+    pub fn new(book_writer: BookWriter<File>, chapter_id: impl Into<ChapterId>) -> Self {
+        let writer = book_writer.new_chapter(chapter_id);
+        Self { writer, offset: 0 }
+    }
+
+    pub fn write_blob(&mut self, blob: &[u8]) -> Result<BlobRange> {
+        self.writer.write_all(blob)?;
+
+        let range = BlobRange {
+            offset: self.offset,
+            size: blob.len(),
+        };
+        self.offset += blob.len() as u64;
+        Ok(range)
+    }
+
+    pub fn close(self) -> bookfile::Result<BookWriter<File>> {
+        self.writer.close()
+    }
+}
--- a/pageserver/src/layered_repository/delta_layer.rs
+++ b/pageserver/src/layered_repository/delta_layer.rs
@@ -0,0 +1,490 @@
+//!
+//! A DeltaLayer represents a collection of WAL records or page images in a range of
+//! LSNs, for one segment. It is stored on a file on disk.
+//!
+//! Usually a delta layer only contains differences - in the form of WAL records against
+//! a base LSN. However, if a segment is newly created, by creating a new relation or
+//! extending an old one, there might be no base image. In that case, all the entries in
+//! the delta layer must be page images or WAL records with the 'will_init' flag set, so
+//! that they can be replayed without referring to an older page version. Also in some
+//! circumstances, the predecessor layer might actually be another delta layer. That
+//! can happen when you create a new branch in the middle of a delta layer, and the WAL
+//! records on the new branch are put in a new delta layer.
+//!
+//! When a delta file needs to be accessed, we slurp the metadata and relsize chapters
+//! into memory, into the DeltaLayerInner struct. See load() and unload() functions.
+//! To access a page/WAL record, we search `page_version_metas` for the block # and LSN.
+//! The byte ranges in the metadata can be used to find the page/WAL record in
+//! PAGE_VERSIONS_CHAPTER.
+//!
+//! On disk, the delta files are stored in timelines/<timelineid> directory.
+//! Currently, there are no subdirectories, and each delta file is named like this:
+//!
+//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>_<end LSN>
+//!
+//! For example:
+//!
+//!    1663_13990_2609_0_5_000000000169C348_000000000169C349
+//!
+//! If a relation is dropped, we add a '_DROPPED' to the end of the filename to indicate that.
+//! So the above example would become:
+//!
+//!    1663_13990_2609_0_5_000000000169C348_000000000169C349_DROPPED
+//!
+//! The end LSN indicates when it was dropped in that case, we don't store it in the
+//! file contents in any way.
+//!
+//! A detlta file is constructed using the 'bookfile' crate. Each file consists of two
+//! parts: the page versions and the relation sizes. They are stored as separate chapters.
+//!
+use crate::layered_repository::blob::BlobWriter;
+use crate::layered_repository::filename::DeltaFileName;
+use crate::layered_repository::storage_layer::{
+    Layer, PageReconstructData, PageVersion, SegmentTag,
+};
+use crate::repository::WALRecord;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
+use anyhow::{bail, Result};
+use bytes::Bytes;
+use log::*;
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
+use std::fs;
+use std::fs::File;
+use std::io::Write;
+use std::ops::Bound::Included;
+use std::path::PathBuf;
+use std::sync::{Arc, Mutex, MutexGuard};
+
+use bookfile::{Book, BookWriter};
+
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
+
+use super::blob::{read_blob, BlobRange};
+
+// Magic constant to identify a Zenith delta file
+static DELTA_FILE_MAGIC: u32 = 0x5A616E01;
+
+/// Mapping from (block #, lsn) -> page/WAL record
+/// byte ranges in PAGE_VERSIONS_CHAPTER
+static PAGE_VERSION_METAS_CHAPTER: u64 = 1;
+/// Page/WAL bytes - cannot be interpreted
+/// without PAGE_VERSION_METAS_CHAPTER
+static PAGE_VERSIONS_CHAPTER: u64 = 2;
+static REL_SIZES_CHAPTER: u64 = 3;
+
+#[derive(Serialize, Deserialize)]
+struct PageVersionMeta {
+    page_image_range: Option<BlobRange>,
+    record_range: Option<BlobRange>,
+}
+
+///
+/// DeltaLayer is the in-memory data structure associated with an
+/// on-disk delta file.  We keep a DeltaLayer in memory for each
+/// file, in the LayerMap. If a layer is in "loaded" state, we have a
+/// copy of the file in memory, in 'inner'. Otherwise the struct is
+/// just a placeholder for a file that exists on disk, and it needs to
+/// be loaded before using it in queries.
+///
+pub struct DeltaLayer {
+    conf: &'static PageServerConf,
+    pub tenantid: ZTenantId,
+    pub timelineid: ZTimelineId,
+    pub seg: SegmentTag,
+
+    //
+    // This entry contains all the changes from 'start_lsn' to 'end_lsn'. The
+    // start is inclusive, and end is exclusive.
+    //
+    pub start_lsn: Lsn,
+    pub end_lsn: Lsn,
+
+    dropped: bool,
+
+    /// Base layer preceding this layer.
+    predecessor: Option<Arc<dyn Layer>>,
+
+    inner: Mutex<DeltaLayerInner>,
+}
+
+pub struct DeltaLayerInner {
+    /// If false, the 'page_version_metas' and 'relsizes' have not been
+    /// loaded into memory yet.
+    loaded: bool,
+
+    /// All versions of all pages in the file are are kept here.
+    /// Indexed by block number and LSN.
+    page_version_metas: BTreeMap<(u32, Lsn), PageVersionMeta>,
+
+    /// `relsizes` tracks the size of the relation at different points in time.
+    relsizes: BTreeMap<Lsn, u32>,
+}
+
+impl Layer for DeltaLayer {
+    fn get_timeline_id(&self) -> ZTimelineId {
+        return self.timelineid;
+    }
+
+    fn get_seg_tag(&self) -> SegmentTag {
+        return self.seg;
+    }
+
+    fn is_dropped(&self) -> bool {
+        return self.dropped;
+    }
+
+    fn get_start_lsn(&self) -> Lsn {
+        return self.start_lsn;
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
+        return self.end_lsn;
+    }
+
+    fn filename(&self) -> PathBuf {
+        PathBuf::from(
+            DeltaFileName {
+                seg: self.seg,
+                start_lsn: self.start_lsn,
+                end_lsn: self.end_lsn,
+                dropped: self.dropped,
+            }
+            .to_string(),
+        )
+    }
+
+    /// Look up given page in the cache.
+    fn get_page_reconstruct_data(
+        &self,
+        blknum: u32,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<Option<Lsn>> {
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let mut need_base_image_lsn: Option<Lsn> = Some(lsn);
+
+        // TODO: avoid opening the snapshot file for each read
+        let (_path, book) = self.open_book()?;
+        let page_version_reader = book.chapter_reader(PAGE_VERSIONS_CHAPTER)?;
+        {
+            let inner = self.load()?;
+            let minkey = (blknum, Lsn(0));
+            let maxkey = (blknum, lsn);
+            let mut iter = inner
+                .page_version_metas
+                .range((Included(&minkey), Included(&maxkey)));
+            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
+                if let Some(img_range) = &entry.page_image_range {
+                    let img = Bytes::from(read_blob(&page_version_reader, img_range)?);
+                    reconstruct_data.page_img = Some(img);
+                    need_base_image_lsn = None;
+                    break;
+                } else if let Some(rec_range) = &entry.record_range {
+                    let rec = WALRecord::des(&read_blob(&page_version_reader, rec_range)?)?;
+                    let will_init = rec.will_init;
+                    reconstruct_data.records.push(rec);
+                    if will_init {
+                        // This WAL record initializes the page, so no need to go further back
+                        need_base_image_lsn = None;
+                        break;
+                    } else {
+                        need_base_image_lsn = Some(*entry_lsn);
+                    }
+                } else {
+                    // No base image, and no WAL record. Huh?
+                    bail!("no page image or WAL record for requested page");
+                }
+            }
+
+            // Use the base image, if needed
+            if let Some(need_lsn) = need_base_image_lsn {
+                if let Some(predecessor) = &self.predecessor {
+                    need_base_image_lsn = predecessor.get_page_reconstruct_data(
+                        blknum,
+                        need_lsn,
+                        reconstruct_data,
+                    )?;
+                } else {
+                    bail!(
+                        "no base img found for {} at blk {} at LSN {}",
+                        self.seg,
+                        blknum,
+                        lsn
+                    );
+                }
+            }
+
+            // release lock on 'inner'
+        }
+
+        Ok(need_base_image_lsn)
+    }
+
+    /// Get size of the relation at given LSN
+    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
+        assert!(lsn >= self.start_lsn);
+
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let inner = self.load()?;
+        let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn)));
+
+        let result;
+        if let Some((_entry_lsn, entry)) = iter.next_back() {
+            result = *entry;
+        // Use the base image if needed
+        } else if let Some(predecessor) = &self.predecessor {
+            result = predecessor.get_seg_size(lsn)?;
+        } else {
+            result = 0;
+        }
+        Ok(result)
+    }
+
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
+        // Is the requested LSN after the rel was dropped?
+        if self.dropped && lsn >= self.end_lsn {
+            return Ok(false);
+        }
+
+        // Otherwise, it exists.
+        Ok(true)
+    }
+
+    ///
+    /// Release most of the memory used by this layer. If it's accessed again later,
+    /// it will need to be loaded back.
+    ///
+    fn unload(&self) -> Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.page_version_metas = BTreeMap::new();
+        inner.relsizes = BTreeMap::new();
+        inner.loaded = false;
+        Ok(())
+    }
+
+    fn delete(&self) -> Result<()> {
+        // delete underlying file
+        fs::remove_file(self.path())?;
+        Ok(())
+    }
+
+    fn is_incremental(&self) -> bool {
+        true
+    }
+}
+
+impl DeltaLayer {
+    fn path(&self) -> PathBuf {
+        Self::path_for(
+            self.conf,
+            self.timelineid,
+            self.tenantid,
+            &DeltaFileName {
+                seg: self.seg,
+                start_lsn: self.start_lsn,
+                end_lsn: self.end_lsn,
+                dropped: self.dropped,
+            },
+        )
+    }
+
+    fn path_for(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        fname: &DeltaFileName,
+    ) -> PathBuf {
+        conf.timeline_path(&timelineid, &tenantid)
+            .join(fname.to_string())
+    }
+
+    /// Create a new delta file, using the given btreemaps containing the page versions and
+    /// relsizes.
+    ///
+    /// This is used to write the in-memory layer to disk. The in-memory layer uses the same
+    /// data structure with two btreemaps as we do, so passing the btreemaps is currently
+    /// expedient.
+    pub fn create(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        seg: SegmentTag,
+        start_lsn: Lsn,
+        end_lsn: Lsn,
+        dropped: bool,
+        predecessor: Option<Arc<dyn Layer>>,
+        page_versions: BTreeMap<(u32, Lsn), PageVersion>,
+        relsizes: BTreeMap<Lsn, u32>,
+    ) -> Result<DeltaLayer> {
+        let delta_layer = DeltaLayer {
+            conf: conf,
+            timelineid: timelineid,
+            tenantid: tenantid,
+            seg: seg,
+            start_lsn: start_lsn,
+            end_lsn,
+            dropped,
+            inner: Mutex::new(DeltaLayerInner {
+                loaded: true,
+                page_version_metas: BTreeMap::new(),
+                relsizes: relsizes,
+            }),
+            predecessor,
+        };
+        let mut inner = delta_layer.inner.lock().unwrap();
+
+        // Write the in-memory btreemaps into a file
+        let path = delta_layer.path();
+
+        // Note: This overwrites any existing file. There shouldn't be any.
+        // FIXME: throw an error instead?
+        let file = File::create(&path)?;
+        let book = BookWriter::new(file, DELTA_FILE_MAGIC)?;
+
+        let mut page_version_writer = BlobWriter::new(book, PAGE_VERSIONS_CHAPTER);
+
+        for (key, page_version) in page_versions {
+            let page_image_range = page_version
+                .page_image
+                .map(|page_image| page_version_writer.write_blob(page_image.as_ref()))
+                .transpose()?;
+
+            let record_range = page_version
+                .record
+                .map(|record| {
+                    let buf = WALRecord::ser(&record)?;
+                    page_version_writer.write_blob(&buf)
+                })
+                .transpose()?;
+
+            let old = inner.page_version_metas.insert(
+                key,
+                PageVersionMeta {
+                    page_image_range,
+                    record_range,
+                },
+            );
+
+            assert!(old.is_none());
+        }
+
+        let book = page_version_writer.close()?;
+
+        // Write out page versions
+        let mut chapter = book.new_chapter(PAGE_VERSION_METAS_CHAPTER);
+        let buf = BTreeMap::ser(&inner.page_version_metas)?;
+        chapter.write_all(&buf)?;
+        let book = chapter.close()?;
+
+        // and relsizes to separate chapter
+        let mut chapter = book.new_chapter(REL_SIZES_CHAPTER);
+        let buf = BTreeMap::ser(&inner.relsizes)?;
+        chapter.write_all(&buf)?;
+        let book = chapter.close()?;
+
+        book.close()?;
+
+        trace!("saved {}", &path.display());
+
+        drop(inner);
+
+        Ok(delta_layer)
+    }
+
+    fn open_book(&self) -> Result<(PathBuf, Book<File>)> {
+        let path = Self::path_for(
+            self.conf,
+            self.timelineid,
+            self.tenantid,
+            &DeltaFileName {
+                seg: self.seg,
+                start_lsn: self.start_lsn,
+                end_lsn: self.end_lsn,
+                dropped: self.dropped,
+            },
+        );
+
+        let file = File::open(&path)?;
+        let book = Book::new(file)?;
+
+        Ok((path, book))
+    }
+
+    ///
+    /// Load the contents of the file into memory
+    ///
+    fn load(&self) -> Result<MutexGuard<DeltaLayerInner>> {
+        // quick exit if already loaded
+        let mut inner = self.inner.lock().unwrap();
+
+        if inner.loaded {
+            return Ok(inner);
+        }
+
+        let (path, book) = self.open_book()?;
+
+        let chapter = book.read_chapter(PAGE_VERSION_METAS_CHAPTER)?;
+        let page_version_metas = BTreeMap::des(&chapter)?;
+
+        let chapter = book.read_chapter(REL_SIZES_CHAPTER)?;
+        let relsizes = BTreeMap::des(&chapter)?;
+
+        debug!("loaded from {}", &path.display());
+
+        *inner = DeltaLayerInner {
+            loaded: true,
+            page_version_metas,
+            relsizes,
+        };
+
+        Ok(inner)
+    }
+
+    /// Create a DeltaLayer struct representing an existing file on disk.
+    pub fn new(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        filename: &DeltaFileName,
+        predecessor: Option<Arc<dyn Layer>>,
+    ) -> DeltaLayer {
+        DeltaLayer {
+            conf,
+            timelineid,
+            tenantid,
+            seg: filename.seg,
+            start_lsn: filename.start_lsn,
+            end_lsn: filename.end_lsn,
+            dropped: filename.dropped,
+            inner: Mutex::new(DeltaLayerInner {
+                loaded: false,
+                page_version_metas: BTreeMap::new(),
+                relsizes: BTreeMap::new(),
+            }),
+            predecessor,
+        }
+    }
+
+    /// debugging function to print out the contents of the layer
+    #[allow(unused)]
+    pub fn dump(&self) -> String {
+        let mut result = format!(
+            "----- snapshot layer for {} {}-{} ----\n",
+            self.seg, self.start_lsn, self.end_lsn
+        );
+
+        let inner = self.inner.lock().unwrap();
+        for (k, v) in inner.relsizes.iter() {
+            result += &format!("{}: {}\n", k, v);
+        }
+        //for (k, v) in inner.page_versions.iter() {
+        //    result += &format!("blk {} at {}: {}/{}\n", k.0, k.1, v.page_image.is_some(), v.record.is_some());
+        //}
+
+        result
+    }
+}
--- a/pageserver/src/layered_repository/filename.rs
+++ b/pageserver/src/layered_repository/filename.rs
@@ -0,0 +1,305 @@
+//!
+//! Helper functions for dealing with filenames of the image and delta layer files.
+//!
+use crate::layered_repository::storage_layer::SegmentTag;
+use crate::relish::*;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
+use std::fmt;
+use std::fs;
+
+use anyhow::Result;
+use log::*;
+use zenith_utils::lsn::Lsn;
+
+// Note: LayeredTimeline::load_layer_map() relies on this sort order
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+pub struct DeltaFileName {
+    pub seg: SegmentTag,
+    pub start_lsn: Lsn,
+    pub end_lsn: Lsn,
+    pub dropped: bool,
+}
+
+/// Represents the filename of a DeltaLayer
+///
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>
+///
+/// or if it was dropped:
+///
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>_DROPPED
+///
+impl DeltaFileName {
+    ///
+    /// Parse a string as a delta file name. Returns None if the filename does not
+    /// match the expected pattern.
+    ///
+    fn from_str(fname: &str) -> Option<Self> {
+        let rel;
+        let mut parts;
+        if let Some(rest) = fname.strip_prefix("rel_") {
+            parts = rest.split('_');
+            rel = RelishTag::Relation(RelTag {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+                relnode: parts.next()?.parse::<u32>().ok()?,
+                forknum: parts.next()?.parse::<u8>().ok()?,
+            });
+        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
+            parts = rest.split('_');
+            rel = RelishTag::FileNodeMap {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
+            parts = rest.split('_');
+            rel = RelishTag::TwoPhase {
+                xid: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
+            parts = rest.split('_');
+            rel = RelishTag::Checkpoint;
+        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
+            parts = rest.split('_');
+            rel = RelishTag::ControlFile;
+        } else {
+            return None;
+        }
+
+        let segno = parts.next()?.parse::<u32>().ok()?;
+
+        let seg = SegmentTag { rel, segno };
+
+        let start_lsn = Lsn::from_hex(parts.next()?).ok()?;
+        let end_lsn = Lsn::from_hex(parts.next()?).ok()?;
+
+        let mut dropped = false;
+        if let Some(suffix) = parts.next() {
+            if suffix == "DROPPED" {
+                dropped = true;
+            } else {
+                return None;
+            }
+        }
+        if parts.next().is_some() {
+            return None;
+        }
+
+        Some(DeltaFileName {
+            seg,
+            start_lsn,
+            end_lsn,
+            dropped,
+        })
+    }
+
+    fn to_string(&self) -> String {
+        let basename = match self.seg.rel {
+            RelishTag::Relation(reltag) => format!(
+                "rel_{}_{}_{}_{}",
+                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
+            ),
+            RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno,
+            } => format!("pg_xact_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno,
+            } => format!("pg_multixact_members_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno,
+            } => format!("pg_multixact_offsets_{:04X}", segno),
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
+            RelishTag::Checkpoint => format!("pg_control_checkpoint"),
+            RelishTag::ControlFile => format!("pg_control"),
+        };
+
+        format!(
+            "{}_{}_{:016X}_{:016X}{}",
+            basename,
+            self.seg.segno,
+            u64::from(self.start_lsn),
+            u64::from(self.end_lsn),
+            if self.dropped { "_DROPPED" } else { "" }
+        )
+    }
+}
+
+impl fmt::Display for DeltaFileName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.to_string())
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+pub struct ImageFileName {
+    pub seg: SegmentTag,
+    pub lsn: Lsn,
+}
+
+///
+/// Represents the filename of an ImageLayer
+///
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<LSN>
+///
+impl ImageFileName {
+    ///
+    /// Parse a string as an image file name. Returns None if the filename does not
+    /// match the expected pattern.
+    ///
+    fn from_str(fname: &str) -> Option<Self> {
+        let rel;
+        let mut parts;
+        if let Some(rest) = fname.strip_prefix("rel_") {
+            parts = rest.split('_');
+            rel = RelishTag::Relation(RelTag {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+                relnode: parts.next()?.parse::<u32>().ok()?,
+                forknum: parts.next()?.parse::<u8>().ok()?,
+            });
+        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
+            parts = rest.split('_');
+            rel = RelishTag::FileNodeMap {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
+            parts = rest.split('_');
+            rel = RelishTag::TwoPhase {
+                xid: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
+            parts = rest.split('_');
+            rel = RelishTag::Checkpoint;
+        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
+            parts = rest.split('_');
+            rel = RelishTag::ControlFile;
+        } else {
+            return None;
+        }
+
+        let segno = parts.next()?.parse::<u32>().ok()?;
+
+        let seg = SegmentTag { rel, segno };
+
+        let lsn = Lsn::from_hex(parts.next()?).ok()?;
+
+        if parts.next().is_some() {
+            return None;
+        }
+
+        Some(ImageFileName { seg, lsn })
+    }
+
+    fn to_string(&self) -> String {
+        let basename = match self.seg.rel {
+            RelishTag::Relation(reltag) => format!(
+                "rel_{}_{}_{}_{}",
+                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
+            ),
+            RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno,
+            } => format!("pg_xact_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno,
+            } => format!("pg_multixact_members_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno,
+            } => format!("pg_multixact_offsets_{:04X}", segno),
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
+            RelishTag::Checkpoint => format!("pg_control_checkpoint"),
+            RelishTag::ControlFile => format!("pg_control"),
+        };
+
+        format!(
+            "{}_{}_{:016X}",
+            basename,
+            self.seg.segno,
+            u64::from(self.lsn),
+        )
+    }
+}
+
+impl fmt::Display for ImageFileName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.to_string())
+    }
+}
+
+/// Scan timeline directory and create ImageFileName and DeltaFilename
+/// structs representing all files on disk
+///
+/// TODO: returning an Iterator would be more idiomatic
+pub fn list_files(
+    conf: &'static PageServerConf,
+    timelineid: ZTimelineId,
+    tenantid: ZTenantId,
+) -> Result<(Vec<ImageFileName>, Vec<DeltaFileName>)> {
+    let path = conf.timeline_path(&timelineid, &tenantid);
+
+    let mut deltafiles: Vec<DeltaFileName> = Vec::new();
+    let mut imgfiles: Vec<ImageFileName> = Vec::new();
+    for direntry in fs::read_dir(path)? {
+        let fname = direntry?.file_name();
+        let fname = fname.to_str().unwrap();
+
+        if let Some(deltafilename) = DeltaFileName::from_str(fname) {
+            deltafiles.push(deltafilename);
+        } else if let Some(imgfilename) = ImageFileName::from_str(fname) {
+            imgfiles.push(imgfilename);
+        } else if fname == "wal" || fname == "metadata" {
+            // ignore these
+        } else {
+            warn!("unrecognized filename in timeline dir: {}", fname);
+        }
+    }
+    return Ok((imgfiles, deltafiles));
+}
--- a/pageserver/src/layered_repository/image_layer.rs
+++ b/pageserver/src/layered_repository/image_layer.rs
@@ -0,0 +1,405 @@
+//! An ImageLayer represents an image or a snapshot of a segment at one particular LSN.
+//! It is stored in a file on disk.
+//!
+//! On disk, the image files are stored in timelines/<timelineid> directory.
+//! Currently, there are no subdirectories, and each snapshot file is named like this:
+//!
+//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<LSN>
+//!
+//! For example:
+//!
+//!    1663_13990_2609_0_5_000000000169C348
+//!
+//! An image file is constructed using the 'bookfile' crate.
+//!
+//! Only metadata is loaded into memory by the load function.
+//! When images are needed, they are read directly from disk.
+//!
+//! For blocky segments, the images are stored in BLOCKY_IMAGES_CHAPTER.
+//! All the images are required to be BLOCK_SIZE, which allows for random access.
+//!
+//! For non-blocky segments, the image can be found in NONBLOCKY_IMAGE_CHAPTER.
+//!
+use crate::layered_repository::filename::ImageFileName;
+use crate::layered_repository::storage_layer::{Layer, PageReconstructData, SegmentTag};
+use crate::layered_repository::LayeredTimeline;
+use crate::layered_repository::RELISH_SEG_SIZE;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
+use anyhow::{anyhow, bail, ensure, Result};
+use bytes::Bytes;
+use log::*;
+use std::convert::TryInto;
+use std::fs;
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+use std::sync::{Mutex, MutexGuard};
+
+use bookfile::{Book, BookWriter};
+
+use zenith_utils::lsn::Lsn;
+
+// Magic constant to identify a Zenith segment image file
+const IMAGE_FILE_MAGIC: u32 = 0x5A616E01 + 1;
+
+/// Contains each block in block # order
+const BLOCKY_IMAGES_CHAPTER: u64 = 1;
+const NONBLOCKY_IMAGE_CHAPTER: u64 = 2;
+
+const BLOCK_SIZE: usize = 8192;
+
+///
+/// ImageLayer is the in-memory data structure associated with an on-disk image
+/// file.  We keep an ImageLayer in memory for each file, in the LayerMap. If a
+/// layer is in "loaded" state, we have a copy of the file in memory, in 'inner'.
+/// Otherwise the struct is just a placeholder for a file that exists on disk,
+/// and it needs to be loaded before using it in queries.
+///
+pub struct ImageLayer {
+    conf: &'static PageServerConf,
+    pub tenantid: ZTenantId,
+    pub timelineid: ZTimelineId,
+    pub seg: SegmentTag,
+
+    // This entry contains an image of all pages as of this LSN
+    pub lsn: Lsn,
+
+    inner: Mutex<ImageLayerInner>,
+}
+
+#[derive(Clone)]
+enum ImageType {
+    Blocky { num_blocks: u32 },
+    NonBlocky,
+}
+
+pub struct ImageLayerInner {
+    /// If false, the 'image_type' has not been
+    /// loaded into memory yet.
+    loaded: bool,
+
+    /// Derived from filename and bookfile chapter metadata
+    image_type: ImageType,
+}
+
+impl Layer for ImageLayer {
+    fn filename(&self) -> PathBuf {
+        PathBuf::from(
+            ImageFileName {
+                seg: self.seg,
+                lsn: self.lsn,
+            }
+            .to_string(),
+        )
+    }
+
+    fn get_timeline_id(&self) -> ZTimelineId {
+        return self.timelineid;
+    }
+
+    fn get_seg_tag(&self) -> SegmentTag {
+        return self.seg;
+    }
+
+    fn is_dropped(&self) -> bool {
+        return false;
+    }
+
+    fn get_start_lsn(&self) -> Lsn {
+        return self.lsn;
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
+        return self.lsn;
+    }
+
+    /// Look up given page in the file
+    fn get_page_reconstruct_data(
+        &self,
+        blknum: u32,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<Option<Lsn>> {
+        assert!(lsn >= self.lsn);
+
+        let inner = self.load()?;
+
+        let base_blknum = blknum % RELISH_SEG_SIZE;
+
+        let (_path, book) = self.open_book()?;
+
+        let buf = match &inner.image_type {
+            ImageType::Blocky { num_blocks } => {
+                if base_blknum >= *num_blocks {
+                    bail!(
+                        "no base img found for {} at blk {} at LSN {}",
+                        self.seg,
+                        base_blknum,
+                        lsn
+                    );
+                }
+
+                let mut buf = vec![0u8; BLOCK_SIZE];
+                let offset = BLOCK_SIZE as u64 * base_blknum as u64;
+
+                let chapter = book.chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
+                chapter.read_exact_at(&mut buf, offset)?;
+
+                buf
+            }
+            ImageType::NonBlocky => {
+                ensure!(base_blknum == 0);
+                book.read_chapter(NONBLOCKY_IMAGE_CHAPTER)?.into_vec()
+            }
+        };
+
+        reconstruct_data.page_img = Some(Bytes::from(buf));
+        Ok(None)
+    }
+
+    /// Get size of the segment
+    fn get_seg_size(&self, _lsn: Lsn) -> Result<u32> {
+        let inner = self.load()?;
+        match inner.image_type {
+            ImageType::Blocky { num_blocks } => Ok(num_blocks),
+            ImageType::NonBlocky => Err(anyhow!("get_seg_size called for non-blocky segment")),
+        }
+    }
+
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, _lsn: Lsn) -> Result<bool> {
+        Ok(true)
+    }
+
+    ///
+    /// Release most of the memory used by this layer. If it's accessed again later,
+    /// it will need to be loaded back.
+    ///
+    fn unload(&self) -> Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.image_type = ImageType::Blocky { num_blocks: 0 };
+        inner.loaded = false;
+        Ok(())
+    }
+
+    fn delete(&self) -> Result<()> {
+        // delete underlying file
+        fs::remove_file(self.path())?;
+        Ok(())
+    }
+
+    fn is_incremental(&self) -> bool {
+        false
+    }
+}
+
+impl ImageLayer {
+    fn path(&self) -> PathBuf {
+        Self::path_for(
+            self.conf,
+            self.timelineid,
+            self.tenantid,
+            &ImageFileName {
+                seg: self.seg,
+                lsn: self.lsn,
+            },
+        )
+    }
+
+    fn path_for(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        fname: &ImageFileName,
+    ) -> PathBuf {
+        conf.timeline_path(&timelineid, &tenantid)
+            .join(fname.to_string())
+    }
+
+    /// Create a new image file, using the given array of pages.
+    fn create(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        seg: SegmentTag,
+        lsn: Lsn,
+        base_images: Vec<Bytes>,
+    ) -> Result<ImageLayer> {
+        let image_type = if seg.rel.is_blocky() {
+            let num_blocks: u32 = base_images.len().try_into()?;
+            ImageType::Blocky { num_blocks }
+        } else {
+            assert_eq!(base_images.len(), 1);
+            ImageType::NonBlocky
+        };
+
+        let layer = ImageLayer {
+            conf: conf,
+            timelineid: timelineid,
+            tenantid: tenantid,
+            seg: seg,
+            lsn: lsn,
+            inner: Mutex::new(ImageLayerInner {
+                loaded: true,
+                image_type: image_type.clone(),
+            }),
+        };
+        let inner = layer.inner.lock().unwrap();
+
+        // Write the images into a file
+        let path = layer.path();
+
+        // Note: This overwrites any existing file. There shouldn't be any.
+        // FIXME: throw an error instead?
+        let file = File::create(&path)?;
+        let book = BookWriter::new(file, IMAGE_FILE_MAGIC)?;
+
+        let book = match &image_type {
+            ImageType::Blocky { .. } => {
+                let mut chapter = book.new_chapter(BLOCKY_IMAGES_CHAPTER);
+                for block_bytes in base_images {
+                    assert_eq!(block_bytes.len(), BLOCK_SIZE);
+                    chapter.write_all(&block_bytes)?;
+                }
+                chapter.close()?
+            }
+            ImageType::NonBlocky => {
+                let mut chapter = book.new_chapter(NONBLOCKY_IMAGE_CHAPTER);
+                chapter.write_all(&base_images[0])?;
+                chapter.close()?
+            }
+        };
+
+        book.close()?;
+
+        trace!("saved {}", &path.display());
+
+        drop(inner);
+
+        Ok(layer)
+    }
+
+    // Create a new image file by materializing every page in a source layer
+    // at given LSN.
+    pub fn create_from_src(
+        conf: &'static PageServerConf,
+        timeline: &LayeredTimeline,
+        src: &dyn Layer,
+        lsn: Lsn,
+    ) -> Result<ImageLayer> {
+        let seg = src.get_seg_tag();
+        let timelineid = timeline.timelineid;
+
+        let startblk;
+        let size;
+        if seg.rel.is_blocky() {
+            size = src.get_seg_size(lsn)?;
+            startblk = seg.segno * RELISH_SEG_SIZE;
+        } else {
+            size = 1;
+            startblk = 0;
+        }
+
+        trace!(
+            "creating new ImageLayer for {} on timeline {} at {}",
+            seg,
+            timelineid,
+            lsn,
+        );
+
+        let mut base_images: Vec<Bytes> = Vec::new();
+        for blknum in startblk..(startblk + size) {
+            let img = timeline.materialize_page(seg, blknum, lsn, &*src)?;
+
+            base_images.push(img);
+        }
+
+        Self::create(conf, timelineid, timeline.tenantid, seg, lsn, base_images)
+    }
+
+    ///
+    /// Load the contents of the file into memory
+    ///
+    fn load(&self) -> Result<MutexGuard<ImageLayerInner>> {
+        // quick exit if already loaded
+        let mut inner = self.inner.lock().unwrap();
+
+        if inner.loaded {
+            return Ok(inner);
+        }
+
+        let (path, book) = self.open_book()?;
+
+        let image_type = if self.seg.rel.is_blocky() {
+            let chapter = book.chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
+            let images_len = chapter.len();
+            ensure!(images_len % BLOCK_SIZE as u64 == 0);
+            let num_blocks: u32 = (images_len / BLOCK_SIZE as u64).try_into()?;
+            ImageType::Blocky { num_blocks }
+        } else {
+            let _chapter = book.chapter_reader(NONBLOCKY_IMAGE_CHAPTER)?;
+            ImageType::NonBlocky
+        };
+
+        debug!("loaded from {}", &path.display());
+
+        *inner = ImageLayerInner {
+            loaded: true,
+            image_type,
+        };
+
+        Ok(inner)
+    }
+
+    fn open_book(&self) -> Result<(PathBuf, Book<File>)> {
+        let path = Self::path_for(
+            self.conf,
+            self.timelineid,
+            self.tenantid,
+            &ImageFileName {
+                seg: self.seg,
+                lsn: self.lsn,
+            },
+        );
+
+        let file = File::open(&path)?;
+        let book = Book::new(file)?;
+
+        Ok((path, book))
+    }
+
+    /// Create an ImageLayer struct representing an existing file on disk
+    pub fn new(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        filename: &ImageFileName,
+    ) -> ImageLayer {
+        ImageLayer {
+            conf,
+            timelineid,
+            tenantid,
+            seg: filename.seg,
+            lsn: filename.lsn,
+            inner: Mutex::new(ImageLayerInner {
+                loaded: false,
+                image_type: ImageType::Blocky { num_blocks: 0 },
+            }),
+        }
+    }
+
+    /// debugging function to print out the contents of the layer
+    #[allow(unused)]
+    pub fn dump(&self) -> String {
+        let mut result = format!("----- image layer for {} at {} ----\n", self.seg, self.lsn);
+
+        //let inner = self.inner.lock().unwrap();
+
+        //for (k, v) in inner.page_versions.iter() {
+        //    result += &format!("blk {} at {}: {}/{}\n", k.0, k.1, v.page_image.is_some(), v.record.is_some());
+        //}
+
+        result
+    }
+}
--- a/pageserver/src/layered_repository/inmemory_layer.rs
+++ b/pageserver/src/layered_repository/inmemory_layer.rs
@@ -2,13 +2,13 @@
 //! An in-memory layer stores recently received page versions in memory. The page versions
 //! are held in a BTreeMap, and there's another BTreeMap to track the size of the relation.
 //!
-
-use crate::layered_repository::storage_layer::Layer;
-use crate::layered_repository::storage_layer::PageVersion;
-use crate::layered_repository::SnapshotLayer;
-use crate::relish::*;
+use crate::layered_repository::filename::DeltaFileName;
+use crate::layered_repository::storage_layer::{
+    Layer, PageReconstructData, PageVersion, SegmentTag, RELISH_SEG_SIZE,
+};
+use crate::layered_repository::LayeredTimeline;
+use crate::layered_repository::{DeltaLayer, ImageLayer};
 use crate::repository::WALRecord;
-use crate::walredo::WalRedoManager;
 use crate::PageServerConf;
 use crate::{ZTenantId, ZTimelineId};
 use anyhow::{bail, Result};
@@ -16,17 +16,16 @@ use bytes::Bytes;
 use log::*;
 use std::collections::BTreeMap;
 use std::ops::Bound::Included;
+use std::path::PathBuf;
 use std::sync::{Arc, Mutex};

 use zenith_utils::lsn::Lsn;

-static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
-
 pub struct InMemoryLayer {
    conf: &'static PageServerConf,
    tenantid: ZTenantId,
    timelineid: ZTimelineId,
-    rel: RelishTag,
+    seg: SegmentTag,

    ///
    /// This layer contains all the changes from 'start_lsn'. The
@@ -35,13 +34,20 @@ pub struct InMemoryLayer {
    ///
    start_lsn: Lsn,

+    /// LSN of the oldest page version stored in this layer
+    oldest_pending_lsn: Lsn,
+
+    /// The above fields never change. The parts that do change are in 'inner',
+    /// and protected by mutex.
    inner: Mutex<InMemoryLayerInner>,
+
+    /// Predecessor layer
+    img_layer: Option<Arc<dyn Layer>>,
 }

 pub struct InMemoryLayerInner {
-    /// If this relation was dropped, remember when that happened. Lsn(0) means
-    /// it hasn't been dropped
-    drop_lsn: Lsn,
+    /// If this relation was dropped, remember when that happened.
+    drop_lsn: Option<Lsn>,

    ///
    /// All versions of all pages in the layer are are kept here.
@@ -50,22 +56,57 @@ pub struct InMemoryLayerInner {
    page_versions: BTreeMap<(u32, Lsn), PageVersion>,

    ///
-    /// `relsizes` tracks the size of the relation at different points in time.
+    /// `segsizes` tracks the size of the segment at different points in time.
    ///
-    relsizes: BTreeMap<Lsn, u32>,
+    segsizes: BTreeMap<Lsn, u32>,
+}
+
+impl InMemoryLayerInner {
+    fn get_seg_size(&self, lsn: Lsn) -> u32 {
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let mut iter = self.segsizes.range((Included(&Lsn(0)), Included(&lsn)));
+
+        if let Some((_entry_lsn, entry)) = iter.next_back() {
+            *entry
+        } else {
+            0
+        }
+    }
 }

 impl Layer for InMemoryLayer {
-    fn is_frozen(&self) -> bool {
-        return false;
+    // An in-memory layer doesn't really have a filename as it's not stored on disk,
+    // but we construct a filename as if it was a delta layer
+    fn filename(&self) -> PathBuf {
+        let inner = self.inner.lock().unwrap();
+
+        let end_lsn;
+        let dropped;
+        if let Some(drop_lsn) = inner.drop_lsn {
+            end_lsn = drop_lsn;
+            dropped = true;
+        } else {
+            end_lsn = Lsn(u64::MAX);
+            dropped = false;
+        }
+
+        let delta_filename = DeltaFileName {
+            seg: self.seg,
+            start_lsn: self.start_lsn,
+            end_lsn: end_lsn,
+            dropped: dropped,
+        }
+        .to_string();
+
+        PathBuf::from(format!("inmem-{}", delta_filename))
    }

    fn get_timeline_id(&self) -> ZTimelineId {
        return self.timelineid;
    }

-    fn get_relish_tag(&self) -> RelishTag {
-        return self.rel;
+    fn get_seg_tag(&self) -> SegmentTag {
+        return self.seg;
    }

    fn get_start_lsn(&self) -> Lsn {
@@ -73,38 +114,46 @@ impl Layer for InMemoryLayer {
    }

    fn get_end_lsn(&self) -> Lsn {
-        return Lsn(u64::MAX);
+        let inner = self.inner.lock().unwrap();
+
+        if let Some(drop_lsn) = inner.drop_lsn {
+            drop_lsn
+        } else {
+            Lsn(u64::MAX)
+        }
    }

    fn is_dropped(&self) -> bool {
        let inner = self.inner.lock().unwrap();
-        return inner.drop_lsn != Lsn(0);
+        inner.drop_lsn.is_some()
    }

    /// Look up given page in the cache.
-    fn get_page_at_lsn(
+    fn get_page_reconstruct_data(
        &self,
-        walredo_mgr: &dyn WalRedoManager,
        blknum: u32,
        lsn: Lsn,
-    ) -> Result<Bytes> {
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let mut records: Vec<WALRecord> = Vec::new();
-        let mut page_img: Option<Bytes> = None;
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<Option<Lsn>> {
+        // Scan the BTreeMap backwards, starting from reconstruct_data.lsn.
        let mut need_base_image_lsn: Option<Lsn> = Some(lsn);

+        assert!(self.seg.blknum_in_seg(blknum));
+
        {
            let inner = self.inner.lock().unwrap();
            let minkey = (blknum, Lsn(0));
            let maxkey = (blknum, lsn);
-            let mut iter = inner. page_versions.range((Included(&minkey), Included(&maxkey)));
+            let mut iter = inner
+                .page_versions
+                .range((Included(&minkey), Included(&maxkey)));
            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
                if let Some(img) = &entry.page_image {
-                    page_img = Some(img.clone());
+                    reconstruct_data.page_img = Some(img.clone());
                    need_base_image_lsn = None;
                    break;
                } else if let Some(rec) = &entry.record {
-                    records.push(rec.clone());
+                    reconstruct_data.records.push(rec.clone());
                    if rec.will_init {
                        // This WAL record initializes the page, so no need to go further back
                        need_base_image_lsn = None;
@@ -118,115 +167,143 @@ impl Layer for InMemoryLayer {
                }
            }

-            // release lock on 'page_versions'
-        }
-        records.reverse();
-
-        // If we needed a base image to apply the WAL records against, we should have found it in memory.
-        if let Some(lsn) = need_base_image_lsn {
-            if records.is_empty() {
-                // no records, and no base image. This can happen if PostgreSQL extends a relation
-                // but never writes the page.
-                //
-                // Would be nice to detect that situation better.
-                warn!("Page {} blk {} at {} not found", self.rel, blknum, lsn);
-                return Ok(ZERO_PAGE.clone());
-            }
-            bail!(
-                "No base image found for page {} blk {} at {}/{}",
-                self.rel,
-                blknum,
-                self.timelineid,
-                lsn
-            );
-        }
-
-        // If we have a page image, and no WAL, we're all set
-        if records.is_empty() {
-            if let Some(img) = page_img {
-                trace!(
-                    "found page image for blk {} in {} at {}/{}, no WAL redo required",
-                    blknum,
-                    self.rel,
-                    self.timelineid,
-                    lsn
-                );
-                Ok(img)
-            } else {
-                // FIXME: this ought to be an error?
-                warn!("Page {} blk {} at {} not found", self.rel, blknum, lsn);
-                Ok(ZERO_PAGE.clone())
-            }
-        } else {
-            // We need to do WAL redo.
-            //
-            // If we don't have a base image, then the oldest WAL record better initialize
-            // the page
-            if page_img.is_none() && !records.first().unwrap().will_init {
-                // FIXME: this ought to be an error?
-                warn!(
-                    "Base image for page {}/{} at {} not found, but got {} WAL records",
-                    self.rel,
-                    blknum,
-                    lsn,
-                    records.len()
-                );
-                Ok(ZERO_PAGE.clone())
-            } else {
-                if page_img.is_some() {
-                    trace!("found {} WAL records and a base image for blk {} in {} at {}/{}, performing WAL redo", records.len(), blknum, self.rel, self.timelineid, lsn);
+            // Use the base image, if needed
+            if let Some(need_lsn) = need_base_image_lsn {
+                if let Some(img_layer) = &self.img_layer {
+                    need_base_image_lsn =
+                        img_layer.get_page_reconstruct_data(blknum, need_lsn, reconstruct_data)?;
                } else {
-                    trace!("found {} WAL records that will init the page for blk {} in {} at {}/{}, performing WAL redo", records.len(), blknum, self.rel, self.timelineid, lsn);
+                    bail!(
+                        "no base img found for {} at blk {} at LSN {}",
+                        self.seg,
+                        blknum,
+                        lsn
+                    );
                }
-                let img = walredo_mgr.request_redo(self.rel, blknum, lsn, page_img, records)?;
-
-                self.put_page_image(blknum, lsn, img.clone())?;
-
-                Ok(img)
            }
+
+            // release lock on 'inner'
        }
+
+        Ok(need_base_image_lsn)
    }

    /// Get size of the relation at given LSN
-    fn get_rel_size(&self, lsn: Lsn) -> Result<u32> {
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let inner = self.inner.lock().unwrap();
-        let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn)));
+    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
+        assert!(lsn >= self.start_lsn);

-        if let Some((_entry_lsn, entry)) = iter.next_back() {
-            let result = *entry;
-            drop(inner);
-            trace!("get_relsize: {} at {} -> {}", self.rel, lsn, result);
-            Ok(result)
-        } else {
-            bail!("No size found for {} at {} in memory", self.rel, lsn);
-        }
+        let inner = self.inner.lock().unwrap();
+        Ok(inner.get_seg_size(lsn))
    }

-    /// Does this relation exist at given LSN?
-    fn get_rel_exists(&self, lsn: Lsn) -> Result<bool> {
-        // Scan the BTreeMap backwards, starting from the given entry.
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
        let inner = self.inner.lock().unwrap();

-        let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn)));
+        // Is the requested LSN after the segment was dropped?
+        if let Some(drop_lsn) = inner.drop_lsn {
+            if lsn >= drop_lsn {
+                return Ok(false);
+            }
+        }

-        let result = if let Some((_entry_lsn, _entry)) = iter.next_back() {
-            true
-        } else {
-            false
-        };
-        Ok(result)
+        // Otherwise, it exists
+        Ok(true)
+    }
+
+    /// Cannot unload anything in an in-memory layer, since there's no backing
+    /// store. To release memory used by an in-memory layer, use 'freeze' to turn
+    /// it into an on-disk layer.
+    fn unload(&self) -> Result<()> {
+        Ok(())
+    }
+
+    /// Nothing to do here. When you drop the last reference to the layer, it will
+    /// be deallocated.
+    fn delete(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn is_incremental(&self) -> bool {
+        self.img_layer.is_some()
+    }
+}
+
+impl InMemoryLayer {
+    /// Return the oldest page version that's stored in this layer
+    pub fn get_oldest_pending_lsn(&self) -> Lsn {
+        self.oldest_pending_lsn
+    }
+
+    ///
+    /// Create a new, empty, in-memory layer
+    ///
+    pub fn create(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        seg: SegmentTag,
+        start_lsn: Lsn,
+        oldest_pending_lsn: Lsn,
+    ) -> Result<InMemoryLayer> {
+        trace!(
+            "initializing new empty InMemoryLayer for writing {} on timeline {} at {}",
+            seg,
+            timelineid,
+            start_lsn
+        );
+
+        Ok(InMemoryLayer {
+            conf,
+            timelineid,
+            tenantid,
+            seg,
+            start_lsn,
+            oldest_pending_lsn,
+            inner: Mutex::new(InMemoryLayerInner {
+                drop_lsn: None,
+                page_versions: BTreeMap::new(),
+                segsizes: BTreeMap::new(),
+            }),
+            img_layer: None,
+        })
    }

    // Write operations

+    /// Remember new page version, as a WAL record over previous version
+    pub fn put_wal_record(&self, blknum: u32, rec: WALRecord) -> Result<()> {
+        self.put_page_version(
+            blknum,
+            rec.lsn,
+            PageVersion {
+                page_image: None,
+                record: Some(rec),
+            },
+        )
+    }
+
+    /// Remember new page version, as a full page image
+    pub fn put_page_image(&self, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()> {
+        self.put_page_version(
+            blknum,
+            lsn,
+            PageVersion {
+                page_image: Some(img),
+                record: None,
+            },
+        )
+    }
+
    /// Common subroutine of the public put_wal_record() and put_page_image() functions.
    /// Adds the page version to the in-memory tree
-    fn put_page_version(&self, blknum: u32, lsn: Lsn, pv: PageVersion) -> Result<()> {
+    pub fn put_page_version(&self, blknum: u32, lsn: Lsn, pv: PageVersion) -> Result<()> {
+        assert!(self.seg.blknum_in_seg(blknum));
+
        trace!(
            "put_page_version blk {} of {} at {}/{}",
            blknum,
-            self.rel,
+            self.seg.rel,
            self.timelineid,
            lsn
        );
@@ -237,31 +314,27 @@ impl Layer for InMemoryLayer {
        if old.is_some() {
            // We already had an entry for this LSN. That's odd..
            warn!(
-                "Page version of rel {:?} blk {} at {} already exists",
-                self.rel, blknum, lsn
+                "Page version of rel {} blk {} at {} already exists",
+                self.seg.rel, blknum, lsn
            );
        }

        // Also update the relation size, if this extended the relation.
-        if self.rel.is_blocky() {
-            let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn)));
+        if self.seg.rel.is_blocky() {
+            let newsize = blknum - self.seg.segno * RELISH_SEG_SIZE + 1;

-            let oldsize;
-            if let Some((_entry_lsn, entry)) = iter.next_back() {
-                oldsize = *entry;
-            } else {
-                oldsize = 0;
-                //bail!("No old size found for {} at {}", self.tag, lsn);
-            }
-            if blknum >= oldsize {
+            // use inner get_seg_size, since calling self.get_seg_size will try to acquire self.inner.lock
+            // which we've just acquired above
+            let oldsize = inner.get_seg_size(lsn);
+            if newsize > oldsize {
                trace!(
-                    "enlarging relation {} from {} to {} blocks at {}",
-                    self.rel,
+                    "enlarging segment {} from {} to {} blocks at {}",
+                    self.seg,
                    oldsize,
-                    blknum + 1,
+                    newsize,
                    lsn
                );
-                inner.relsizes.insert(lsn, blknum + 1);
+                inner.segsizes.insert(lsn, newsize);
            }
        }

@@ -269,9 +342,9 @@ impl Layer for InMemoryLayer {
    }

    /// Remember that the relation was truncated at given LSN
-    fn put_truncation(&self, lsn: Lsn, relsize: u32) -> anyhow::Result<()> {
+    pub fn put_truncation(&self, lsn: Lsn, segsize: u32) -> anyhow::Result<()> {
        let mut inner = self.inner.lock().unwrap();
-        let old = inner.relsizes.insert(lsn, relsize);
+        let old = inner.segsizes.insert(lsn, segsize);

        if old.is_some() {
            // We already had an entry for this LSN. That's odd..
@@ -281,204 +354,211 @@ impl Layer for InMemoryLayer {
        Ok(())
    }

-    /// Remember that the relation was truncated at given LSN
-    fn put_unlink(&self, lsn: Lsn) -> anyhow::Result<()> {
+    /// Remember that the segment was dropped at given LSN
+    pub fn put_unlink(&self, lsn: Lsn) -> anyhow::Result<()> {
        let mut inner = self.inner.lock().unwrap();

-        assert!(inner.drop_lsn == Lsn(0));
-        inner.drop_lsn = lsn;
+        assert!(inner.drop_lsn.is_none());
+        inner.drop_lsn = Some(lsn);

-        info!("dropped relation {} at {}", self.rel, lsn);
+        info!("dropped segment {} at {}", self.seg, lsn);

        Ok(())
    }

-    ///
-    /// Write the this in-memory layer to disk, as a snapshot layer.
-    ///
-    /// The cutoff point for the layer that's written to disk is 'end_lsn'.
-    /// If there were page versions newer than 'end_lsn', a new in-memory
-    /// layer is returned with those page versions. Otherwise returns None.
-    ///
-    fn freeze(&self, end_lsn: Lsn, walredo_mgr: &dyn WalRedoManager) -> Result<Vec<Arc<dyn Layer>>> {
-        info!(
-            "freezing in memory layer for {} on timeline {} at {}",
-            self.rel, self.timelineid, end_lsn
-        );
-
-        let inner = self.inner.lock().unwrap();
-        
-        let dropped = inner.drop_lsn != Lsn(0);
-
-        // Divide all the page versions into old and new at the 'end_lsn' cutoff point.
-        let mut old_page_versions = BTreeMap::new();
-        let mut old_relsizes = BTreeMap::new();
-        let mut new_relsizes = BTreeMap::new();
-        let mut new_page_versions = BTreeMap::new();
-
-        if !dropped {
-            for (lsn, size) in inner.relsizes.iter() {
-                if *lsn > end_lsn {
-                    new_relsizes.insert(*lsn, *size);
-                } else {
-                    old_relsizes.insert(*lsn, *size);
-                }
-            }
-
-            for ((blknum, lsn), pv) in inner.page_versions.iter() {
-                if *lsn > end_lsn {
-                    new_page_versions.insert((*blknum, *lsn), pv.clone());
-                } else {
-                    old_page_versions.insert((*blknum, *lsn), pv.clone());
-                }
-            }
-        }
-
-        let end_lsn = if dropped {
-            assert!(inner.drop_lsn < end_lsn);
-            inner.drop_lsn
-        } else {
-            end_lsn
-        };
-
-        // Write the old page versions to disk.
-        let snapfile = SnapshotLayer::create(
-            self.conf,
-            self.timelineid,
-            self.tenantid,
-            self.rel,
-            self.start_lsn,
-            end_lsn,
-            dropped,
-            old_page_versions,
-            old_relsizes,
-        )?;
-        let mut result: Vec<Arc<dyn Layer>> = Vec::new();
-
-        // If there were any "new" page versions, initialize a new in-memory layer to hold
-        // them
-        if !new_relsizes.is_empty() || !new_page_versions.is_empty() {
-            info!("created new in-mem layer for {} {}-", self.rel, end_lsn);
-
-            let new_layer = Self::copy_snapshot(self.conf, walredo_mgr, &snapfile, self.timelineid, self.tenantid, end_lsn)?;
-            let mut new_inner = new_layer.inner.lock().unwrap();
-            new_inner.page_versions.append(&mut new_page_versions);
-            new_inner.relsizes.append(&mut new_relsizes);
-            drop(new_inner);
-
-            result.push(Arc::new(new_layer));
-        }
-        result.push(Arc::new(snapfile));
-
-        Ok(result)
-    }
-
-    fn delete(&self) -> Result<()> {
-        // Nothing to do. When the reference is dropped, the memory is released.
-        Ok(())
-    }
-
-    fn unload(&self) -> Result<()> {
-        // cannot unload in-memory layer. Freeze instead
-        Ok(())
-    }
-}
-
-impl InMemoryLayer {
-    ///
-    /// Create a new, empty, in-memory layer
-    ///
-    pub fn create(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        rel: RelishTag,
-        start_lsn: Lsn,
-    ) -> Result<InMemoryLayer> {
-        trace!(
-            "initializing new empty InMemoryLayer for writing {} on timeline {} at {}",
-            rel,
-            timelineid,
-            start_lsn
-        );
-
-        Ok(InMemoryLayer {
-            conf,
-            timelineid,
-            tenantid,
-            rel,
-            start_lsn,
-            inner: Mutex::new(InMemoryLayerInner {
-                drop_lsn: Lsn(0),
-                page_versions: BTreeMap::new(),
-                relsizes: BTreeMap::new(),
-            }),
-        })
-    }
-
    ///
    /// Initialize a new InMemoryLayer for, by copying the state at the given
    /// point in time from given existing layer.
    ///
-    pub fn copy_snapshot(
+    pub fn create_successor_layer(
        conf: &'static PageServerConf,
-        walredo_mgr: &dyn WalRedoManager,
-        src: &dyn Layer,
+        src: Arc<dyn Layer>,
        timelineid: ZTimelineId,
        tenantid: ZTenantId,
-        lsn: Lsn,
+        start_lsn: Lsn,
+        oldest_pending_lsn: Lsn,
    ) -> Result<InMemoryLayer> {
+        let seg = src.get_seg_tag();
+
        trace!(
            "initializing new InMemoryLayer for writing {} on timeline {} at {}",
-            src.get_relish_tag(),
+            seg,
            timelineid,
-            lsn
+            start_lsn,
        );
-        let mut page_versions = BTreeMap::new();
-        let mut relsizes = BTreeMap::new();

-        let size;
-        if src.get_relish_tag().is_blocky() {
-            size = src.get_rel_size(lsn)?;
-            relsizes.insert(lsn, size);
-        } else {
-            size = 1;
-        }
-
-        for blknum in 0..size {
-            let img = src.get_page_at_lsn(walredo_mgr, blknum, lsn)?;
-            let pv = PageVersion {
-                page_image: Some(img),
-                record: None,
-            };
-            page_versions.insert((blknum, lsn), pv);
+        // For convenience, copy the segment size from the predecessor layer
+        let mut segsizes = BTreeMap::new();
+        if seg.rel.is_blocky() {
+            let size = src.get_seg_size(start_lsn)?;
+            segsizes.insert(start_lsn, size);
        }

        Ok(InMemoryLayer {
            conf,
            timelineid,
            tenantid,
-            rel: src.get_relish_tag(),
-            start_lsn: lsn,
+            seg,
+            start_lsn,
+            oldest_pending_lsn,
            inner: Mutex::new(InMemoryLayerInner {
-                drop_lsn: Lsn(0),
-                page_versions: page_versions,
-                relsizes: relsizes,
+                drop_lsn: None,
+                page_versions: BTreeMap::new(),
+                segsizes: segsizes,
            }),
+            img_layer: Some(src),
        })
    }

+    ///
+    /// Write the this in-memory layer to disk, as a snapshot layer.
+    ///
+    /// The cutoff point for the layer that's written to disk is 'end_lsn'.
+    ///
+    /// Returns new layers that replace this one. Always returns a new image
+    /// layer containing the page versions at the cutoff LSN, that were written
+    /// to disk, and usually also a DeltaLayer that includes all the WAL records
+    /// between start LSN and the cutoff. (The delta layer is not needed when
+    /// a new relish is created with a single LSN, so that the start and end LSN
+    /// are the same.) If there were page versions newer than 'end_lsn', also
+    /// returns a new in-memory layer containing those page versions. The caller
+    /// replaces this layer with the returned layers in the layer map.
+    ///
+    pub fn freeze(
+        &self,
+        cutoff_lsn: Lsn,
+        // This is needed just to call materialize_page()
+        timeline: &LayeredTimeline,
+    ) -> Result<(Vec<Arc<dyn Layer>>, Option<Arc<InMemoryLayer>>)> {
+        info!(
+            "freezing in memory layer for {} on timeline {} at {}",
+            self.seg, self.timelineid, cutoff_lsn
+        );
+
+        let inner = self.inner.lock().unwrap();
+
+        // Normally, use the cutoff LSN as the end of the frozen layer.
+        // But if the relation was dropped, we know that there are no
+        // more changes coming in for it, and in particular we know that
+        // there are no changes "in flight" for the LSN anymore, so we use
+        // the drop LSN instead. The drop-LSN could be ahead of the
+        // caller-specified LSN!
+        let dropped = inner.drop_lsn.is_some();
+        let end_lsn = if dropped {
+            inner.drop_lsn.unwrap()
+        } else {
+            cutoff_lsn
+        };
+
+        // Divide all the page versions into old and new at the 'end_lsn' cutoff point.
+        let mut before_page_versions;
+        let mut before_segsizes;
+        let mut after_page_versions;
+        let mut after_segsizes;
+        if !dropped {
+            before_segsizes = BTreeMap::new();
+            after_segsizes = BTreeMap::new();
+            for (lsn, size) in inner.segsizes.iter() {
+                if *lsn > end_lsn {
+                    after_segsizes.insert(*lsn, *size);
+                } else {
+                    before_segsizes.insert(*lsn, *size);
+                }
+            }
+
+            before_page_versions = BTreeMap::new();
+            after_page_versions = BTreeMap::new();
+            for ((blknum, lsn), pv) in inner.page_versions.iter() {
+                if *lsn == end_lsn {
+                    // Page versions at the cutoff LSN will be stored in the
+                    // materialized image layer.
+                } else if *lsn > end_lsn {
+                    after_page_versions.insert((*blknum, *lsn), pv.clone());
+                } else {
+                    before_page_versions.insert((*blknum, *lsn), pv.clone());
+                }
+            }
+        } else {
+            before_page_versions = inner.page_versions.clone();
+            before_segsizes = inner.segsizes.clone();
+            after_segsizes = BTreeMap::new();
+            after_page_versions = BTreeMap::new();
+        }
+
+        // we can release the lock now.
+        drop(inner);
+
+        let mut frozen_layers: Vec<Arc<dyn Layer>> = Vec::new();
+
+        if self.start_lsn != end_lsn {
+            // Write the page versions before the cutoff to disk.
+            let delta_layer = DeltaLayer::create(
+                self.conf,
+                self.timelineid,
+                self.tenantid,
+                self.seg,
+                self.start_lsn,
+                end_lsn,
+                dropped,
+                self.img_layer.clone(),
+                before_page_versions,
+                before_segsizes,
+            )?;
+            let delta_layer_rc: Arc<dyn Layer> = Arc::new(delta_layer);
+            frozen_layers.push(delta_layer_rc);
+            trace!(
+                "freeze: created delta layer {} {}-{}",
+                self.seg,
+                self.start_lsn,
+                end_lsn
+            );
+        } else {
+            assert!(before_page_versions.is_empty());
+        }
+
+        let mut new_open_rc = None;
+        if !dropped {
+            // Write a new base image layer at the cutoff point
+            let imgfile = ImageLayer::create_from_src(self.conf, timeline, self, end_lsn)?;
+            let imgfile_rc: Arc<dyn Layer> = Arc::new(imgfile);
+            frozen_layers.push(Arc::clone(&imgfile_rc));
+            trace!("freeze: created image layer {} at {}", self.seg, end_lsn);
+
+            // If there were any page versions newer than the cutoff, initialize a new in-memory
+            // layer to hold them
+            if !after_segsizes.is_empty() || !after_page_versions.is_empty() {
+                let new_open = Self::create_successor_layer(
+                    self.conf,
+                    imgfile_rc,
+                    self.timelineid,
+                    self.tenantid,
+                    end_lsn,
+                    end_lsn,
+                )?;
+                let mut new_inner = new_open.inner.lock().unwrap();
+                new_inner.page_versions.append(&mut after_page_versions);
+                new_inner.segsizes.append(&mut after_segsizes);
+                drop(new_inner);
+                trace!("freeze: created new in-mem layer {} {}-", self.seg, end_lsn);
+
+                new_open_rc = Some(Arc::new(new_open))
+            }
+        }
+
+        Ok((frozen_layers, new_open_rc))
+    }
+
    /// debugging function to print out the contents of the layer
    #[allow(unused)]
    pub fn dump(&self) -> String {
        let mut result = format!(
            "----- inmemory layer for {} {}-> ----\n",
-            self.rel, self.start_lsn
+            self.seg, self.start_lsn
        );

        let inner = self.inner.lock().unwrap();

-        for (k, v) in inner.relsizes.iter() {
+        for (k, v) in inner.segsizes.iter() {
            result += &format!("{}: {}\n", k, v);
        }
        for (k, v) in inner.page_versions.iter() {
--- a/pageserver/src/layered_repository/layer_map.rs
+++ b/pageserver/src/layered_repository/layer_map.rs
@@ -1,127 +1,342 @@
-///
-/// The layer map tracks what layers exist for all the relations in a timeline.
-///
-/// When the timeline is first accessed, the server lists of all snapshot files
-/// in the timelines/<timelineid> directory, and populates this map with
-/// SnapshotLayers corresponding to each file. When new WAL is received,
-/// we create InMemoryLayers to hold the incoming records. Now and then,
-/// in the checkpoint() function, the in-memory layers are frozen, forming
-/// new snapshot layers and corresponding files are written to disk.
-///
+//!
+//! The layer map tracks what layers exist for all the relations in a timeline.
+//!
+//! When the timeline is first accessed, the server lists of all layer files
+//! in the timelines/<timelineid> directory, and populates this map with
+//! ImageLayer and DeltaLayer structs corresponding to each file. When new WAL
+//! is received, we create InMemoryLayers to hold the incoming records. Now and
+//! then, in the checkpoint() function, the in-memory layers are frozen, forming
+//! new image and delta layers and corresponding files are written to disk.
+//!

-use anyhow::Result;
-use log::*;
+use crate::layered_repository::storage_layer::{Layer, SegmentTag};
+use crate::layered_repository::InMemoryLayer;
 use crate::relish::*;
-use crate::layered_repository::storage_layer::Layer;
+use anyhow::Result;
+use lazy_static::lazy_static;
+use log::*;
+use std::cmp::Ordering;
 use std::collections::HashSet;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BinaryHeap, HashMap};
 use std::ops::Bound::Included;
-use zenith_utils::lsn::Lsn;
 use std::sync::Arc;
+use zenith_metrics::{register_int_gauge, IntGauge};
+use zenith_utils::lsn::Lsn;

-/// LayerMap is a BTreeMap keyed by RelishTag and the layer's start LSN.
-/// It provides a couple of convenience functions over a plain BTreeMap
-pub struct LayerMap {
-    pub inner: BTreeMap<(RelishTag, Lsn), Arc<dyn Layer>>,
+lazy_static! {
+    static ref NUM_INMEMORY_LAYERS: IntGauge =
+        register_int_gauge!("pageserver_inmemory_layers", "Number of layers in memory")
+            .expect("failed to define a metric");
+    static ref NUM_ONDISK_LAYERS: IntGauge =
+        register_int_gauge!("pageserver_ondisk_layers", "Number of layers on-disk")
+            .expect("failed to define a metric");
 }

+///
+/// LayerMap tracks what layers exist on a timeline.
+///
+pub struct LayerMap {
+    /// All the layers keyed by segment tag
+    segs: HashMap<SegmentTag, SegEntry>,
+
+    /// All in-memory layers, ordered by 'oldest_pending_lsn' of each layer.
+    /// This allows easy access to the in-memory layer that contains the
+    /// oldest WAL record.
+    open_segs: BinaryHeap<OpenSegEntry>,
+}
+
+///
+/// Per-segment entry in the LayerMap.segs hash map
+///
+/// The last layer that is open for writes is always an InMemoryLayer,
+/// and is kept in a separate field, because there can be only one for
+/// each segment. The older layers, stored on disk, are kept in a
+/// BTreeMap keyed by the layer's start LSN.
+struct SegEntry {
+    pub open: Option<Arc<InMemoryLayer>>,
+    pub historic: BTreeMap<Lsn, Arc<dyn Layer>>,
+}
+
+/// Entry held LayerMap.open_segs, with boilerplate comparison
+/// routines to implement a min-heap ordered by 'oldest_pending_lsn'
+struct OpenSegEntry {
+    pub oldest_pending_lsn: Lsn,
+    pub layer: Arc<InMemoryLayer>,
+}
+impl Ord for OpenSegEntry {
+    fn cmp(&self, other: &Self) -> Ordering {
+        // BinaryHeap is a max-heap, and we want a min-heap. Reverse the ordering here
+        // to get that.
+        other.oldest_pending_lsn.cmp(&self.oldest_pending_lsn)
+    }
+}
+impl PartialOrd for OpenSegEntry {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        // BinaryHeap is a max-heap, and we want a min-heap. Reverse the ordering here
+        // to get that.
+        other
+            .oldest_pending_lsn
+            .partial_cmp(&self.oldest_pending_lsn)
+    }
+}
+impl PartialEq for OpenSegEntry {
+    fn eq(&self, other: &Self) -> bool {
+        self.oldest_pending_lsn.eq(&other.oldest_pending_lsn)
+    }
+}
+impl Eq for OpenSegEntry {}
+
 impl LayerMap {
    ///
-    /// Look up using the given rel tag and LSN. This differs from a plain
-    /// key-value lookup in that if there is any layer that covers the
+    /// Look up a layer using the given segment tag and LSN. This differs from a
+    /// plain key-value lookup in that if there is any layer that covers the
    /// given LSN, or precedes the given LSN, it is returned. In other words,
    /// you don't need to know the exact start LSN of the layer.
    ///
-    pub fn get(&self, tag: RelishTag, lsn: Lsn) -> Option<Arc<dyn Layer>> {
-        let startkey = (tag, Lsn(0));
-        let endkey = (tag, lsn);
+    pub fn get(&self, tag: &SegmentTag, lsn: Lsn) -> Option<Arc<dyn Layer>> {
+        let segentry = self.segs.get(tag)?;

-        if let Some((_k, v)) = self
-            .inner
-            .range((Included(startkey), Included(endkey)))
+        if let Some(open) = &segentry.open {
+            if open.get_start_lsn() <= lsn {
+                let x: Arc<dyn Layer> = Arc::clone(&open) as _;
+                return Some(x);
+            }
+        }
+
+        if let Some((_k, v)) = segentry
+            .historic
+            .range((Included(Lsn(0)), Included(lsn)))
            .next_back()
        {
-            Some(Arc::clone(v))
+            let x: Arc<dyn Layer> = Arc::clone(&v) as _;
+            Some(x)
        } else {
            None
        }
    }

-    pub fn insert(&mut self, layer: Arc<dyn Layer>) {
-        let rel = layer.get_relish_tag();
-        let start_lsn = layer.get_start_lsn();
+    ///
+    /// Get the open layer for given segment for writing. Or None if no open
+    /// layer exists.
+    ///
+    pub fn get_open(&self, tag: &SegmentTag) -> Option<Arc<InMemoryLayer>> {
+        let segentry = self.segs.get(tag)?;

-        self.inner.insert((rel, start_lsn), Arc::clone(&layer));
+        if let Some(open) = &segentry.open {
+            Some(Arc::clone(open))
+        } else {
+            None
+        }
    }

-    pub fn remove(&mut self, layer: &dyn Layer) {
-        let rel = layer.get_relish_tag();
+    ///
+    /// Insert an open in-memory layer
+    ///
+    pub fn insert_open(&mut self, layer: Arc<InMemoryLayer>) {
+        let tag = layer.get_seg_tag();
+
+        if let Some(segentry) = self.segs.get_mut(&tag) {
+            if let Some(_old) = &segentry.open {
+                // FIXME: shouldn't exist, but check
+            }
+            segentry.open = Some(Arc::clone(&layer));
+        } else {
+            let segentry = SegEntry {
+                open: Some(Arc::clone(&layer)),
+                historic: BTreeMap::new(),
+            };
+            self.segs.insert(tag, segentry);
+        }
+
+        let opensegentry = OpenSegEntry {
+            oldest_pending_lsn: layer.get_oldest_pending_lsn(),
+            layer: layer,
+        };
+        self.open_segs.push(opensegentry);
+
+        NUM_INMEMORY_LAYERS.inc();
+    }
+
+    /// Remove the oldest in-memory layer
+    pub fn pop_oldest_open(&mut self) {
+        let opensegentry = self.open_segs.pop().unwrap();
+        let segtag = opensegentry.layer.get_seg_tag();
+
+        let mut segentry = self.segs.get_mut(&segtag).unwrap();
+        segentry.open = None;
+        NUM_INMEMORY_LAYERS.dec();
+    }
+
+    ///
+    /// Insert an on-disk layer
+    ///
+    pub fn insert_historic(&mut self, layer: Arc<dyn Layer>) {
+        let tag = layer.get_seg_tag();
        let start_lsn = layer.get_start_lsn();

-        self.inner.remove(&(rel, start_lsn));
+        if let Some(segentry) = self.segs.get_mut(&tag) {
+            segentry.historic.insert(start_lsn, layer);
+        } else {
+            let mut historic = BTreeMap::new();
+            historic.insert(start_lsn, layer);
+
+            let segentry = SegEntry {
+                open: None,
+                historic,
+            };
+            self.segs.insert(tag, segentry);
+        }
+        NUM_ONDISK_LAYERS.inc();
    }
-    
-    pub fn list_rels(&self,
-        spcnode: u32,
-        dbnode: u32,
-    ) -> Result<HashSet<RelTag>> {
+
+    ///
+    /// Remove an on-disk layer from the map.
+    ///
+    /// This should be called when the corresponding file on disk has been deleted.
+    ///
+    pub fn remove_historic(&mut self, layer: &dyn Layer) {
+        let tag = layer.get_seg_tag();
+        let start_lsn = layer.get_start_lsn();
+
+        if let Some(segentry) = self.segs.get_mut(&tag) {
+            segentry.historic.remove(&start_lsn);
+        }
+        NUM_ONDISK_LAYERS.dec();
+    }
+
+    // List relations that exist at the lsn
+    pub fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashSet<RelTag>> {
        let mut rels: HashSet<RelTag> = HashSet::new();

-        // Scan the timeline directory to get all rels in this timeline.
-        for ((rel, _lsn), _l) in self.inner.iter() {
-            if let RelishTag::Relation(reltag) = rel {
-                // FIXME: skip if it was dropped before the requested LSN. But there is no
-                // LSN argument
-
+        for (seg, segentry) in self.segs.iter() {
+            if let RelishTag::Relation(reltag) = seg.rel {
                if (spcnode == 0 || reltag.spcnode == spcnode)
                    && (dbnode == 0 || reltag.dbnode == dbnode)
                {
-                    rels.insert(*reltag);
+                    // Add only if it exists at the requested LSN.
+                    if let Some(open) = &segentry.open {
+                        if open.get_end_lsn() > lsn {
+                            rels.insert(reltag);
+                        }
+                    } else if let Some((_k, _v)) = segentry
+                        .historic
+                        .range((Included(Lsn(0)), Included(lsn)))
+                        .next_back()
+                    {
+                        rels.insert(reltag);
+                    }
                }
            }
        }
        Ok(rels)
    }

-    pub fn list_nonrels(&self, _lsn: Lsn) -> Result<HashSet<RelishTag>> {
+    // List non-relation relishes that exist at the lsn
+    pub fn list_nonrels(&self, lsn: Lsn) -> Result<HashSet<RelishTag>> {
        let mut rels: HashSet<RelishTag> = HashSet::new();

        // Scan the timeline directory to get all rels in this timeline.
-        for ((rel, _lsn), _l) in self.inner.iter() {
-            // FIXME: skip if it was dropped before the requested LSN.
-
-            if let RelishTag::Relation(_) = rel {
+        for (seg, segentry) in self.segs.iter() {
+            if let RelishTag::Relation(_) = seg.rel {
            } else {
-                rels.insert(*rel);
+                // Add only if it exists at the requested LSN.
+                if let Some(open) = &segentry.open {
+                    if open.get_end_lsn() > lsn {
+                        rels.insert(seg.rel);
+                    }
+                } else if let Some((_k, _v)) = segentry
+                    .historic
+                    .range((Included(Lsn(0)), Included(lsn)))
+                    .next_back()
+                {
+                    rels.insert(seg.rel);
+                }
            }
        }
        Ok(rels)
    }

-    /// Is there a newer layer for given relation?
-    pub fn newer_layer_exists(&self, rel: RelishTag, lsn: Lsn) -> bool {
-        let startkey = (rel, lsn);
-        let endkey = (rel, Lsn(u64::MAX));
-
-        for ((_rel, newer_lsn), layer) in self
-            .inner
-            .range((Included(startkey), Included(endkey)))
-        {
-            if layer.get_end_lsn() > lsn {
-                info!("found later layer for rel {}, {} {}-{}", rel, lsn, newer_lsn, layer.get_end_lsn());
-                return true;
-            } else {
-                info!("found singleton layer for rel {}, {} {}", rel, lsn, newer_lsn);
-                continue;
+    /// Is there a newer image layer for given segment?
+    ///
+    /// This is used for garbage collection, to determine if an old layer can
+    /// be deleted. We ignore in-memory layers because they are not durable
+    /// on disk, and delta layers because they depend on an older layer.
+    pub fn newer_image_layer_exists(&self, seg: SegmentTag, lsn: Lsn) -> bool {
+        if let Some(segentry) = self.segs.get(&seg) {
+            for (newer_lsn, layer) in segentry
+                .historic
+                .range((Included(lsn), Included(Lsn(u64::MAX))))
+            {
+                // Ignore delta layers.
+                if layer.is_incremental() {
+                    continue;
+                }
+                if layer.get_end_lsn() > lsn {
+                    trace!(
+                        "found later layer for {}, {} {}-{}",
+                        seg,
+                        lsn,
+                        newer_lsn,
+                        layer.get_end_lsn()
+                    );
+                    return true;
+                } else {
+                    trace!("found singleton layer for {}, {} {}", seg, lsn, newer_lsn);
+                    continue;
+                }
            }
        }
-        info!("no later layer found for rel {}, {}", rel, lsn);
+        trace!("no later layer found for {}, {}", seg, lsn);
        false
    }
+
+    /// Return the oldest in-memory layer.
+    pub fn peek_oldest_open(&self) -> Option<Arc<InMemoryLayer>> {
+        if let Some(opensegentry) = self.open_segs.peek() {
+            Some(Arc::clone(&opensegentry.layer))
+        } else {
+            None
+        }
+    }
+
+    pub fn iter_historic_layers(&self) -> HistoricLayerIter {
+        HistoricLayerIter {
+            segiter: self.segs.iter(),
+            iter: None,
+        }
+    }
 }

 impl Default for LayerMap {
    fn default() -> Self {
-        LayerMap { inner: BTreeMap::new() }
+        LayerMap {
+            segs: HashMap::new(),
+            open_segs: BinaryHeap::new(),
+        }
+    }
+}
+
+pub struct HistoricLayerIter<'a> {
+    segiter: std::collections::hash_map::Iter<'a, SegmentTag, SegEntry>,
+    iter: Option<std::collections::btree_map::Iter<'a, Lsn, Arc<dyn Layer>>>,
+}
+
+impl<'a> Iterator for HistoricLayerIter<'a> {
+    type Item = Arc<dyn Layer>;
+
+    fn next(&mut self) -> std::option::Option<<Self as std::iter::Iterator>::Item> {
+        loop {
+            if let Some(x) = &mut self.iter {
+                if let Some(x) = x.next() {
+                    return Some(Arc::clone(&*x.1));
+                }
+            }
+            if let Some(seg) = self.segiter.next() {
+                self.iter = Some(seg.1.historic.iter());
+                continue;
+            } else {
+                return None;
+            }
+        }
    }
 }
--- a/pageserver/src/layered_repository/snapshot_layer.rs
+++ b/pageserver/src/layered_repository/snapshot_layer.rs
@@ -1,632 +0,0 @@
-//!
-//! A SnapshotLayer represents one snapshot file on disk. One file holds all page versions
-//! and size information of one relation, in a range of LSN.
-//! The name "snapshot file" is a bit of a misnomer because a snapshot file doesn't
-//! contain a snapshot at a specific LSN, but rather all the page versions in a range
-//! of LSNs.
-//!
-//! Currently, a snapshot file contains full information needed to reconstruct any
-//! page version in the LSN range, without consulting any other snapshot files. When
-//! a new snapshot file is created for writing, the full contents of relation are
-//! materialized as it is at the beginning of the LSN range. That can be very expensive,
-//! we should find a way to store differential files. But this keeps the read-side
-//! of things simple. You can find the correct snapshot file based on RelishTag and
-//! timeline+LSN, and once you've located it, you have all the data you need to in that
-//! file.
-//!
-//! When a snapshot file needs to be accessed, we slurp the whole file into memory, into
-//! a SnapshotLayer struct.
-//!
-//! On disk, the snapshot files are stored in .zenith/timelines/<timelineid> directory.
-//! Currently, there are no subdirectories, and each snapshot file is named like this:
-//!
-//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<start LSN>_<end LSN>
-//!
-//! For example:
-//!
-//!    1663_13990_2609_0_000000000169C348_000000000169C349
-//!
-//! If a relation is dropped, we add a '_DROPPED' to the end of the filename to indicate that.
-//! So the above example would become:
-//!
-//!    1663_13990_2609_0_000000000169C348_000000000169C349_DROPPED
-//!
-//! The end LSN indicates when it was dropped in that case, we don't store it in the
-//! file contents in any way.
-//!
-//! A snapshot file is constructed using the 'bookfile' crate. Each file consists of two
-//! parts: the page versions and the relation sizes. They are stored as separate chapters.
-//!
-use crate::layered_repository::storage_layer::Layer;
-use crate::layered_repository::storage_layer::PageVersion;
-use crate::layered_repository::storage_layer::ZERO_PAGE;
-use crate::relish::*;
-use crate::repository::WALRecord;
-use crate::walredo::WalRedoManager;
-use crate::PageServerConf;
-use crate::{ZTimelineId, ZTenantId};
-use anyhow::{anyhow, bail, Result};
-use bytes::Bytes;
-use log::*;
-use std::collections::BTreeMap;
-use std::fmt;
-use std::fs;
-use std::fs::File;
-use std::io::Write;
-use std::ops::Bound::Included;
-use std::path::PathBuf;
-use std::sync::{Arc, Mutex, MutexGuard};
-
-use bookfile::{Book, BookWriter};
-
-use zenith_utils::bin_ser::BeSer;
-use zenith_utils::lsn::Lsn;
-
-// Magic constant to identify a Zenith snapshot file
-static SNAPSHOT_FILE_MAGIC: u32 = 0x5A616E01;
-
-static PAGE_VERSIONS_CHAPTER: u64 = 1;
-static REL_SIZES_CHAPTER: u64 = 2;
-
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
-struct SnapshotFileName {
-    rel: RelishTag,
-    start_lsn: Lsn,
-    end_lsn: Lsn,
-    dropped: bool,
-}
-
-impl SnapshotFileName {
-    fn from_str(fname: &str) -> Option<Self> {
-        // Split the filename into parts
-        //
-        //    <spcnode>_<dbnode>_<relnode>_<forknum>_<start LSN>_<end LSN>
-        //
-        // or if it was dropped:
-        //
-        //    <spcnode>_<dbnode>_<relnode>_<forknum>_<start LSN>_<end LSN>_DROPPED
-        //
-        let rel;
-        let mut parts;
-        if let Some(rest) = fname.strip_prefix("rel_") {
-            parts = rest.split('_');
-            rel = RelishTag::Relation(RelTag {
-                spcnode: parts.next()?.parse::<u32>().ok()?,
-                dbnode: parts.next()?.parse::<u32>().ok()?,
-                relnode: parts.next()?.parse::<u32>().ok()?,
-                forknum: parts.next()?.parse::<u8>().ok()?,
-            });
-        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::Clog,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::MultiXactMembers,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::MultiXactOffsets,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
-            parts = rest.split('_');
-            rel = RelishTag::FileNodeMap {
-                spcnode: parts.next()?.parse::<u32>().ok()?,
-                dbnode: parts.next()?.parse::<u32>().ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
-            parts = rest.split('_');
-            rel = RelishTag::TwoPhase {
-                xid: parts.next()?.parse::<u32>().ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
-            parts = rest.split('_');
-            rel = RelishTag::Checkpoint;
-        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
-            parts = rest.split('_');
-            rel = RelishTag::ControlFile;
-        } else {
-            return None;
-        }
-
-        let start_lsn = Lsn::from_hex(parts.next()?).ok()?;
-        let end_lsn = Lsn::from_hex(parts.next()?).ok()?;
-
-        let mut dropped = false;
-        if let Some(suffix) = parts.next() {
-            if suffix == "DROPPED" {
-                dropped = true;
-            } else {
-                warn!("unrecognized filename in timeline dir: {}", fname);
-                return None;
-            }
-        }
-        if parts.next().is_some() {
-            warn!("unrecognized filename in timeline dir: {}", fname);
-            return None;
-        }
-
-        Some(SnapshotFileName {
-            rel,
-            start_lsn,
-            end_lsn,
-            dropped,
-        })
-    }
-
-    fn to_string(&self) -> String {
-        let basename = match self.rel {
-            RelishTag::Relation(reltag) => format!(
-                "rel_{}_{}_{}_{}",
-                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
-            ),
-            RelishTag::Slru {
-                slru: SlruKind::Clog,
-                segno,
-            } => format!("pg_xact_{:04X}", segno),
-            RelishTag::Slru {
-                slru: SlruKind::MultiXactMembers,
-                segno,
-            } => format!("pg_multixact_members_{:04X}", segno),
-            RelishTag::Slru {
-                slru: SlruKind::MultiXactOffsets,
-                segno,
-            } => format!("pg_multixact_offsets_{:04X}", segno),
-            RelishTag::FileNodeMap { spcnode, dbnode } => {
-                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
-            }
-            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
-            RelishTag::Checkpoint => format!("pg_control_checkpoint"),
-            RelishTag::ControlFile => format!("pg_control"),
-        };
-
-        format!(
-            "{}_{:016X}_{:016X}{}",
-            basename,
-            u64::from(self.start_lsn),
-            u64::from(self.end_lsn),
-            if self.dropped { "_DROPPED" } else { "" }
-        )
-    }
-}
-
-impl fmt::Display for SnapshotFileName {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}", self.to_string())
-    }
-}
-
-///
-/// SnapshotLayer is the in-memory data structure associated with an on-disk snapshot file.
-/// We hold a SnapshotLayer in memory for each file, in the LayerMap. If a layer is in
-/// "loaded" state, we have a copy of the file in memory, in 'inner'. Otherwise the struct
-/// is just a placeholder for a file that exists in memory, and it needs to be loaded
-/// before using it in queries.
-///
-pub struct SnapshotLayer {
-    conf: &'static PageServerConf,
-    pub tenantid: ZTenantId,
-    pub timelineid: ZTimelineId,
-    pub rel: RelishTag,
-
-    //
-    // This entry contains all the changes from 'start_lsn' to 'end_lsn'. The
-    // start is inclusive, and end is exclusive.
-    pub start_lsn: Lsn,
-    pub end_lsn: Lsn,
-
-    dropped: bool,
-
-    inner: Mutex<SnapshotLayerInner>
-}
-
-pub struct SnapshotLayerInner {
-    // If false, the 'page_versions' and 'relsizes' have not been loaded into memory
-    // yet.
-    loaded: bool,
-    
-    ///
-    /// All versions of all pages in the file are are kept here.
-    /// Indexed by block number and LSN.
-    ///
-    page_versions: BTreeMap<(u32, Lsn), PageVersion>,
-
-    ///
-    /// `relsizes` tracks the size of the relation at different points in time.
-    ///
-    relsizes: BTreeMap<Lsn, u32>,
-}
-
-impl Layer for SnapshotLayer {
-    fn is_frozen(&self) -> bool {
-        return true;
-    }
-
-    fn get_timeline_id(&self) -> ZTimelineId {
-        return self.timelineid;
-    }
-
-    fn get_relish_tag(&self) -> RelishTag {
-        return self.rel;
-    }
-
-    fn is_dropped(&self) -> bool {
-        return self.dropped;
-    }
-
-    fn get_start_lsn(&self) -> Lsn {
-        return self.start_lsn;
-    }
-
-    fn get_end_lsn(&self) -> Lsn {
-        return self.end_lsn;
-    }
-
-    /// Look up given page in the cache.
-    fn get_page_at_lsn(
-        &self,
-        walredo_mgr: &dyn WalRedoManager,
-        blknum: u32,
-        lsn: Lsn,
-    ) -> Result<Bytes> {
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let mut records: Vec<WALRecord> = Vec::new();
-        let mut page_img: Option<Bytes> = None;
-        let mut need_base_image_lsn: Option<Lsn> = Some(lsn);
-        {
-            let inner = self.load()?;
-            let minkey = (blknum, Lsn(0));
-            let maxkey = (blknum, lsn);
-            let mut iter = inner.page_versions.range((Included(&minkey), Included(&maxkey)));
-            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
-                if let Some(img) = &entry.page_image {
-                    page_img = Some(img.clone());
-                    need_base_image_lsn = None;
-                    break;
-                } else if let Some(rec) = &entry.record {
-                    records.push(rec.clone());
-                    if rec.will_init {
-                        // This WAL record initializes the page, so no need to go further back
-                        need_base_image_lsn = None;
-                        break;
-                    } else {
-                        need_base_image_lsn = Some(*entry_lsn);
-                    }
-                } else {
-                    // No base image, and no WAL record. Huh?
-                    bail!("no page image or WAL record for requested page");
-                }
-            }
-
-            // release lock on 'inner'
-        }
-        records.reverse();
-
-        // If we needed a base image to apply the WAL records against, we should have found it in memory.
-        if let Some(lsn) = need_base_image_lsn {
-            if records.is_empty() {
-                // no records, and no base image. This can happen if PostgreSQL extends a relation
-                // but never writes the page.
-                //
-                // Would be nice to detect that situation better.
-                warn!("Page {} blk {} at {} not found", self.rel, blknum, lsn);
-                return Ok(ZERO_PAGE.clone());
-            }
-            bail!(
-                "No base image found for page {} blk {} at {}/{}",
-                self.rel,
-                blknum,
-                self.timelineid,
-                lsn
-            );
-        }
-
-        // If we have a page image, and no WAL, we're all set
-        if records.is_empty() {
-            if let Some(img) = page_img {
-                trace!(
-                    "found page image for blk {} in {} at {}/{}, no WAL redo required",
-                    blknum,
-                    self.rel,
-                    self.timelineid,
-                    lsn
-                );
-                Ok(img)
-            } else {
-                // FIXME: this ought to be an error?
-                warn!("Page {} blk {} at {} not found", self.rel, blknum, lsn);
-                Ok(ZERO_PAGE.clone())
-            }
-        } else {
-            // We need to do WAL redo.
-            //
-            // If we don't have a base image, then the oldest WAL record better initialize
-            // the page
-            if page_img.is_none() && !records.first().unwrap().will_init {
-                // FIXME: this ought to be an error?
-                warn!(
-                    "Base image for page {} blk {} at {} not found, but got {} WAL records",
-                    self.rel,
-                    blknum,
-                    lsn,
-                    records.len()
-                );
-                Ok(ZERO_PAGE.clone())
-            } else {
-                if page_img.is_some() {
-                    trace!("found {} WAL records and a base image for blk {} in {} at {}/{}, performing WAL redo", records.len(), blknum, self.rel, self.timelineid, lsn);
-                } else {
-                    trace!("found {} WAL records that will init the page for blk {} in {} at {}/{}, performing WAL redo", records.len(), blknum, self.rel, self.timelineid, lsn);
-                }
-                let img = walredo_mgr.request_redo(self.rel, blknum, lsn, page_img, records)?;
-
-                // FIXME: Should we memoize the page image in memory, so that
-                // we wouldn't need to reconstruct it again, if it's requested again?
-                //self.put_page_image(blknum, lsn, img.clone())?;
-
-                Ok(img)
-            }
-        }
-    }
-
-    /// Get size of the relation at given LSN
-    fn get_rel_size(&self, lsn: Lsn) -> Result<u32> {
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let inner = self.load()?;
-        let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn)));
-
-        if let Some((_entry_lsn, entry)) = iter.next_back() {
-            let result = *entry;
-            drop(inner);
-            trace!("get_relsize: {} at {} -> {}", self.rel, lsn, result);
-            Ok(result)
-        } else {
-            error!("No size found for {} at {} in snapshot layer {} {} {}", self.rel, lsn, self.rel, self.start_lsn, self.end_lsn);
-            bail!("No size found for {} at {} in snapshot layer", self.rel, lsn);
-        }
-    }
-
-    /// Does this relation exist at given LSN?
-    fn get_rel_exists(&self, lsn: Lsn) -> Result<bool> {
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let inner = self.load()?;
-
-        let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn)));
-
-        let result = if let Some((_entry_lsn, _entry)) = iter.next_back() {
-            true
-        } else {
-            false
-        };
-        Ok(result)
-    }
-
-    // Unsupported write operations
-    fn put_page_version(&self, blknum: u32, lsn: Lsn, _pv: PageVersion) -> Result<()> {
-        panic!(
-            "cannot modify historical snapshot layer, rel {} blk {} at {}/{}, {}-{}",
-            self.rel, blknum, self.timelineid, lsn, self.start_lsn, self.end_lsn
-        );
-    }
-    fn put_truncation(&self, _lsn: Lsn, _relsize: u32) -> anyhow::Result<()> {
-        bail!("cannot modify historical snapshot layer");
-    }
-
-    fn put_unlink(&self, _lsn: Lsn) -> anyhow::Result<()> {
-        bail!("cannot modify historical snapshot layer");
-    }
-
-    fn freeze(&self, _end_lsn: Lsn, _walredo_mgr: &dyn WalRedoManager) -> Result<Vec<Arc<dyn Layer>>> {
-        bail!("cannot freeze historical snapshot layer");
-    }
-
-    fn delete(&self) -> Result<()> {
-        // delete underlying file
-        fs::remove_file(self.path())?;
-        Ok(())
-    }
-
-    fn unload(&self) -> Result<()> {
-        let mut inner = self.inner.lock().unwrap();
-        inner.page_versions = BTreeMap::new();
-        inner.relsizes = BTreeMap::new();
-        inner.loaded = false;
-        Ok(())
-    }
-}
-
-impl SnapshotLayer {
-    fn path(&self) -> PathBuf {
-        Self::path_for(
-            self.conf,
-            self.timelineid,
-            self.tenantid,
-            &SnapshotFileName {
-                rel: self.rel,
-                start_lsn: self.start_lsn,
-                end_lsn: self.end_lsn,
-                dropped: self.dropped,
-            },
-        )
-    }
-
-    fn path_for(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        fname: &SnapshotFileName,
-    ) -> PathBuf {
-        conf.timeline_path(&timelineid, &tenantid).join(fname.to_string())
-    }
-
-    /// Create a new snapshot file, using the given btreemaps containing the page versions and
-    /// relsizes.
-    ///
-    /// This is used to write the in-memory layer to disk. The in-memory layer uses the same
-    /// data structure with two btreemaps as we do, so passing the btreemaps is currently
-    /// expedient.
-    pub fn create(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        rel: RelishTag,
-        start_lsn: Lsn,
-        end_lsn: Lsn,
-        dropped: bool,
-        page_versions: BTreeMap<(u32, Lsn), PageVersion>,
-        relsizes: BTreeMap<Lsn, u32>,
-    ) -> Result<SnapshotLayer> {
-        let snapfile = SnapshotLayer {
-            conf: conf,
-            timelineid: timelineid,
-            tenantid: tenantid,
-            rel: rel,
-            start_lsn: start_lsn,
-            end_lsn,
-            dropped,
-            inner: Mutex::new(SnapshotLayerInner {
-                loaded: true,
-                page_versions: page_versions,
-                relsizes: relsizes,
-            }),
-        };
-        let inner = snapfile.inner.lock().unwrap();
-
-        // Write the in-memory btreemaps into files
-        let path = snapfile.path();
-
-        // Note: This overwrites any existing file. There shouldn't be any.
-        // FIXME: throw an error instead?
-
-        let file = File::create(&path)?;
-        let book = BookWriter::new(file, SNAPSHOT_FILE_MAGIC)?;
-
-        // Write out page versions
-        let mut chapter = book.new_chapter(PAGE_VERSIONS_CHAPTER);
-        let buf = BTreeMap::ser(&inner.page_versions)?;
-        chapter.write_all(&buf)?;
-        let book = chapter.close()?;
-
-        // and relsizes to separate chapter
-        let mut chapter = book.new_chapter(REL_SIZES_CHAPTER);
-        let buf = BTreeMap::ser(&inner.relsizes)?;
-        chapter.write_all(&buf)?;
-        let book = chapter.close()?;
-
-        book.close()?;
-
-        trace!("saved {}", &path.display());
-
-        drop(inner);
-
-        Ok(snapfile)
-    }
-
-    fn load(&self) -> Result<MutexGuard<SnapshotLayerInner>> {
-
-        // quick exit if already loaded
-        let mut inner = self.inner.lock().unwrap();
-
-        if inner.loaded {
-            return Ok(inner);
-        }
-
-        let path = Self::path_for(
-            self.conf,
-            self.timelineid,
-            self.tenantid,
-            &SnapshotFileName {
-                rel: self.rel,
-                start_lsn: self.start_lsn,
-                end_lsn: self.end_lsn,
-                dropped: self.dropped,
-            },
-        );
-
-        let file = File::open(&path)?;
-        let mut book = Book::new(file)?;
-
-        let chapter_index = book
-            .find_chapter(PAGE_VERSIONS_CHAPTER)
-            .ok_or_else(|| anyhow!("could not find page versions chapter in {}", path.display()))?;
-        let chapter = book.read_chapter(chapter_index)?;
-        let page_versions = BTreeMap::des(&chapter)?;
-
-        let chapter_index = book
-            .find_chapter(REL_SIZES_CHAPTER)
-            .ok_or_else(|| anyhow!("could not find relsizes chapter in {}", path.display()))?;
-        let chapter = book.read_chapter(chapter_index)?;
-        let relsizes = BTreeMap::des(&chapter)?;
-
-        debug!("loaded from {}", &path.display());
-
-        *inner = SnapshotLayerInner {
-            loaded: true,
-            page_versions,
-            relsizes,
-        };
-
-        Ok(inner)
-    }
-
-    // TODO: returning an Iterator would be more idiomatic
-    pub fn list_snapshot_files(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-    ) -> Result<Vec<Arc<dyn Layer>>> {
-        let path = conf.timeline_path(&timelineid, &tenantid);
-
-        let mut snapfiles: Vec<Arc<dyn Layer>> = Vec::new();
-        for direntry in fs::read_dir(path)? {
-            let fname = direntry?.file_name();
-            let fname = fname.to_str().unwrap();
-
-            if let Some(snapfilename) = SnapshotFileName::from_str(fname) {
-
-                let snapfile = SnapshotLayer {
-                    conf,
-                    timelineid,
-                    tenantid,
-                    rel: snapfilename.rel,
-                    start_lsn: snapfilename.start_lsn,
-                    end_lsn: snapfilename.end_lsn,
-                    dropped: snapfilename.dropped,
-                    inner: Mutex::new(SnapshotLayerInner {
-                        loaded: false,
-                        page_versions: BTreeMap::new(),
-                        relsizes: BTreeMap::new(),
-                    }),
-                };
-
-                snapfiles.push(Arc::new(snapfile));
-            }
-        }
-        return Ok(snapfiles);
-    }
-
-    /// debugging function to print out the contents of the layer
-    #[allow(unused)]
-    pub fn dump(&self) -> String {
-        let mut result = format!(
-            "----- snapshot layer for {} {}-{} ----\n",
-            self.rel, self.start_lsn, self.end_lsn
-        );
-
-        let inner = self.inner.lock().unwrap();
-        for (k, v) in inner.relsizes.iter() {
-            result += &format!("{}: {}\n", k, v);
-        }
-        //for (k, v) in inner.page_versions.iter() {
-        //    result += &format!("blk {} at {}: {}/{}\n", k.0, k.1, v.page_image.is_some(), v.record.is_some());
-        //}
-
-        result
-    }
-}
--- a/pageserver/src/layered_repository/storage_layer.rs
+++ b/pageserver/src/layered_repository/storage_layer.rs
@@ -1,15 +1,49 @@
+//!
+//! Common traits and structs for layers
+//!
+
 use crate::relish::RelishTag;
 use crate::repository::WALRecord;
-use crate::walredo::WalRedoManager;
 use crate::ZTimelineId;
 use anyhow::Result;
 use bytes::Bytes;
 use serde::{Deserialize, Serialize};
-use std::sync::Arc;
+use std::fmt;
+use std::path::PathBuf;

 use zenith_utils::lsn::Lsn;

-pub static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
+// Size of one segment in pages (10 MB)
+pub const RELISH_SEG_SIZE: u32 = 10 * 1024 * 1024 / 8192;
+
+///
+/// Each relish stored in the repository is divided into fixed-sized "segments",
+/// with 10 MB of key-space, or 1280 8k pages each.
+///
+#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy)]
+pub struct SegmentTag {
+    pub rel: RelishTag,
+    pub segno: u32,
+}
+
+impl fmt::Display for SegmentTag {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}.{}", self.rel, self.segno)
+    }
+}
+
+impl SegmentTag {
+    pub const fn from_blknum(rel: RelishTag, blknum: u32) -> SegmentTag {
+        SegmentTag {
+            rel,
+            segno: blknum / RELISH_SEG_SIZE,
+        }
+    }
+
+    pub fn blknum_in_seg(&self, blknum: u32) -> bool {
+        blknum / RELISH_SEG_SIZE == self.segno
+    }
+}

 ///
 /// Represents a version of a page at a specific LSN. The LSN is the key of the
@@ -18,6 +52,16 @@ pub static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
 /// A page version can be stored as a full page image, or as WAL record that needs
 /// to be applied over the previous page version to reconstruct this version.
 ///
+/// It's also possible to have both a WAL record and a page image in the same
+/// PageVersion. That happens if page version is originally stored as a WAL record
+/// but it is later reconstructed by a GetPage@LSN request by performing WAL
+/// redo. The get_page_at_lsn() code will store the reconstructed pag image next to
+/// the WAL record in that case. TODO: That's pretty accidental, not the result
+/// of any grand design. If we want to keep reconstructed page versions around, we
+/// probably should have a separate buffer cache so that we could control the
+/// replacement policy globally. Or if we keep a reconstructed page image, we
+/// could throw away the WAL record.
+///
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct PageVersion {
    /// an 8kb page image
@@ -26,60 +70,79 @@ pub struct PageVersion {
    pub record: Option<WALRecord>,
 }

-pub trait Layer: Send + Sync {
-    fn is_frozen(&self) -> bool;
+///
+/// Data needed to reconstruct a page version
+///
+/// 'page_img' is the old base image of the page to start the WAL replay with.
+/// It can be None, if the first WAL record initializes the page (will_init)
+/// 'records' contains the records to apply over the base image.
+///
+pub struct PageReconstructData {
+    pub records: Vec<WALRecord>,
+    pub page_img: Option<Bytes>,
+}

+///
+/// A Layer holds all page versions for one segment of a relish, in a range of LSNs.
+/// There are two kinds of layers, in-memory and snapshot layers. In-memory
+/// layers are used to ingest incoming WAL, and provide fast access
+/// to the recent page versions. Snaphot layers are stored on disk, and
+/// are immutable. This trait presents the common functionality of
+/// in-memory and snapshot layers.
+///
+/// Each layer contains a full snapshot of the segment at the start
+/// LSN. In addition to that, it contains WAL (or more page images)
+/// needed to recontruct any page version up to the end LSN.
+///
+pub trait Layer: Send + Sync {
+    // These functions identify the relish segment and the LSN range
+    // that this Layer holds.
    fn get_timeline_id(&self) -> ZTimelineId;
-    fn get_relish_tag(&self) -> RelishTag;
+    fn get_seg_tag(&self) -> SegmentTag;
    fn get_start_lsn(&self) -> Lsn;
    fn get_end_lsn(&self) -> Lsn;
    fn is_dropped(&self) -> bool;

-    fn get_page_at_lsn(
+    /// Filename used to store this layer on disk. (Even in-memory layers
+    /// implement this, to print a handy unique identifier for the layer for
+    /// log messages, even though they're never not on disk.)
+    fn filename(&self) -> PathBuf;
+
+    ///
+    /// Return data needed to reconstruct given page at LSN.
+    ///
+    /// It is up to the caller to collect more data from previous layer and
+    /// perform WAL redo, if necessary.
+    ///
+    /// If returns Some, the returned data is not complete. The caller needs
+    /// to continue with the returned 'lsn'.
+    ///
+    /// Note that the 'blknum' is the offset of the page from the beginning
+    /// of the *relish*, not the beginning of the segment. The requested
+    /// 'blknum' must be covered by this segment.
+    fn get_page_reconstruct_data(
        &self,
-        walredo_mgr: &dyn WalRedoManager,
        blknum: u32,
        lsn: Lsn,
-    ) -> Result<Bytes>;
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<Option<Lsn>>;

-    fn get_rel_size(&self, lsn: Lsn) -> Result<u32>;
+    /// Return size of the segment at given LSN. (Only for blocky relations.)
+    fn get_seg_size(&self, lsn: Lsn) -> Result<u32>;

-    fn get_rel_exists(&self, lsn: Lsn) -> Result<bool>;
+    /// Does the segment exist at given LSN? Or was it dropped before it.
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool>;

-    fn put_page_version(&self, blknum: u32, lsn: Lsn, pv: PageVersion) -> Result<()>;
-
-    fn put_truncation(&self, lsn: Lsn, relsize: u32) -> anyhow::Result<()>;
-
-    fn put_unlink(&self, lsn: Lsn) -> anyhow::Result<()>;
-
-    /// Remember new page version, as a WAL record over previous version
-    fn put_wal_record(&self, blknum: u32, rec: WALRecord) -> Result<()> {
-        // FIXME: If this is the first version of this page, reconstruct the image
-        self.put_page_version(
-            blknum,
-            rec.lsn,
-            PageVersion {
-                page_image: None,
-                record: Some(rec),
-            },
-        )
-    }
-
-    /// Remember new page version, as a full page image
-    fn put_page_image(&self, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()> {
-        self.put_page_version(
-            blknum,
-            lsn,
-            PageVersion {
-                page_image: Some(img),
-                record: None,
-            },
-        )
-    }
-
-    fn freeze(&self, end_lsn: Lsn, walredo_mgr: &dyn WalRedoManager) -> Result<Vec<Arc<dyn Layer>>>;
-
-    fn delete(&self) -> Result<()>;
+    /// Does this layer only contain some data for the segment (incremental),
+    /// or does it contain a version of every page? This is important to know
+    /// for garbage collecting old layers: an incremental layer depends on
+    /// the previous non-incremental layer.
+    fn is_incremental(&self) -> bool;

+    /// Release memory used by this layer. There is no corresponding 'load'
+    /// function, that's done implicitly when you call one of the get-functions.
    fn unload(&self) -> Result<()>;
+
+    /// Permanently remove this layer from disk.
+    fn delete(&self) -> Result<()>;
 }
--- a/pageserver/src/lib.rs
+++ b/pageserver/src/lib.rs
@@ -1,33 +1,40 @@
-use hex::FromHex;
-use rand::Rng;
-use serde::{Deserialize, Serialize};
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::{ZTenantId, ZTimelineId};

-use std::fmt;
 use std::path::PathBuf;
-use std::str::FromStr;
 use std::time::Duration;

+use lazy_static::lazy_static;
+use zenith_metrics::{register_int_gauge_vec, IntGaugeVec};
+
 pub mod basebackup;
 pub mod branches;
+pub mod http;
 pub mod layered_repository;
 pub mod logger;
-pub mod object_key;
-pub mod object_repository;
-pub mod object_store;
-pub mod page_cache;
 pub mod page_service;
 pub mod relish;
 pub mod repository;
 pub mod restore_local_repo;
-pub mod rocksdb_storage;
+pub mod tenant_mgr;
 pub mod waldecoder;
 pub mod walreceiver;
 pub mod walredo;

+lazy_static! {
+    static ref LIVE_CONNECTIONS_COUNT: IntGaugeVec = register_int_gauge_vec!(
+        "pageserver_live_connections_count",
+        "Number of live network connections",
+        &["pageserver_connection_kind"]
+    )
+    .expect("failed to define a metric");
+}
+
 #[derive(Debug, Clone)]
 pub struct PageServerConf {
    pub daemonize: bool,
    pub listen_addr: String,
+    pub http_endpoint_addr: String,
    pub gc_horizon: u64,
    pub gc_period: Duration,
    pub superuser: String,
@@ -42,13 +49,9 @@ pub struct PageServerConf {

    pub pg_distrib_dir: PathBuf,

-    pub repository_format: RepositoryFormat,
-}
+    pub auth_type: AuthType,

-#[derive(Debug, Clone, PartialEq)]
-pub enum RepositoryFormat {
-    Layered,
-    RocksDb,
+    pub auth_validation_public_key_path: Option<PathBuf>,
 }

 impl PageServerConf {
@@ -108,178 +111,3 @@ impl PageServerConf {
        self.pg_distrib_dir.join("lib")
    }
 }
-
-// Zenith ID is a 128-bit random ID.
-// Used to represent various identifiers. Provides handy utility methods and impls.
-// TODO (LizardWizzard) figure out best way to remove boiler plate with trait impls caused by newtype pattern
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)]
-struct ZId([u8; 16]);
-
-impl ZId {
-    pub fn get_from_buf(buf: &mut dyn bytes::Buf) -> ZId {
-        let mut arr = [0u8; 16];
-        buf.copy_to_slice(&mut arr);
-        ZId::from(arr)
-    }
-
-    pub fn as_arr(&self) -> [u8; 16] {
-        self.0
-    }
-
-    pub fn generate() -> Self {
-        let mut tli_buf = [0u8; 16];
-        rand::thread_rng().fill(&mut tli_buf);
-        ZId::from(tli_buf)
-    }
-}
-
-impl FromStr for ZId {
-    type Err = hex::FromHexError;
-
-    fn from_str(s: &str) -> Result<ZId, Self::Err> {
-        Self::from_hex(s)
-    }
-}
-
-// this is needed for pretty serialization and deserialization of ZId's using serde integration with hex crate
-impl FromHex for ZId {
-    type Error = hex::FromHexError;
-
-    fn from_hex<T: AsRef<[u8]>>(hex: T) -> Result<Self, Self::Error> {
-        let mut buf: [u8; 16] = [0u8; 16];
-        hex::decode_to_slice(hex, &mut buf)?;
-        Ok(ZId(buf))
-    }
-}
-
-impl AsRef<[u8]> for ZId {
-    fn as_ref(&self) -> &[u8] {
-        &self.0
-    }
-}
-
-impl From<[u8; 16]> for ZId {
-    fn from(b: [u8; 16]) -> Self {
-        ZId(b)
-    }
-}
-
-impl fmt::Display for ZId {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_str(&hex::encode(self.0))
-    }
-}
-
-/// Zenith timeline IDs are different from PostgreSQL timeline
-/// IDs. They serve a similar purpose though: they differentiate
-/// between different "histories" of the same cluster.  However,
-/// PostgreSQL timeline IDs are a bit cumbersome, because they are only
-/// 32-bits wide, and they must be in ascending order in any given
-/// timeline history.  Those limitations mean that we cannot generate a
-/// new PostgreSQL timeline ID by just generating a random number. And
-/// that in turn is problematic for the "pull/push" workflow, where you
-/// have a local copy of a zenith repository, and you periodically sync
-/// the local changes with a remote server. When you work "detached"
-/// from the remote server, you cannot create a PostgreSQL timeline ID
-/// that's guaranteed to be different from all existing timelines in
-/// the remote server. For example, if two people are having a clone of
-/// the repository on their laptops, and they both create a new branch
-/// with different name. What timeline ID would they assign to their
-/// branches? If they pick the same one, and later try to push the
-/// branches to the same remote server, they will get mixed up.
-///
-/// To avoid those issues, Zenith has its own concept of timelines that
-/// is separate from PostgreSQL timelines, and doesn't have those
-/// limitations. A zenith timeline is identified by a 128-bit ID, which
-/// is usually printed out as a hex string.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
-pub struct ZTimelineId(ZId);
-
-impl FromStr for ZTimelineId {
-    type Err = hex::FromHexError;
-
-    fn from_str(s: &str) -> Result<ZTimelineId, Self::Err> {
-        let value = ZId::from_str(s)?;
-        Ok(ZTimelineId(value))
-    }
-}
-
-impl From<[u8; 16]> for ZTimelineId {
-    fn from(b: [u8; 16]) -> Self {
-        ZTimelineId(ZId::from(b))
-    }
-}
-
-impl ZTimelineId {
-    pub fn get_from_buf(buf: &mut dyn bytes::Buf) -> ZTimelineId {
-        ZTimelineId(ZId::get_from_buf(buf))
-    }
-
-    pub fn as_arr(&self) -> [u8; 16] {
-        self.0.as_arr()
-    }
-
-    pub fn generate() -> Self {
-        ZTimelineId(ZId::generate())
-    }
-}
-
-impl fmt::Display for ZTimelineId {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.0.fmt(f)
-    }
-}
-
-// Zenith Tenant Id represents identifiar of a particular tenant.
-// Is used for distinguishing requests and data belonging to different users.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)]
-pub struct ZTenantId(ZId);
-
-impl FromStr for ZTenantId {
-    type Err = hex::FromHexError;
-
-    fn from_str(s: &str) -> Result<ZTenantId, Self::Err> {
-        let value = ZId::from_str(s)?;
-        Ok(ZTenantId(value))
-    }
-}
-
-impl From<[u8; 16]> for ZTenantId {
-    fn from(b: [u8; 16]) -> Self {
-        ZTenantId(ZId::from(b))
-    }
-}
-
-impl FromHex for ZTenantId {
-    type Error = hex::FromHexError;
-
-    fn from_hex<T: AsRef<[u8]>>(hex: T) -> Result<Self, Self::Error> {
-        Ok(ZTenantId(ZId::from_hex(hex)?))
-    }
-}
-
-impl AsRef<[u8]> for ZTenantId {
-    fn as_ref(&self) -> &[u8] {
-        &self.0 .0
-    }
-}
-
-impl ZTenantId {
-    pub fn get_from_buf(buf: &mut dyn bytes::Buf) -> ZTenantId {
-        ZTenantId(ZId::get_from_buf(buf))
-    }
-
-    pub fn as_arr(&self) -> [u8; 16] {
-        self.0.as_arr()
-    }
-
-    pub fn generate() -> Self {
-        ZTenantId(ZId::generate())
-    }
-}
-
-impl fmt::Display for ZTenantId {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.0.fmt(f)
-    }
-}
--- a/pageserver/src/object_key.rs
+++ b/pageserver/src/object_key.rs
@@ -1,49 +0,0 @@
-//!
-//! Common structs shared by object_repository.rs and object_store.rs.
-//!
-
-use crate::relish::RelishTag;
-use crate::ZTimelineId;
-use serde::{Deserialize, Serialize};
-
-///
-/// ObjectKey is the key type used to identify objects stored in an object
-/// repository. It is shared between object_repository.rs and object_store.rs.
-/// It is mostly opaque to ObjectStore, it just stores and retrieves objects
-/// using the key given by the caller.
-///
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ObjectKey {
-    pub timeline: ZTimelineId,
-    pub tag: ObjectTag,
-}
-
-///
-/// ObjectTag is a part of ObjectKey that is specific to the type of
-/// the stored object.
-///
-/// NB: the order of the enum values is significant!  In particular,
-/// rocksdb_storage.rs assumes that TimelineMetadataTag is first
-///
-/// Buffer is the kind of object that is accessible by the public
-/// get_page_at_lsn() / put_page_image() / put_wal_record() functions in
-/// the repository.rs interface. The rest are internal objects stored in
-/// the key-value store, to store various metadata. They're not directly
-/// accessible outside object_repository.rs
-///
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
-pub enum ObjectTag {
-    // dummy tag preceeding all other keys
-    FirstTag,
-
-    // Metadata about a timeline. Not versioned.
-    TimelineMetadataTag,
-
-    // These objects store metadata about one relish. Currently it's used
-    // just to track the relish's size. It's not used for non-blocky relishes
-    // at all.
-    RelationMetadata(RelishTag),
-
-    // These are the pages exposed in the public Repository/Timeline interface.
-    Buffer(RelishTag, u32),
-}
--- a/pageserver/src/object_repository.rs
+++ b/pageserver/src/object_repository.rs
--- a/pageserver/src/object_store.rs
+++ b/pageserver/src/object_store.rs
@@ -1,93 +0,0 @@
-//! Low-level key-value storage abstraction.
-//!
-use crate::object_key::*;
-use crate::relish::*;
-use crate::ZTimelineId;
-use anyhow::Result;
-use std::collections::HashSet;
-use std::iter::Iterator;
-use zenith_utils::lsn::Lsn;
-
-///
-/// Low-level storage abstraction.
-///
-/// All the data in the repository is stored in a key-value store. This trait
-/// abstracts the details of the key-value store.
-///
-/// A simple key-value store would support just GET and PUT operations with
-/// a key, but the upper layer needs slightly complicated read operations
-///
-/// The most frequently used function is 'object_versions'. It is used
-/// to look up a page version. It is LSN aware, in that the caller
-/// specifies an LSN, and the function returns all values for that
-/// block with the same or older LSN.
-///
-pub trait ObjectStore: Send + Sync {
-    ///
-    /// Store a value with given key.
-    ///
-    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()>;
-
-    /// Read entry with the exact given key.
-    ///
-    /// This is used for retrieving metadata with special key that doesn't
-    /// correspond to any real relation.
-    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>>;
-
-    /// Read key greater or equal than specified
-    fn get_next_key(&self, key: &ObjectKey) -> Result<Option<ObjectKey>>;
-
-    /// Iterate through all page versions of one object.
-    ///
-    /// Returns all page versions in descending LSN order, along with the LSN
-    /// of each page version.
-    fn object_versions<'a>(
-        &'a self,
-        key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>>;
-
-    /// Iterate through versions of all objects in a timeline.
-    ///
-    /// Returns objects in increasing key-version order.
-    /// Returns all versions up to and including the specified LSN.
-    fn objects<'a>(
-        &'a self,
-        timeline: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = Result<(ObjectTag, Lsn, Vec<u8>)>> + 'a>>;
-
-    /// Iterate through all keys with given tablespace and database ID, and LSN <= 'lsn'.
-    /// Both dbnode and spcnode can be InvalidId (0) which means get all relations in tablespace/cluster
-    ///
-    /// This is used to implement 'create database'
-    fn list_rels(
-        &self,
-        timelineid: ZTimelineId,
-        spcnode: u32,
-        dbnode: u32,
-        lsn: Lsn,
-    ) -> Result<HashSet<RelTag>>;
-
-    /// Iterate through non-rel relishes
-    ///
-    /// This is used to prepare tarball for new node startup.
-    /// Returns objects in increasing key-version order.
-    fn list_nonrels<'a>(&'a self, timelineid: ZTimelineId, lsn: Lsn) -> Result<HashSet<RelishTag>>;
-
-    /// Iterate through objects tags. If nonrel_only, then only non-relationa data is iterated.
-    ///
-    /// This is used to implement GC and preparing tarball for new node startup
-    /// Returns objects in increasing key-version order.
-    fn list_objects<'a>(
-        &'a self,
-        timelineid: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = ObjectTag> + 'a>>;
-
-    /// Unlink object (used by GC). This mehod may actually delete object or just mark it for deletion.
-    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()>;
-
-    // Compact storage and remove versions marged for deletion
-    fn compact(&self);
-}
--- a/pageserver/src/page_service.rs
+++ b/pageserver/src/page_service.rs
@@ -10,31 +10,34 @@
 //     *callmemaybe <zenith timelineid> $url* -- ask pageserver to start walreceiver on $url
 //

-use anyhow::{anyhow, bail, ensure};
+use anyhow::{anyhow, bail, ensure, Context, Result};
 use bytes::{Buf, BufMut, Bytes, BytesMut};
+use lazy_static::lazy_static;
 use log::*;
 use regex::Regex;
-use std::io::Write;
 use std::net::TcpListener;
+use std::str;
 use std::str::FromStr;
+use std::sync::Arc;
 use std::thread;
 use std::{io, net::TcpStream};
+use zenith_metrics::{register_histogram_vec, HistogramVec};
+use zenith_utils::auth::{self, JwtAuth};
+use zenith_utils::auth::{Claims, Scope};
+use zenith_utils::lsn::Lsn;
 use zenith_utils::postgres_backend::PostgresBackend;
 use zenith_utils::postgres_backend::{self, AuthType};
 use zenith_utils::pq_proto::{
    BeMessage, FeMessage, RowDescriptor, HELLO_WORLD_ROW, SINGLE_COL_ROWDESC,
 };
-use zenith_utils::{bin_ser::BeSer, lsn::Lsn};
+use zenith_utils::zid::{ZTenantId, ZTimelineId};

 use crate::basebackup;
 use crate::branches;
-use crate::page_cache;
 use crate::relish::*;
-use crate::repository::Modification;
+use crate::tenant_mgr;
 use crate::walreceiver;
 use crate::PageServerConf;
-use crate::ZTenantId;
-use crate::ZTimelineId;

 // Wrapped in libpq CopyData
 enum PagestreamFeMessage {
@@ -138,34 +141,77 @@ impl PagestreamBeMessage {
 ///
 /// Listens for connections, and launches a new handler thread for each.
 ///
-pub fn thread_main(conf: &'static PageServerConf, listener: TcpListener) -> anyhow::Result<()> {
+pub fn thread_main(
+    conf: &'static PageServerConf,
+    auth: Option<Arc<JwtAuth>>,
+    listener: TcpListener,
+    auth_type: AuthType,
+) -> anyhow::Result<()> {
    loop {
        let (socket, peer_addr) = listener.accept()?;
        debug!("accepted connection from {}", peer_addr);
        socket.set_nodelay(true).unwrap();
-
+        let local_auth = auth.clone();
        thread::spawn(move || {
-            if let Err(err) = page_service_conn_main(conf, socket) {
+            if let Err(err) = page_service_conn_main(conf, local_auth, socket, auth_type) {
                error!("error: {}", err);
            }
        });
    }
 }

-fn page_service_conn_main(conf: &'static PageServerConf, socket: TcpStream) -> anyhow::Result<()> {
-    let mut conn_handler = PageServerHandler::new(conf);
-    let mut pgbackend = PostgresBackend::new(socket, AuthType::Trust)?;
+fn page_service_conn_main(
+    conf: &'static PageServerConf,
+    auth: Option<Arc<JwtAuth>>,
+    socket: TcpStream,
+    auth_type: AuthType,
+) -> anyhow::Result<()> {
+    // Immediatsely increment the gauge, then create a job to decrement it on thread exit.
+    // One of the pros of `defer!` is that this will *most probably*
+    // get called, even in presence of panics.
+    let gauge = crate::LIVE_CONNECTIONS_COUNT.with_label_values(&["page_service"]);
+    gauge.inc();
+    scopeguard::defer! {
+        gauge.dec();
+    }
+
+    let mut conn_handler = PageServerHandler::new(conf, auth);
+    let pgbackend = PostgresBackend::new(socket, auth_type)?;
    pgbackend.run(&mut conn_handler)
 }

 #[derive(Debug)]
 struct PageServerHandler {
    conf: &'static PageServerConf,
+    auth: Option<Arc<JwtAuth>>,
+    claims: Option<Claims>,
+}
+
+const TIME_BUCKETS: &[f64] = &[
+    0.00001, // 1/100000 s
+    0.0001, 0.00015, 0.0002, 0.00025, 0.0003, 0.00035, 0.0005, 0.00075, // 1/10000 s
+    0.001, 0.0025, 0.005, 0.0075, // 1/1000 s
+    0.01, 0.0125, 0.015, 0.025, 0.05, // 1/100 s
+    0.1,  // 1/10 s
+];
+
+lazy_static! {
+    static ref SMGR_QUERY_TIME: HistogramVec = register_histogram_vec!(
+        "pageserver_smgr_query_time",
+        "Time spent on smgr query handling",
+        &["smgr_query_type"],
+        TIME_BUCKETS.into()
+    )
+    .expect("failed to define a metric");
 }

 impl PageServerHandler {
-    pub fn new(conf: &'static PageServerConf) -> Self {
-        PageServerHandler { conf }
+    pub fn new(conf: &'static PageServerConf, auth: Option<Arc<JwtAuth>>) -> Self {
+        PageServerHandler {
+            conf,
+            auth,
+            claims: None,
+        }
    }

    fn handle_controlfile(&self, pgb: &mut PostgresBackend) -> io::Result<()> {
@@ -183,13 +229,10 @@ impl PageServerHandler {
        tenantid: ZTenantId,
    ) -> anyhow::Result<()> {
        // Check that the timeline exists
-        let repository = page_cache::get_repository_for_tenant(&tenantid)?;
-        let timeline = repository.get_timeline(timelineid).map_err(|_| {
-            anyhow!(
-                "client requested pagestream on timeline {} which does not exist in page server",
-                timelineid
-            )
-        })?;
+        let repository = tenant_mgr::get_repository_for_tenant(&tenantid)?;
+        let timeline = repository
+            .get_timeline(timelineid)
+            .context(format!("error fetching timeline {}", timelineid))?;

        /* switch client to COPYBOTH */
        pgb.write_message(&BeMessage::CopyBothResponse)?;
@@ -214,7 +257,11 @@ impl PageServerHandler {
                    };
                    let tag = RelishTag::Relation(rel);

-                    let exist = timeline.get_rel_exists(tag, req.lsn).unwrap_or(false);
+                    let exist = SMGR_QUERY_TIME
+                        .with_label_values(&["get_rel_exists"])
+                        .observe_closure_duration(|| {
+                            timeline.get_rel_exists(tag, req.lsn).unwrap_or(false)
+                        });

                    PagestreamBeMessage::Status(PagestreamStatusResponse {
                        ok: exist,
@@ -230,7 +277,16 @@ impl PageServerHandler {
                    };
                    let tag = RelishTag::Relation(rel);

-                    let n_blocks = timeline.get_rel_size(tag, req.lsn).unwrap_or(0);
+                    let n_blocks = SMGR_QUERY_TIME
+                        .with_label_values(&["get_rel_size"])
+                        .observe_closure_duration(|| {
+                            // Return 0 if relation is not found.
+                            // This is what postgres smgr expects.
+                            timeline
+                                .get_relish_size(tag, req.lsn)
+                                .unwrap_or(Some(0))
+                                .unwrap_or(0)
+                        });

                    PagestreamBeMessage::Nblocks(PagestreamStatusResponse { ok: true, n_blocks })
                }
@@ -243,22 +299,26 @@ impl PageServerHandler {
                    };
                    let tag = RelishTag::Relation(rel);

-                    let read_response = match timeline.get_page_at_lsn(tag, req.blkno, req.lsn) {
-                        Ok(p) => PagestreamReadResponse {
-                            ok: true,
-                            n_blocks: 0,
-                            page: p,
-                        },
-                        Err(e) => {
-                            const ZERO_PAGE: [u8; 8192] = [0; 8192];
-                            error!("get_page_at_lsn: {}", e);
-                            PagestreamReadResponse {
-                                ok: false,
-                                n_blocks: 0,
-                                page: Bytes::from_static(&ZERO_PAGE),
+                    let read_response = SMGR_QUERY_TIME
+                        .with_label_values(&["get_page_at_lsn"])
+                        .observe_closure_duration(|| {
+                            match timeline.get_page_at_lsn(tag, req.blkno, req.lsn) {
+                                Ok(p) => PagestreamReadResponse {
+                                    ok: true,
+                                    n_blocks: 0,
+                                    page: p,
+                                },
+                                Err(e) => {
+                                    const ZERO_PAGE: [u8; 8192] = [0; 8192];
+                                    error!("get_page_at_lsn: {}", e);
+                                    PagestreamReadResponse {
+                                        ok: false,
+                                        n_blocks: 0,
+                                        page: Bytes::from_static(&ZERO_PAGE),
+                                    }
+                                }
                            }
-                        }
-                    };
+                        });

                    PagestreamBeMessage::Read(read_response)
                }
@@ -278,14 +338,10 @@ impl PageServerHandler {
        tenantid: ZTenantId,
    ) -> anyhow::Result<()> {
        // check that the timeline exists
-        let repository = page_cache::get_repository_for_tenant(&tenantid)?;
-        let timeline = repository.get_timeline(timelineid).map_err(|e| {
-            error!("error fetching timeline: {:?}", e);
-            anyhow!(
-                "client requested basebackup on timeline {} which does not exist in page server",
-                timelineid
-            )
-        })?;
+        let repository = tenant_mgr::get_repository_for_tenant(&tenantid)?;
+        let timeline = repository
+            .get_timeline(timelineid)
+            .context(format!("error fetching timeline {}", timelineid))?;
        /* switch client to COPYOUT */
        pgb.write_message(&BeMessage::CopyOutResponse)?;
        info!("sent CopyOut");
@@ -309,9 +365,56 @@ impl PageServerHandler {

        Ok(())
    }
+
+    // when accessing management api supply None as an argument
+    // when using to authorize tenant pass corresponding tenant id
+    fn check_permission(&self, tenantid: Option<ZTenantId>) -> Result<()> {
+        if self.auth.is_none() {
+            // auth is set to Trust, nothing to check so just return ok
+            return Ok(());
+        }
+        // auth is some, just checked above, when auth is some
+        // then claims are always present because of checks during connetion init
+        // so this expect won't trigger
+        let claims = self
+            .claims
+            .as_ref()
+            .expect("claims presence already checked");
+        Ok(auth::check_permission(claims, tenantid)?)
+    }
 }

 impl postgres_backend::Handler for PageServerHandler {
+    fn check_auth_jwt(
+        &mut self,
+        _pgb: &mut PostgresBackend,
+        jwt_response: &[u8],
+    ) -> anyhow::Result<()> {
+        // this unwrap is never triggered, because check_auth_jwt only called when auth_type is ZenithJWT
+        // which requires auth to be present
+        let data = self
+            .auth
+            .as_ref()
+            .as_ref()
+            .unwrap()
+            .decode(&str::from_utf8(jwt_response)?)?;
+
+        if matches!(data.claims.scope, Scope::Tenant) {
+            ensure!(
+                data.claims.tenant_id.is_some(),
+                "jwt token scope is Tenant, but tenant id is missing"
+            )
+        }
+
+        info!(
+            "jwt auth succeeded for scope: {:#?} by tenantid: {:?}",
+            data.claims.scope, data.claims.tenant_id,
+        );
+
+        self.claims = Some(data.claims);
+        Ok(())
+    }
+
    fn process_query(
        &mut self,
        pgb: &mut PostgresBackend,
@@ -338,6 +441,8 @@ impl postgres_backend::Handler for PageServerHandler {
            let tenantid = ZTenantId::from_str(params[0])?;
            let timelineid = ZTimelineId::from_str(params[1])?;

+            self.check_permission(Some(tenantid))?;
+
            self.handle_pagerequests(pgb, timelineid, tenantid)?;
        } else if query_string.starts_with("basebackup ") {
            let (_, params_raw) = query_string.split_at("basebackup ".len());
@@ -350,6 +455,8 @@ impl postgres_backend::Handler for PageServerHandler {
            let tenantid = ZTenantId::from_str(params[0])?;
            let timelineid = ZTimelineId::from_str(params[1])?;

+            self.check_permission(Some(tenantid))?;
+
            // TODO are there any tests with lsn option?
            let lsn = if params.len() == 3 {
                Some(Lsn::from_str(params[2])?)
@@ -376,11 +483,13 @@ impl postgres_backend::Handler for PageServerHandler {
            let timelineid = ZTimelineId::from_str(caps.get(2).unwrap().as_str())?;
            let connstr = caps.get(3).unwrap().as_str().to_owned();

+            self.check_permission(Some(tenantid))?;
+
            // Check that the timeline exists
-            let repository = page_cache::get_repository_for_tenant(&tenantid)?;
-            if repository.get_timeline(timelineid).is_err() {
-                bail!("client requested callmemaybe on timeline {} which does not exist in page server", timelineid);
-            }
+            let repository = tenant_mgr::get_repository_for_tenant(&tenantid)?;
+            repository
+                .get_timeline(timelineid)
+                .context(format!("error fetching timeline {}", timelineid))?;

            walreceiver::launch_wal_receiver(&self.conf, timelineid, &connstr, tenantid.to_owned());

@@ -399,6 +508,8 @@ impl postgres_backend::Handler for PageServerHandler {
            let branchname = caps.get(2).ok_or_else(err)?.as_str().to_owned();
            let startpoint_str = caps.get(3).ok_or_else(err)?.as_str().to_owned();

+            self.check_permission(Some(tenantid))?;
+
            let branch =
                branches::create_branch(&self.conf, &branchname, &startpoint_str, &tenantid)?;
            let branch = serde_json::to_vec(&branch)?;
@@ -406,72 +517,6 @@ impl postgres_backend::Handler for PageServerHandler {
            pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
                .write_message_noflush(&BeMessage::DataRow(&[Some(&branch)]))?
                .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
-        } else if query_string.starts_with("push ") {
-            // push <zenith tenantid as hex string> <zenith timelineid as hex string>
-            let re = Regex::new(r"^push ([[:xdigit:]]+) ([[:xdigit:]]+)$").unwrap();
-
-            let caps = re
-                .captures(query_string)
-                .ok_or_else(|| anyhow!("invalid push: '{}'", query_string))?;
-
-            let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;
-            let timelineid = ZTimelineId::from_str(caps.get(2).unwrap().as_str())?;
-
-            let start_lsn = Lsn(0); // TODO this needs to come from the repo
-            let timeline = page_cache::get_repository_for_tenant(&tenantid)?
-                .create_empty_timeline(timelineid, start_lsn)?;
-
-            pgb.write_message(&BeMessage::CopyInResponse)?;
-
-            let mut last_lsn = Lsn(0);
-
-            while let Some(msg) = pgb.read_message()? {
-                match msg {
-                    FeMessage::CopyData(bytes) => {
-                        let modification = Modification::des(&bytes)?;
-
-                        last_lsn = modification.lsn;
-                        timeline.put_raw_data(modification.tag, modification.lsn, &modification.data)?;
-                    }
-                    FeMessage::CopyDone => {
-                        timeline.advance_last_valid_lsn(last_lsn);
-                        break;
-                    }
-                    FeMessage::Sync => {}
-                    _ => bail!("unexpected message {:?}", msg),
-                }
-            }
-
-            pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
-        } else if query_string.starts_with("request_push ") {
-            // request_push <zenith tenantid as hex string> <zenith timelineid as hex string> <postgres_connection_uri>
-            let re = Regex::new(r"^request_push ([[:xdigit:]]+) ([[:xdigit:]]+) (.*)$").unwrap();
-
-            let caps = re
-                .captures(query_string)
-                .ok_or_else(|| anyhow!("invalid request_push: '{}'", query_string))?;
-
-            let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;
-            let timelineid = ZTimelineId::from_str(caps.get(2).unwrap().as_str())?;
-            let postgres_connection_uri = caps.get(3).unwrap().as_str();
-
-            let timeline =
-                page_cache::get_repository_for_tenant(&tenantid)?.get_timeline(timelineid)?;
-
-            let mut conn = postgres::Client::connect(postgres_connection_uri, postgres::NoTls)?;
-            let mut copy_in = conn.copy_in(format!("push {}", timelineid.to_string()).as_str())?;
-
-            let history = timeline.history()?;
-            for update_res in history {
-                let update = update_res?;
-                let update_bytes = update.ser()?;
-                copy_in.write_all(&update_bytes)?;
-                copy_in.flush()?; // ensure that messages are sent inside individual CopyData packets
-            }
-
-            copy_in.finish()?;
-
-            pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
        } else if query_string.starts_with("branch_list ") {
            // branch_list <zenith tenantid as hex string>
            let re = Regex::new(r"^branch_list ([[:xdigit:]]+)$").unwrap();
@@ -501,9 +546,11 @@ impl postgres_backend::Handler for PageServerHandler {
            let re = Regex::new(r"^tenant_create ([[:xdigit:]]+)$").unwrap();
            let caps = re.captures(&query_string).ok_or_else(err)?;

+            self.check_permission(None)?;
+
            let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;

-            page_cache::create_repository_for_tenant(&self.conf, tenantid)?;
+            tenant_mgr::create_repository_for_tenant(&self.conf, tenantid)?;

            pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
                .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
@@ -537,18 +584,11 @@ impl postgres_backend::Handler for PageServerHandler {
                .map(|h| h.as_str().parse())
                .unwrap_or(Ok(self.conf.gc_horizon))?;

-            let repo = page_cache::get_repository_for_tenant(&tenantid)?;
+            let repo = tenant_mgr::get_repository_for_tenant(&tenantid)?;

            let result = repo.gc_iteration(Some(timelineid), gc_horizon, true)?;

            pgb.write_message_noflush(&BeMessage::RowDescription(&[
-                RowDescriptor::int8_col(b"n_relations"),
-                RowDescriptor::int8_col(b"truncated"),
-                RowDescriptor::int8_col(b"deleted"),
-                RowDescriptor::int8_col(b"prep_deleted"),
-                RowDescriptor::int8_col(b"slru_deleted"),
-                RowDescriptor::int8_col(b"chkp_deleted"),
-                RowDescriptor::int8_col(b"dropped"),
                RowDescriptor::int8_col(b"snapshot_relfiles_total"),
                RowDescriptor::int8_col(b"snapshot_relfiles_needed_by_cutoff"),
                RowDescriptor::int8_col(b"snapshot_relfiles_needed_by_branches"),
@@ -564,55 +604,42 @@ impl postgres_backend::Handler for PageServerHandler {
                RowDescriptor::int8_col(b"elapsed"),
            ]))?
            .write_message_noflush(&BeMessage::DataRow(&[
-                Some(&result.n_relations.to_string().as_bytes()),
-                Some(&result.truncated.to_string().as_bytes()),
-                Some(&result.deleted.to_string().as_bytes()),
-                Some(&result.prep_deleted.to_string().as_bytes()),
-                Some(&result.slru_deleted.to_string().as_bytes()),
-                Some(&result.chkp_deleted.to_string().as_bytes()),
-                Some(&result.dropped.to_string().as_bytes()),
-                Some(&result.snapshot_relfiles_total.to_string().as_bytes()),
+                Some(&result.ondisk_relfiles_total.to_string().as_bytes()),
                Some(
                    &result
-                        .snapshot_relfiles_needed_by_cutoff
+                        .ondisk_relfiles_needed_by_cutoff
                        .to_string()
                        .as_bytes(),
                ),
                Some(
                    &result
-                        .snapshot_relfiles_needed_by_branches
+                        .ondisk_relfiles_needed_by_branches
                        .to_string()
                        .as_bytes(),
                ),
-                Some(&result.snapshot_relfiles_not_updated.to_string().as_bytes()),
-                Some(&result.snapshot_relfiles_removed.to_string().as_bytes()),
-                Some(&result.snapshot_relfiles_dropped.to_string().as_bytes()),
-                Some(&result.snapshot_nonrelfiles_total.to_string().as_bytes()),
+                Some(&result.ondisk_relfiles_not_updated.to_string().as_bytes()),
+                Some(&result.ondisk_relfiles_removed.to_string().as_bytes()),
+                Some(&result.ondisk_relfiles_dropped.to_string().as_bytes()),
+                Some(&result.ondisk_nonrelfiles_total.to_string().as_bytes()),
                Some(
                    &result
-                        .snapshot_nonrelfiles_needed_by_cutoff
+                        .ondisk_nonrelfiles_needed_by_cutoff
                        .to_string()
                        .as_bytes(),
                ),
                Some(
                    &result
-                        .snapshot_nonrelfiles_needed_by_branches
+                        .ondisk_nonrelfiles_needed_by_branches
                        .to_string()
                        .as_bytes(),
                ),
-                Some(
-                    &result
-                        .snapshot_nonrelfiles_not_updated
-                        .to_string()
-                        .as_bytes(),
-                ),
-                Some(&result.snapshot_nonrelfiles_removed.to_string().as_bytes()),
-                Some(&result.snapshot_nonrelfiles_dropped.to_string().as_bytes()),
+                Some(&result.ondisk_nonrelfiles_not_updated.to_string().as_bytes()),
+                Some(&result.ondisk_nonrelfiles_removed.to_string().as_bytes()),
+                Some(&result.ondisk_nonrelfiles_dropped.to_string().as_bytes()),
                Some(&result.elapsed.as_millis().to_string().as_bytes()),
            ]))?
            .write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;
        } else {
-            error!("received unknown command from client");
            bail!("unknown command");
        }

--- a/pageserver/src/relish.rs
+++ b/pageserver/src/relish.rs
@@ -107,6 +107,22 @@ impl RelishTag {
        }
    }

+    // Physical relishes represent files and use
+    // RelationSizeEntry to track existing and dropped files.
+    // They can be both blocky and non-blocky.
+    pub const fn is_physical(&self) -> bool {
+        match self {
+            // These relishes represent physical files
+            RelishTag::Relation(_)
+            | RelishTag::Slru { .. }
+            | RelishTag::FileNodeMap { .. }
+            | RelishTag::TwoPhase { .. } => true,
+
+            // and these don't
+            RelishTag::ControlFile | RelishTag::Checkpoint => false,
+        }
+    }
+
    // convenience function to check if this relish is a normal relation.
    pub const fn is_relation(&self) -> bool {
        if let RelishTag::Relation(_) = self {
@@ -204,7 +220,7 @@ pub enum SlruKind {
 }

 impl SlruKind {
-    fn to_str(&self) -> &'static str {
+    pub fn to_str(&self) -> &'static str {
        match self {
            Self::Clog => "pg_xact",
            Self::MultiXactMembers => "pg_multixact/members",
--- a/pageserver/src/repository.rs
+++ b/pageserver/src/repository.rs
@@ -1,18 +1,13 @@
-use crate::object_key::*;
 use crate::relish::*;
-use crate::ZTimelineId;
 use anyhow::Result;
 use bytes::{Buf, BufMut, Bytes, BytesMut};
-use postgres_ffi::nonrelfile_utils::transaction_id_get_status;
-use postgres_ffi::pg_constants;
-use postgres_ffi::TransactionId;
 use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
-use std::iter::Iterator;
 use std::ops::AddAssign;
 use std::sync::Arc;
 use std::time::Duration;
 use zenith_utils::lsn::Lsn;
+use zenith_utils::zid::ZTimelineId;

 ///
 /// A repository corresponds to one .zenith directory. One repository holds multiple
@@ -35,6 +30,7 @@ pub trait Repository: Send + Sync {
    /// garbage collection is periodically performed by gc thread,
    /// but it can be explicitly requested through page server api.
    ///
+    /// 'timelineid' specifies the timeline to GC, or None for all.
    /// `horizon` specifies delta from last lsn to preserve all object versions (pitr interval).
    /// `compact` parameter is used to force compaction of storage.
    /// some storage implementation are based on lsm tree and require periodic merge (compaction).
@@ -43,8 +39,13 @@ pub trait Repository: Send + Sync {
    /// to make sure that all detected garbage is removed.
    /// so right now `compact` is set to true when gc explicitly requested through page srver api,
    /// and is st to false in gc threads which infinitely repeats gc iterations in loop.
-    fn gc_iteration(&self, timelineid: Option<ZTimelineId>, horizon: u64, compact: bool) -> Result<GcResult>;
-    
+    fn gc_iteration(
+        &self,
+        timelineid: Option<ZTimelineId>,
+        horizon: u64,
+        compact: bool,
+    ) -> Result<GcResult>;
+
    // TODO get timelines?
    //fn get_stats(&self) -> RepositoryStats;
 }
@@ -54,56 +55,38 @@ pub trait Repository: Send + Sync {
 ///
 #[derive(Default)]
 pub struct GcResult {
-    // FIXME: These counters make sense for the ObjectRepository. They are not used
-    // by the LayeredRepository.
-    pub n_relations: u64,
-    pub inspected: u64,
-    pub truncated: u64,
-    pub deleted: u64,
-    pub prep_deleted: u64, // 2PC prepare
-    pub slru_deleted: u64, // SLRU (clog, multixact)
-    pub chkp_deleted: u64, // Checkpoints
-    pub dropped: u64,
+    pub ondisk_relfiles_total: u64,
+    pub ondisk_relfiles_needed_by_cutoff: u64,
+    pub ondisk_relfiles_needed_by_branches: u64,
+    pub ondisk_relfiles_not_updated: u64,
+    pub ondisk_relfiles_removed: u64, // # of layer files removed because they have been made obsolete by newer ondisk files.
+    pub ondisk_relfiles_dropped: u64, // # of layer files removed because the relation was dropped

-    // These are used for the LayeredRepository instead
-    pub snapshot_relfiles_total: u64,
-    pub snapshot_relfiles_needed_by_cutoff: u64,
-    pub snapshot_relfiles_needed_by_branches: u64,
-    pub snapshot_relfiles_not_updated: u64,
-    pub snapshot_relfiles_removed: u64, // # of snapshot files removed because they have been made obsolete by newer snapshot files.
-    pub snapshot_relfiles_dropped: u64, // # of snapshot files removed because the relation was dropped
-
-    pub snapshot_nonrelfiles_total: u64,
-    pub snapshot_nonrelfiles_needed_by_cutoff: u64,
-    pub snapshot_nonrelfiles_needed_by_branches: u64,
-    pub snapshot_nonrelfiles_not_updated: u64,
-    pub snapshot_nonrelfiles_removed: u64, // # of snapshot files removed because they have been made obsolete by newer snapshot files.
-    pub snapshot_nonrelfiles_dropped: u64, // # of snapshot files removed because the relation was dropped
+    pub ondisk_nonrelfiles_total: u64,
+    pub ondisk_nonrelfiles_needed_by_cutoff: u64,
+    pub ondisk_nonrelfiles_needed_by_branches: u64,
+    pub ondisk_nonrelfiles_not_updated: u64,
+    pub ondisk_nonrelfiles_removed: u64, // # of layer files removed because they have been made obsolete by newer ondisk files.
+    pub ondisk_nonrelfiles_dropped: u64, // # of layer files removed because the relation was dropped

    pub elapsed: Duration,
 }

 impl AddAssign for GcResult {
    fn add_assign(&mut self, other: Self) {
-        self.n_relations += other.n_relations;
-        self.truncated += other.truncated;
-        self.deleted += other.deleted;
-        self.dropped += other.dropped;
+        self.ondisk_relfiles_total += other.ondisk_relfiles_total;
+        self.ondisk_relfiles_needed_by_cutoff += other.ondisk_relfiles_needed_by_cutoff;
+        self.ondisk_relfiles_needed_by_branches += other.ondisk_relfiles_needed_by_branches;
+        self.ondisk_relfiles_not_updated += other.ondisk_relfiles_not_updated;
+        self.ondisk_relfiles_removed += other.ondisk_relfiles_removed;
+        self.ondisk_relfiles_dropped += other.ondisk_relfiles_dropped;

-        self.snapshot_relfiles_total += other.snapshot_relfiles_total;
-        self.snapshot_relfiles_needed_by_cutoff += other.snapshot_relfiles_needed_by_cutoff;
-        self.snapshot_relfiles_needed_by_branches += other.snapshot_relfiles_needed_by_branches;
-        self.snapshot_relfiles_not_updated += other.snapshot_relfiles_not_updated;
-        self.snapshot_relfiles_removed += other.snapshot_relfiles_removed;
-        self.snapshot_relfiles_dropped += other.snapshot_relfiles_dropped;
-
-        self.snapshot_nonrelfiles_total += other.snapshot_nonrelfiles_total;
-        self.snapshot_nonrelfiles_needed_by_cutoff += other.snapshot_nonrelfiles_needed_by_cutoff;
-        self.snapshot_nonrelfiles_needed_by_branches +=
-            other.snapshot_nonrelfiles_needed_by_branches;
-        self.snapshot_nonrelfiles_not_updated += other.snapshot_nonrelfiles_not_updated;
-        self.snapshot_nonrelfiles_removed += other.snapshot_nonrelfiles_removed;
-        self.snapshot_nonrelfiles_dropped += other.snapshot_nonrelfiles_dropped;
+        self.ondisk_nonrelfiles_total += other.ondisk_nonrelfiles_total;
+        self.ondisk_nonrelfiles_needed_by_cutoff += other.ondisk_nonrelfiles_needed_by_cutoff;
+        self.ondisk_nonrelfiles_needed_by_branches += other.ondisk_nonrelfiles_needed_by_branches;
+        self.ondisk_nonrelfiles_not_updated += other.ondisk_nonrelfiles_not_updated;
+        self.ondisk_nonrelfiles_removed += other.ondisk_nonrelfiles_removed;
+        self.ondisk_nonrelfiles_dropped += other.ondisk_nonrelfiles_dropped;

        self.elapsed += other.elapsed;
    }
@@ -120,8 +103,8 @@ pub trait Timeline: Send + Sync {
    /// Look up given page in the cache.
    fn get_page_at_lsn_nowait(&self, tag: RelishTag, blknum: u32, lsn: Lsn) -> Result<Bytes>;

-    /// Get size of relation
-    fn get_rel_size(&self, tag: RelishTag, lsn: Lsn) -> Result<u32>;
+    /// Get size of a relish
+    fn get_relish_size(&self, tag: RelishTag, lsn: Lsn) -> Result<Option<u32>>;

    /// Does relation exist?
    fn get_rel_exists(&self, tag: RelishTag, lsn: Lsn) -> Result<bool>;
@@ -145,27 +128,15 @@ pub trait Timeline: Send + Sync {
    fn put_wal_record(&self, tag: RelishTag, blknum: u32, rec: WALRecord) -> Result<()>;

    /// Like put_wal_record, but with ready-made image of the page.
-    fn put_page_image(
-        &self,
-        tag: RelishTag,
-        blknum: u32,
-        lsn: Lsn,
-        img: Bytes,
-        update_meta: bool,
-    ) -> Result<()>;
+    fn put_page_image(&self, tag: RelishTag, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()>;

    /// Truncate relation
    fn put_truncation(&self, rel: RelishTag, lsn: Lsn, nblocks: u32) -> Result<()>;

-    /// Unlink relation. This method is used for marking dropped relations.
+    /// Unlink relish.
+    /// This method is used for marking dropped relations and truncated SLRU segments
    fn put_unlink(&self, tag: RelishTag, lsn: Lsn) -> Result<()>;

-    /// Put raw data
-    fn put_raw_data(&self, tag: ObjectTag, lsn: Lsn, data: &[u8]) -> Result<()>;
-
-    // Get object tag greater or equal than specified
-    fn get_next_tag(&self, tag: ObjectTag) -> Result<Option<ObjectTag>>;
-
    /// Remember the all WAL before the given LSN has been processed.
    ///
    /// The WAL receiver calls this after the put_* functions, to indicate that
@@ -192,58 +163,6 @@ pub trait Timeline: Send + Sync {
    /// NOTE: This has nothing to do with checkpoint in PostgreSQL. We don't
    /// know anything about them here in the repository.
    fn checkpoint(&self) -> Result<()>;
-
-    /// Events for all relations in the timeline.
-    /// Contains updates from start up to the last valid LSN
-    /// at time of history() call. This lsn can be read via the lsn() function.
-    ///
-    /// Relation size is increased implicitly and decreased with Truncate updates.
-    // TODO ordering guarantee?
-    fn history<'a>(&'a self) -> Result<Box<dyn History + 'a>>;
-
-    // Check transaction status
-    fn get_tx_status(&self, xid: TransactionId, lsn: Lsn) -> anyhow::Result<u8> {
-        let pageno = xid / pg_constants::CLOG_XACTS_PER_PAGE;
-        let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-        let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-
-        let clog_page = self.get_page_at_lsn(
-            RelishTag::Slru {
-                slru: SlruKind::Clog,
-                segno,
-            },
-            rpageno,
-            lsn,
-        )?;
-        let status = transaction_id_get_status(xid, &clog_page[..]);
-        Ok(status)
-    }
-}
-
-pub trait History: Iterator<Item = Result<Modification>> {
-    /// The last_valid_lsn at the time of history() call.
-    fn lsn(&self) -> Lsn;
-}
-
-//
-// Structure representing any update operation of object storage.
-// It is used to copy object storage content in PUSH method.
-//
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Modification {
-    pub tag: ObjectTag,
-    pub lsn: Lsn,
-    pub data: Vec<u8>,
-}
-
-impl Modification {
-    pub fn new(entry: (ObjectTag, Lsn, Vec<u8>)) -> Modification {
-        Modification {
-            tag: entry.0,
-            lsn: entry.1,
-            data: entry.2,
-        }
-    }
 }

 #[derive(Clone)]
@@ -295,17 +214,15 @@ impl WALRecord {
 mod tests {
    use super::*;
    use crate::layered_repository::LayeredRepository;
-    use crate::object_repository::ObjectRepository;
-    use crate::object_repository::{ObjectValue, PageEntry, RelationSizeEntry};
-    use crate::rocksdb_storage::RocksObjectStore;
    use crate::walredo::{WalRedoError, WalRedoManager};
-    use crate::{PageServerConf, RepositoryFormat, ZTenantId};
+    use crate::PageServerConf;
    use postgres_ffi::pg_constants;
    use std::fs;
    use std::path::PathBuf;
    use std::str::FromStr;
    use std::time::Duration;
-    use zenith_utils::bin_ser::BeSer;
+    use zenith_utils::postgres_backend::AuthType;
+    use zenith_utils::zid::ZTenantId;

    /// Arbitrary relation tag, for testing.
    const TESTREL_A: RelishTag = RelishTag::Relation(RelTag {
@@ -333,10 +250,7 @@ mod tests {

    static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);

-    fn get_test_repo(
-        test_name: &str,
-        repository_format: RepositoryFormat,
-    ) -> Result<Box<dyn Repository>> {
+    fn get_test_repo(test_name: &str) -> Result<Box<dyn Repository>> {
        let repo_dir = PathBuf::from(format!("../tmp_check/test_{}", test_name));
        let _ = fs::remove_dir_all(&repo_dir);
        fs::create_dir_all(&repo_dir)?;
@@ -347,10 +261,12 @@ mod tests {
            gc_horizon: 64 * 1024 * 1024,
            gc_period: Duration::from_secs(10),
            listen_addr: "127.0.0.1:5430".to_string(),
+            http_endpoint_addr: "127.0.0.1:9898".to_string(),
            superuser: "zenith_admin".to_string(),
            workdir: repo_dir,
            pg_distrib_dir: "".into(),
-            repository_format,
+            auth_type: AuthType::Trust,
+            auth_validation_public_key_path: None,
        };
        // Make a static copy of the config. This can never be free'd, but that's
        // OK in a test.
@@ -360,39 +276,18 @@ mod tests {

        let walredo_mgr = TestRedoManager {};

-        let repo: Box<dyn Repository + Sync + Send> = match conf.repository_format {
-            RepositoryFormat::Layered => {
-                Box::new(LayeredRepository::new(conf, Arc::new(walredo_mgr), tenantid))
-            }
-            RepositoryFormat::RocksDb => {
-                let obj_store = RocksObjectStore::create(conf, &tenantid)?;
-
-                Box::new(ObjectRepository::new(
-                    conf,
-                    Arc::new(obj_store),
-                    Arc::new(walredo_mgr),
-                    tenantid
-                ))
-            }
-        };
+        let repo = Box::new(LayeredRepository::new(
+            conf,
+            Arc::new(walredo_mgr),
+            tenantid,
+        ));

        Ok(repo)
    }

-    /// Test get_relsize() and truncation.
    #[test]
-    fn test_relsize_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_relsize_rocksdb", RepositoryFormat::RocksDb)?;
-        test_relsize(&*repo)
-    }
-
-    #[test]
-    fn test_relsize_layered() -> Result<()> {
-        let repo = get_test_repo("test_relsize_layered", RepositoryFormat::Layered)?;
-        test_relsize(&*repo)
-    }
-
-    fn test_relsize(repo: &dyn Repository) -> Result<()> {
+    fn test_relsize() -> Result<()> {
+        let repo = get_test_repo("test_relsize")?;
        // get_timeline() with non-existent timeline id should fail
        //repo.get_timeline("11223344556677881122334455667788");

@@ -401,21 +296,21 @@ mod tests {
        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;

        tline.init_valid_lsn(Lsn(1));
-        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"), true)?;
-        tline.put_page_image(TESTREL_A, 1, Lsn(4), TEST_IMG("foo blk 1 at 4"), true)?;
-        tline.put_page_image(TESTREL_A, 2, Lsn(5), TEST_IMG("foo blk 2 at 5"), true)?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"))?;
+        tline.put_page_image(TESTREL_A, 1, Lsn(4), TEST_IMG("foo blk 1 at 4"))?;
+        tline.put_page_image(TESTREL_A, 2, Lsn(5), TEST_IMG("foo blk 2 at 5"))?;

        tline.advance_last_valid_lsn(Lsn(5));

        // The relation was created at LSN 2, not visible at LSN 1 yet.
        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(1))?, false);
-        assert!(tline.get_rel_size(TESTREL_A, Lsn(1)).is_err());
+        assert!(tline.get_relish_size(TESTREL_A, Lsn(1))?.is_none());

        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(2))?, true);
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(2))?, 1);
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(5))?, 3);
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(2))?.unwrap(), 1);
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(5))?.unwrap(), 3);

        // Check page contents at each LSN
        assert_eq!(
@@ -455,7 +350,7 @@ mod tests {
        tline.advance_last_valid_lsn(Lsn(6));

        // Check reported size and contents after truncation
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(6))?, 2);
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(6))?.unwrap(), 2);
        assert_eq!(
            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(6))?,
            TEST_IMG("foo blk 0 at 3")
@@ -466,7 +361,7 @@ mod tests {
        );

        // should still see the truncated block with older LSN
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(5))?, 3);
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(5))?.unwrap(), 3);
        assert_eq!(
            tline.get_page_at_lsn(TESTREL_A, 2, Lsn(5))?,
            TEST_IMG("foo blk 2 at 5")
@@ -477,22 +372,9 @@ mod tests {

    /// Test get_relsize() and truncation with a file larger than 1 GB, so that it's
    /// split into multiple 1 GB segments in Postgres.
-    ///
-    /// This isn't very interesting with the RocksDb implementation, as we don't pay
-    /// any attention to Postgres segment boundaries there.
    #[test]
-    fn test_large_rel_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_large_rel_rocksdb", RepositoryFormat::RocksDb)?;
-        test_large_rel(&*repo)
-    }
-
-    #[test]
-    fn test_large_rel_layered() -> Result<()> {
-        let repo = get_test_repo("test_large_rel_layered", RepositoryFormat::Layered)?;
-        test_large_rel(&*repo)
-    }
-
-    fn test_large_rel(repo: &dyn Repository) -> Result<()> {
+    fn test_large_rel() -> Result<()> {
+        let repo = get_test_repo("test_large_rel")?;
        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;

@@ -502,12 +384,12 @@ mod tests {
        for blknum in 0..pg_constants::RELSEG_SIZE + 1 {
            let img = TEST_IMG(&format!("foo blk {} at {}", blknum, Lsn(lsn)));
            lsn += 1;
-            tline.put_page_image(TESTREL_A, blknum as u32, Lsn(lsn), img, true)?;
+            tline.put_page_image(TESTREL_A, blknum as u32, Lsn(lsn), img)?;
        }
        tline.advance_last_valid_lsn(Lsn(lsn));

        assert_eq!(
-            tline.get_rel_size(TESTREL_A, Lsn(lsn))?,
+            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
            pg_constants::RELSEG_SIZE + 1
        );

@@ -516,7 +398,7 @@ mod tests {
        tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE)?;
        tline.advance_last_valid_lsn(Lsn(lsn));
        assert_eq!(
-            tline.get_rel_size(TESTREL_A, Lsn(lsn))?,
+            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
            pg_constants::RELSEG_SIZE
        );

@@ -525,56 +407,49 @@ mod tests {
        tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE - 1)?;
        tline.advance_last_valid_lsn(Lsn(lsn));
        assert_eq!(
-            tline.get_rel_size(TESTREL_A, Lsn(lsn))?,
+            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
            pg_constants::RELSEG_SIZE - 1
        );

+        // Truncate to 1500, and then truncate all the way down to 0, one block at a time
+        // This tests the behavior at segment boundaries
+        let mut size: i32 = 3000;
+        while size >= 0 {
+            lsn += 1;
+            tline.put_truncation(TESTREL_A, Lsn(lsn), size as u32)?;
+            tline.advance_last_valid_lsn(Lsn(lsn));
+            assert_eq!(
+                tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
+                size as u32
+            );
+
+            size -= 1;
+        }
+
        Ok(())
    }

-    fn skip_nonrel_objects<'a>(
-        snapshot: Box<dyn History + 'a>,
-    ) -> Result<impl Iterator<Item = <dyn History as Iterator>::Item> + 'a> {
-        Ok(snapshot.skip_while(|r| match r {
-            Ok(m) => match m.tag {
-                ObjectTag::RelationMetadata(_) => false,
-                _ => true,
-            },
-            _ => panic!("Iteration error"),
-        }))
-    }
-
-    #[test]
-    fn test_branch_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_branch_rocksdb", RepositoryFormat::RocksDb)?;
-        test_branch(&*repo)
-    }
-
-    #[test]
-    fn test_branch_layered() -> Result<()> {
-        let repo = get_test_repo("test_branch_layered", RepositoryFormat::Layered)?;
-        test_branch(&*repo)
-    }
-
    ///
    /// Test branch creation
    ///
-    fn test_branch(repo: &dyn Repository) -> Result<()> {
+    #[test]
+    fn test_branch() -> Result<()> {
+        let repo = get_test_repo("test_branch")?;
        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;

        // Import initial dummy checkpoint record, otherwise the get_timeline() call
        // after branching fails below
-        tline.put_page_image(RelishTag::Checkpoint, 0, Lsn(1), ZERO_PAGE.clone(), false)?;
+        tline.put_page_image(RelishTag::Checkpoint, 0, Lsn(1), ZERO_PAGE.clone())?;

        // Create a relation on the timeline
        tline.init_valid_lsn(Lsn(1));
-        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("foo blk 0 at 4"), true)?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("foo blk 0 at 4"))?;

        // Create another relation
-        tline.put_page_image(TESTREL_B, 0, Lsn(2), TEST_IMG("foobar blk 0 at 2"), true)?;
+        tline.put_page_image(TESTREL_B, 0, Lsn(2), TEST_IMG("foobar blk 0 at 2"))?;

        tline.advance_last_valid_lsn(Lsn(4));

@@ -583,7 +458,7 @@ mod tests {
        repo.branch_timeline(timelineid, newtimelineid, Lsn(3))?;
        let newtline = repo.get_timeline(newtimelineid)?;

-        newtline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("bar blk 0 at 4"), true)?;
+        newtline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("bar blk 0 at 4"))?;
        newtline.advance_last_valid_lsn(Lsn(4));

        // Check page contents on both branches
@@ -602,93 +477,7 @@ mod tests {
            TEST_IMG("foobar blk 0 at 2")
        );

-        assert_eq!(newtline.get_rel_size(TESTREL_B, Lsn(4))?, 1);
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_history_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_history_rocksdb", RepositoryFormat::RocksDb)?;
-        test_history(&*repo)
-    }
-    #[test]
-    // TODO: This doesn't work with the layered storage, the functions needed for push/pull
-    // functionality haven't been implemented yet.
-    #[ignore]
-    fn test_history_layered() -> Result<()> {
-        let repo = get_test_repo("test_history_layered", RepositoryFormat::Layered)?;
-        test_history(&*repo)
-    }
-    fn test_history(repo: &dyn Repository) -> Result<()> {
-        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
-
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(0));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        // add a page and advance the last valid LSN
-        let rel = TESTREL_A;
-        tline.put_page_image(rel, 1, Lsn(1), TEST_IMG("blk 1 @ lsn 1"), true)?;
-        tline.advance_last_valid_lsn(Lsn(1));
-
-        let expected_page = Modification {
-            tag: ObjectTag::Buffer(rel, 1),
-            lsn: Lsn(1),
-            data: ObjectValue::ser(&ObjectValue::Page(PageEntry::Page(TEST_IMG(
-                "blk 1 @ lsn 1",
-            ))))?,
-        };
-        let expected_init_size = Modification {
-            tag: ObjectTag::RelationMetadata(rel),
-            lsn: Lsn(1),
-            data: ObjectValue::ser(&ObjectValue::RelationSize(RelationSizeEntry::Size(2)))?,
-        };
-        let expected_trunc_size = Modification {
-            tag: ObjectTag::RelationMetadata(rel),
-            lsn: Lsn(2),
-            data: ObjectValue::ser(&ObjectValue::RelationSize(RelationSizeEntry::Size(0)))?,
-        };
-
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(1));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(
-            Some(&expected_init_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        // truncate to zero, but don't advance the last valid LSN
-        tline.put_truncation(rel, Lsn(2), 0)?;
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(1));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(
-            Some(&expected_init_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        // advance the last valid LSN and the truncation should be observable
-        tline.advance_last_valid_lsn(Lsn(2));
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(2));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(
-            Some(&expected_init_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(
-            Some(&expected_trunc_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
-        assert_eq!(None, snapshot.next().transpose()?);
+        assert_eq!(newtline.get_relish_size(TESTREL_B, Lsn(4))?.unwrap(), 1);

        Ok(())
    }
--- a/pageserver/src/restore_local_repo.rs
+++ b/pageserver/src/restore_local_repo.rs
@@ -19,8 +19,8 @@ use crate::repository::*;
 use crate::waldecoder::*;
 use postgres_ffi::relfile_utils::*;
 use postgres_ffi::xlog_utils::*;
-use postgres_ffi::Oid;
 use postgres_ffi::{pg_constants, CheckPoint, ControlFileData};
+use postgres_ffi::{Oid, TransactionId};
 use zenith_utils::lsn::Lsn;

 const MAX_MBR_BLKNO: u32 =
@@ -50,7 +50,7 @@ pub fn import_timeline_from_postgres_datadir(
                    timeline.get_page_at_lsn_nowait(RelishTag::ControlFile, 0, lsn)?;
                let pg_control = ControlFileData::decode(&pg_control_bytes)?;
                let checkpoint_bytes = pg_control.checkPointCopy.encode();
-                timeline.put_page_image(RelishTag::Checkpoint, 0, lsn, checkpoint_bytes, false)?;
+                timeline.put_page_image(RelishTag::Checkpoint, 0, lsn, checkpoint_bytes)?;
            }
            Some("pg_filenode.map") => import_nonrel_file(
                timeline,
@@ -170,7 +170,7 @@ fn import_relfile(
                    forknum,
                };
                let tag = RelishTag::Relation(rel);
-                timeline.put_page_image(tag, blknum, lsn, Bytes::copy_from_slice(&buf), true)?;
+                timeline.put_page_image(tag, blknum, lsn, Bytes::copy_from_slice(&buf))?;
            }

            // TODO: UnexpectedEof is expected
@@ -211,7 +211,7 @@ fn import_nonrel_file(

    info!("importing non-rel file {}", path.display());

-    timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buffer[..]), false)?;
+    timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buffer[..]))?;
    Ok(())
 }

@@ -236,7 +236,6 @@ fn import_slru_file(timeline: &dyn Timeline, lsn: Lsn, slru: SlruKind, path: &Pa
                    rpageno,
                    lsn,
                    Bytes::copy_from_slice(&buf),
-                    true,
                )?;
            }

@@ -334,7 +333,7 @@ pub fn import_timeline_wal(walpath: &Path, timeline: &dyn Timeline, startpoint:

    info!("reached end of WAL at {}", last_lsn);
    let checkpoint_bytes = checkpoint.encode();
-    timeline.put_page_image(RelishTag::Checkpoint, 0, last_lsn, checkpoint_bytes, false)?;
+    timeline.put_page_image(RelishTag::Checkpoint, 0, last_lsn, checkpoint_bytes)?;

    timeline.advance_last_valid_lsn(last_lsn);
    timeline.checkpoint()?;
@@ -352,7 +351,9 @@ pub fn save_decoded_record(
    recdata: Bytes,
    lsn: Lsn,
 ) -> Result<()> {
-    checkpoint.update_next_xid(decoded.xl_xid);
+    if decoded.xl_xid > 0 {
+        checkpoint.update_next_xid(decoded.xl_xid);
+    }

    // Iterate through all the blocks that the record modifies, and
    // "put" a separate copy of the record for each block.
@@ -387,9 +388,30 @@ pub fn save_decoded_record(
        if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) == pg_constants::XLOG_DBASE_CREATE {
            let createdb = XlCreateDatabase::decode(&mut buf);
            save_xlog_dbase_create(timeline, lsn, &createdb)?;
-        } else {
-            // TODO
-            trace!("XLOG_DBASE_DROP is not handled yet");
+        } else if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK)
+            == pg_constants::XLOG_DBASE_DROP
+        {
+            let dropdb = XlDropDatabase::decode(&mut buf);
+
+            for tablespace_id in dropdb.tablespace_ids {
+                let rels = timeline.list_rels(tablespace_id, dropdb.db_id, lsn)?;
+                for rel in rels {
+                    timeline.put_unlink(RelishTag::Relation(rel), lsn)?;
+                }
+                trace!(
+                    "Unlink FileNodeMap {}, {} at lsn {}",
+                    tablespace_id,
+                    dropdb.db_id,
+                    lsn
+                );
+                timeline.put_unlink(
+                    RelishTag::FileNodeMap {
+                        spcnode: tablespace_id,
+                        dbnode: dropdb.db_id,
+                    },
+                    lsn,
+                )?;
+            }
        }
    } else if decoded.xl_rmid == pg_constants::RM_TBLSPC_ID {
        trace!("XLOG_TBLSPC_CREATE/DROP is not handled yet");
@@ -407,7 +429,6 @@ pub fn save_decoded_record(
                rpageno,
                lsn,
                ZERO_PAGE,
-                true,
            )?;
        } else {
            assert!(info == pg_constants::CLOG_TRUNCATE);
@@ -416,26 +437,38 @@ pub fn save_decoded_record(
        }
    } else if decoded.xl_rmid == pg_constants::RM_XACT_ID {
        let info = decoded.xl_info & pg_constants::XLOG_XACT_OPMASK;
-        if info == pg_constants::XLOG_XACT_COMMIT
-            || info == pg_constants::XLOG_XACT_COMMIT_PREPARED
-            || info == pg_constants::XLOG_XACT_ABORT
+        if info == pg_constants::XLOG_XACT_COMMIT || info == pg_constants::XLOG_XACT_ABORT {
+            let parsed_xact = XlXactParsedRecord::decode(&mut buf, decoded.xl_xid, decoded.xl_info);
+            save_xact_record(timeline, lsn, &parsed_xact, decoded)?;
+        } else if info == pg_constants::XLOG_XACT_COMMIT_PREPARED
            || info == pg_constants::XLOG_XACT_ABORT_PREPARED
        {
            let parsed_xact = XlXactParsedRecord::decode(&mut buf, decoded.xl_xid, decoded.xl_info);
            save_xact_record(timeline, lsn, &parsed_xact, decoded)?;
-        } else if info == pg_constants::XLOG_XACT_PREPARE {
-            let rec = WALRecord {
+            // Remove twophase file. see RemoveTwoPhaseFile() in postgres code
+            trace!(
+                "unlink twophaseFile for xid {} parsed_xact.xid {} here at {}",
+                decoded.xl_xid,
+                parsed_xact.xid,
+                lsn
+            );
+            timeline.put_unlink(
+                RelishTag::TwoPhase {
+                    xid: parsed_xact.xid,
+                },
                lsn,
-                will_init: true,
-                rec: recdata.clone(),
-                main_data_offset: decoded.main_data_offset as u32,
-            };
-            timeline.put_wal_record(
+            )?;
+        } else if info == pg_constants::XLOG_XACT_PREPARE {
+            let mut buf = decoded.record.clone();
+            buf.advance(decoded.main_data_offset);
+
+            timeline.put_page_image(
                RelishTag::TwoPhase {
                    xid: decoded.xl_xid,
                },
                0,
-                rec,
+                lsn,
+                Bytes::copy_from_slice(&buf[..]),
            )?;
        }
    } else if decoded.xl_rmid == pg_constants::RM_MULTIXACT_ID {
@@ -453,7 +486,6 @@ pub fn save_decoded_record(
                rpageno,
                lsn,
                ZERO_PAGE,
-                true,
            )?;
        } else if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE {
            let pageno = buf.get_u32_le();
@@ -467,7 +499,6 @@ pub fn save_decoded_record(
                rpageno,
                lsn,
                ZERO_PAGE,
-                true,
            )?;
        } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
            let xlrec = XlMultiXactCreate::decode(&mut buf);
@@ -478,7 +509,7 @@ pub fn save_decoded_record(
        }
    } else if decoded.xl_rmid == pg_constants::RM_RELMAP_ID {
        let xlrec = XlRelmapUpdate::decode(&mut buf);
-        save_relmap_record(timeline, lsn, &xlrec, decoded)?;
+        save_relmap_page(timeline, lsn, &xlrec, decoded)?;
    } else if decoded.xl_rmid == pg_constants::RM_XLOG_ID {
        let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
        if info == pg_constants::XLOG_NEXTOID {
@@ -532,7 +563,9 @@ fn save_xlog_dbase_create(timeline: &dyn Timeline, lsn: Lsn, rec: &XlCreateDatab
        assert_eq!(src_rel.spcnode, src_tablespace_id);
        assert_eq!(src_rel.dbnode, src_db_id);

-        let nblocks = timeline.get_rel_size(RelishTag::Relation(src_rel), req_lsn)?;
+        let nblocks = timeline
+            .get_relish_size(RelishTag::Relation(src_rel), req_lsn)?
+            .unwrap_or(0);
        let dst_rel = RelTag {
            spcnode: tablespace_id,
            dbnode: db_id,
@@ -547,7 +580,7 @@ fn save_xlog_dbase_create(timeline: &dyn Timeline, lsn: Lsn, rec: &XlCreateDatab

            debug!("copying block {} from {} to {}", blknum, src_rel, dst_rel);

-            timeline.put_page_image(RelishTag::Relation(dst_rel), blknum, lsn, content, true)?;
+            timeline.put_page_image(RelishTag::Relation(dst_rel), blknum, lsn, content)?;
            num_blocks_copied += 1;
        }

@@ -558,7 +591,10 @@ fn save_xlog_dbase_create(timeline: &dyn Timeline, lsn: Lsn, rec: &XlCreateDatab

        num_rels_copied += 1;
    }
+
    // Copy relfilemap
+    // TODO This implementation is very inefficient -
+    // it scans all non-rels only to find FileNodeMaps
    for tag in timeline.list_nonrels(req_lsn)? {
        match tag {
            RelishTag::FileNodeMap { spcnode, dbnode } => {
@@ -568,7 +604,7 @@ fn save_xlog_dbase_create(timeline: &dyn Timeline, lsn: Lsn, rec: &XlCreateDatab
                        spcnode: tablespace_id,
                        dbnode: db_id,
                    };
-                    timeline.put_page_image(new_tag, 0, lsn, img, false)?;
+                    timeline.put_page_image(new_tag, 0, lsn, img)?;
                    break;
                }
            }
@@ -703,42 +739,104 @@ fn save_xact_record(
    Ok(())
 }

+// See TransactionIdPrecedes in transam.c
+fn transaction_id_precedes(id1: TransactionId, id2: TransactionId) -> bool {
+    /*
+     * If either ID is a permanent XID then we can just do unsigned
+     * comparison.  If both are normal, do a modulo-2^32 comparison.
+     */
+
+    if !(id1 >= pg_constants::FIRST_NORMAL_TRANSACTION_ID)
+        || !(id2 >= pg_constants::FIRST_NORMAL_TRANSACTION_ID)
+    {
+        return id1 < id2;
+    }
+
+    let diff = id1.wrapping_sub(id2) as i32;
+    return diff < 0;
+}
+
+// See CLOGPagePrecedes in clog.c
+fn clogpage_precedes(page1: u32, page2: u32) -> bool {
+    let mut xid1 = page1 * pg_constants::CLOG_XACTS_PER_PAGE;
+    xid1 += pg_constants::FIRST_NORMAL_TRANSACTION_ID + 1;
+    let mut xid2 = page2 * pg_constants::CLOG_XACTS_PER_PAGE;
+    xid2 += pg_constants::FIRST_NORMAL_TRANSACTION_ID + 1;
+
+    return transaction_id_precedes(xid1, xid2)
+        && transaction_id_precedes(xid1, xid2 + pg_constants::CLOG_XACTS_PER_PAGE - 1);
+}
+
+// See SlruMayDeleteSegment() in slru.c
+fn slru_may_delete_clogsegment(segpage: u32, cutoff_page: u32) -> bool {
+    let seg_last_page = segpage + pg_constants::SLRU_PAGES_PER_SEGMENT - 1;
+
+    assert_eq!(segpage % pg_constants::SLRU_PAGES_PER_SEGMENT, 0);
+
+    return clogpage_precedes(segpage, cutoff_page)
+        && clogpage_precedes(seg_last_page, cutoff_page);
+}
+
 fn save_clog_truncate_record(
    checkpoint: &mut CheckPoint,
-    _timeline: &dyn Timeline,
-    _lsn: Lsn,
+    timeline: &dyn Timeline,
+    lsn: Lsn,
    xlrec: &XlClogTruncate,
 ) -> Result<()> {
-    trace!(
-        "RM_CLOG_ID truncate pageno {} oldestXid {} oldestXidDB {}",
-        xlrec.pageno,
-        xlrec.oldest_xid,
-        xlrec.oldest_xid_db
+    info!(
+        "RM_CLOG_ID truncate pageno {} oldestXid {} oldestXidDB {} lsn {}",
+        xlrec.pageno, xlrec.oldest_xid, xlrec.oldest_xid_db, lsn
    );

+    // Here we treat oldestXid and oldestXidDB
+    // differently from postgres redo routines.
+    // In postgres checkpoint.oldestXid lags behind xlrec.oldest_xid
+    // until checkpoint happens and updates the value.
+    // Here we can use the most recent value.
+    // It's just an optimization, though and can be deleted.
+    // TODO Figure out if there will be any issues with replica.
    checkpoint.oldestXid = xlrec.oldest_xid;
    checkpoint.oldestXidDB = xlrec.oldest_xid_db;

-    // FIXME: Handle XID wraparound! I just commented this out,
-    // because it was wrong in a dangerous way. But what this should
-    // now do is identify the CLOG segments in the repository that are
-    // older than the threshold in the WAL recor - taking XID
-    // wraparound into account like the corresponding PostgreSQL code
-    // does! - and call put_unlink() for the segments that are no
-    // longer needed.
+    // TODO Treat AdvanceOldestClogXid() or write a comment why we don't need it

-    /*
-        if let Some(ObjectTag::Clog(first_slru_tag)) =
-            timeline.get_next_tag(ObjectTag::Clog(SlruBufferTag { blknum: 0 }))?
-        {
-            for trunc_blknum in first_slru_tag.blknum..=pageno {
-                let tag = ObjectTag::Clog(SlruBufferTag {
-                    blknum: trunc_blknum,
-                });
-                timeline.put_slru_truncate(tag, lsn)?;
+    let latest_page_number = checkpoint.nextXid.value as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
+
+    // Now delete all segments containing pages between xlrec.pageno
+    // and latest_page_number.
+
+    // First, make an important safety check:
+    // the current endpoint page must not be eligible for removal.
+    // See SimpleLruTruncate() in slru.c
+    if clogpage_precedes(latest_page_number, xlrec.pageno) {
+        info!("could not truncate directory pg_xact apparent wraparound");
+        return Ok(());
+    }
+
+    // Iterate via SLRU CLOG segments and unlink segments that we're ready to truncate
+    // TODO This implementation is very inefficient -
+    // it scans all non-rels only to find Clog
+    //
+    // We cannot pass 'lsn' to the Timeline.list_nonrels(), or it
+    // will block waiting for the last valid LSN to advance up to
+    // it. So we use the previous record's LSN in the get calls
+    // instead.
+    let req_lsn = min(timeline.get_last_record_lsn(), lsn);
+    for obj in timeline.list_nonrels(req_lsn)? {
+        match obj {
+            RelishTag::Slru { slru, segno } => {
+                if slru == SlruKind::Clog {
+                    let segpage = segno * pg_constants::SLRU_PAGES_PER_SEGMENT;
+                    if slru_may_delete_clogsegment(segpage, xlrec.pageno) {
+                        timeline.put_unlink(RelishTag::Slru { slru, segno }, lsn)?;
+                        trace!("unlink CLOG segment {:>04X} at lsn {}", segno, lsn);
+                    }
+                }
            }
+            _ => {}
        }
-    */
+    }
+
    Ok(())
 }

@@ -811,6 +909,7 @@ fn save_multixact_create_record(
            acc
        }
    });
+
    checkpoint.update_next_xid(max_mbr_xid);
    Ok(())
 }
@@ -867,22 +966,23 @@ fn save_multixact_truncate_record(
    Ok(())
 }

-fn save_relmap_record(
+fn save_relmap_page(
    timeline: &dyn Timeline,
    lsn: Lsn,
    xlrec: &XlRelmapUpdate,
    decoded: &DecodedWALRecord,
 ) -> Result<()> {
-    let rec = WALRecord {
-        lsn,
-        will_init: true,
-        rec: decoded.record.clone(),
-        main_data_offset: decoded.main_data_offset as u32,
-    };
    let tag = RelishTag::FileNodeMap {
        spcnode: xlrec.tsid,
        dbnode: xlrec.dbid,
    };
-    timeline.put_wal_record(tag, 0, rec)?;
+
+    let mut buf = decoded.record.clone();
+    buf.advance(decoded.main_data_offset);
+    // skip xl_relmap_update
+    buf.advance(12);
+
+    timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buf[..]))?;
+
    Ok(())
 }
--- a/pageserver/src/rocksdb_storage.rs
+++ b/pageserver/src/rocksdb_storage.rs
@@ -1,474 +0,0 @@
-//!
-//! An implementation of the ObjectStore interface, backed by RocksDB
-//!
-use crate::object_key::*;
-use crate::object_store::ObjectStore;
-use crate::relish::*;
-use crate::PageServerConf;
-use crate::ZTenantId;
-use crate::ZTimelineId;
-use anyhow::{bail, Result};
-use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
-use std::sync::{Arc, Mutex};
-use zenith_utils::bin_ser::BeSer;
-use zenith_utils::lsn::Lsn;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-struct StorageKey {
-    obj_key: ObjectKey,
-    lsn: Lsn,
-}
-
-impl StorageKey {
-    /// The first key for a given timeline
-    fn timeline_start(timeline: ZTimelineId) -> Self {
-        Self {
-            obj_key: ObjectKey {
-                timeline,
-                tag: ObjectTag::TimelineMetadataTag,
-            },
-            lsn: Lsn(0),
-        }
-    }
-}
-
-///
-/// RocksDB very inefficiently delete random record. Instead of it we have to use merge
-/// filter, which allows to throw away records at LSM merge phase.
-/// Unfortunately, it is hard (if ever possible)  to determine whether version can be removed
-/// at merge time. Version ca be removed if:
-/// 1. It is above PITR horizon (we need to get current LSN and gc_horizon from config)
-/// 2. Page is reconstructed at horizon (all WAL records above horizon are applied and can be removed)
-///
-/// So we have GC process which reconstructs pages at horizon and mark deteriorated WAL record
-/// for deletion. To mark object for deletion we can either set some flag in object itself.
-/// But it is complicated with new object value format, because RocksDB storage knows nothing about
-/// this format. Also updating whole record just to set one bit seems to be inefficient in any case.
-/// This is why we keep keys of marked for deletion versions in HashSet in memory.
-/// When LSM merge filter found key in this map, it removes it from the set preventing memory overflow.
-///
-struct GarbageCollector {
-    garbage: Mutex<HashSet<Vec<u8>>>,
-}
-
-impl GarbageCollector {
-    fn new() -> GarbageCollector {
-        GarbageCollector {
-            garbage: Mutex::new(HashSet::new()),
-        }
-    }
-
-    /// Called by GC to mark version as delete
-    fn mark_for_deletion(&self, key: &[u8]) {
-        let mut garbage = self.garbage.lock().unwrap();
-        garbage.insert(key.to_vec());
-    }
-
-    /// Called by LSM merge filter. If it finds key in the set, then
-    /// it doesn't merge it and removes from this set.
-    fn was_deleted(&self, key: &[u8]) -> bool {
-        let key = key.to_vec();
-        let mut garbage = self.garbage.lock().unwrap();
-        garbage.remove(&key)
-    }
-}
-
-pub struct RocksObjectStore {
-    _conf: &'static PageServerConf,
-
-    // RocksDB handle
-    db: rocksdb::DB,
-    gc: Arc<GarbageCollector>,
-}
-
-impl ObjectStore for RocksObjectStore {
-    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>> {
-        let val = self.db.get(StorageKey::ser(&StorageKey {
-            obj_key: key.clone(),
-            lsn,
-        })?)?;
-        if let Some(val) = val {
-            Ok(val)
-        } else {
-            bail!("could not find page {:?}", key);
-        }
-    }
-
-    fn get_next_key(&self, key: &ObjectKey) -> Result<Option<ObjectKey>> {
-        let mut iter = self.db.raw_iterator();
-        let search_key = StorageKey {
-            obj_key: key.clone(),
-            lsn: Lsn(0),
-        };
-        iter.seek(search_key.ser()?);
-        if !iter.valid() {
-            Ok(None)
-        } else {
-            let key = StorageKey::des(iter.key().unwrap())?;
-            Ok(Some(key.obj_key.clone()))
-        }
-    }
-
-    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()> {
-        self.db.put(
-            StorageKey::ser(&StorageKey {
-                obj_key: key.clone(),
-                lsn,
-            })?,
-            value,
-        )?;
-        Ok(())
-    }
-
-    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()> {
-        self.gc.mark_for_deletion(&StorageKey::ser(&StorageKey {
-            obj_key: key.clone(),
-            lsn,
-        })?);
-        Ok(())
-    }
-
-    /// Iterate through page versions of given page, starting from the given LSN.
-    /// The versions are walked in descending LSN order.
-    fn object_versions<'a>(
-        &'a self,
-        key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>> {
-        let iter = RocksObjectVersionIter::new(&self.db, key, lsn)?;
-        Ok(Box::new(iter))
-    }
-
-    /// Iterate through all timeline objects
-    fn list_objects<'a>(
-        &'a self,
-        timeline: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = ObjectTag> + 'a>> {
-        let iter = RocksObjectIter::new(&self.db, timeline, lsn)?;
-        Ok(Box::new(iter))
-    }
-
-    /// Get a list of all distinct relations in given tablespace and database.
-    ///
-    /// TODO: This implementation is very inefficient, it scans
-    /// through all entries in the given database. In practice, this
-    /// is used for CREATE DATABASE, and usually the template database is small.
-    /// But if it's not, this will be slow.
-    fn list_rels(
-        &self,
-        timelineid: ZTimelineId,
-        spcnode: u32,
-        dbnode: u32,
-        lsn: Lsn,
-    ) -> Result<HashSet<RelTag>> {
-        // FIXME: This scans everything. Very slow
-
-        let mut rels: HashSet<RelTag> = HashSet::new();
-
-        let mut search_rel_tag = RelTag {
-            spcnode,
-            dbnode,
-            relnode: 0,
-            forknum: 0u8,
-        };
-        let mut iter = self.db.raw_iterator();
-        loop {
-            let search_key = StorageKey {
-                obj_key: ObjectKey {
-                    timeline: timelineid,
-                    tag: ObjectTag::RelationMetadata(RelishTag::Relation(search_rel_tag)),
-                },
-                lsn: Lsn(0),
-            };
-            iter.seek(search_key.ser()?);
-            if !iter.valid() {
-                break;
-            }
-            let key = StorageKey::des(iter.key().unwrap())?;
-
-            if let ObjectTag::RelationMetadata(RelishTag::Relation(rel_tag)) = key.obj_key.tag {
-                if spcnode != 0 && rel_tag.spcnode != spcnode
-                    || dbnode != 0 && rel_tag.dbnode != dbnode
-                {
-                    break;
-                }
-                if key.lsn <= lsn {
-                    // visible in this snapshot
-                    rels.insert(rel_tag);
-                }
-                search_rel_tag = rel_tag;
-                // skip to next relation
-                // FIXME: What if relnode is u32::MAX ?
-                search_rel_tag.relnode += 1;
-            } else {
-                // no more relation metadata entries
-                break;
-            }
-        }
-
-        Ok(rels)
-    }
-
-    /// Get a list of all distinct NON-relations in timeline
-    ///
-    /// TODO: This implementation is very inefficient, it scans
-    /// through all non-rel page versions in the system. In practice, this
-    /// is used when initializing a new compute node, and the non-rel files
-    /// are never very large nor change very frequently, so this will do for now.
-    fn list_nonrels(&self, timelineid: ZTimelineId, lsn: Lsn) -> Result<HashSet<RelishTag>> {
-        let mut rels: HashSet<RelishTag> = HashSet::new();
-
-        let search_key = StorageKey {
-            obj_key: ObjectKey {
-                timeline: timelineid,
-                tag: ObjectTag::Buffer(FIRST_NONREL_RELISH_TAG, 0),
-            },
-            lsn: Lsn(0),
-        };
-
-        let mut iter = self.db.raw_iterator();
-        iter.seek(search_key.ser()?);
-        while iter.valid() {
-            let key = StorageKey::des(iter.key().unwrap())?;
-
-            if key.obj_key.timeline != timelineid {
-                // reached end of this timeline in the store
-                break;
-            }
-
-            if let ObjectTag::Buffer(rel_tag, _blknum) = key.obj_key.tag {
-                if key.lsn <= lsn {
-                    // visible in this snapshot
-                    rels.insert(rel_tag);
-                }
-            }
-            // TODO: we could skip to next relation here like we do in list_rels(),
-            // but hopefully there are not that many SLRU segments or other non-rel
-            // entries for it to matter.
-            iter.next();
-        }
-
-        Ok(rels)
-    }
-
-    /// Iterate through versions of all objects in a timeline.
-    ///
-    /// Returns objects in increasing key-version order.
-    /// Returns all versions up to and including the specified LSN.
-    fn objects<'a>(
-        &'a self,
-        timeline: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = Result<(ObjectTag, Lsn, Vec<u8>)>> + 'a>> {
-        let start_key = StorageKey::timeline_start(timeline);
-        let start_key_bytes = StorageKey::ser(&start_key)?;
-        let iter = self.db.iterator(rocksdb::IteratorMode::From(
-            &start_key_bytes,
-            rocksdb::Direction::Forward,
-        ));
-
-        Ok(Box::new(RocksObjects {
-            iter,
-            timeline,
-            lsn,
-        }))
-    }
-
-    fn compact(&self) {
-        self.db.compact_range::<&[u8], &[u8]>(None, None);
-    }
-}
-
-impl RocksObjectStore {
-    /// Open a RocksDB database.
-    pub fn open(conf: &'static PageServerConf, tenantid: &ZTenantId) -> Result<RocksObjectStore> {
-        let opts = Self::get_rocksdb_opts();
-        let obj_store = Self::new(conf, opts, tenantid)?;
-        Ok(obj_store)
-    }
-
-    /// Create a new, empty RocksDB database.
-    pub fn create(conf: &'static PageServerConf, tenantid: &ZTenantId) -> Result<RocksObjectStore> {
-        let path = conf.tenant_path(&tenantid).join("rocksdb-storage");
-        std::fs::create_dir(&path)?;
-
-        let mut opts = Self::get_rocksdb_opts();
-        opts.create_if_missing(true);
-        opts.set_error_if_exists(true);
-        let obj_store = Self::new(conf, opts, tenantid)?;
-        Ok(obj_store)
-    }
-
-    fn new(
-        conf: &'static PageServerConf,
-        mut opts: rocksdb::Options,
-        tenantid: &ZTenantId,
-    ) -> Result<RocksObjectStore> {
-        let path = conf.tenant_path(&tenantid).join("rocksdb-storage");
-        let gc = Arc::new(GarbageCollector::new());
-        let gc_ref = gc.clone();
-        opts.set_compaction_filter("ttl", move |_level: u32, key: &[u8], _val: &[u8]| {
-            if gc_ref.was_deleted(key) {
-                rocksdb::compaction_filter::Decision::Remove
-            } else {
-                rocksdb::compaction_filter::Decision::Keep
-            }
-        });
-        let db = rocksdb::DB::open(&opts, &path)?;
-        let obj_store = RocksObjectStore {
-            _conf: conf,
-            db,
-            gc,
-        };
-        Ok(obj_store)
-    }
-
-    /// common options used by `open` and `create`
-    fn get_rocksdb_opts() -> rocksdb::Options {
-        let mut opts = rocksdb::Options::default();
-        opts.set_use_fsync(true);
-        opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
-        opts
-    }
-}
-
-///
-/// Iterator for `object_versions`. Returns all page versions of a given block, in
-/// reverse LSN order.
-///
-struct RocksObjectVersionIter<'a> {
-    obj_key: ObjectKey,
-    dbiter: rocksdb::DBRawIterator<'a>,
-    first_call: bool,
-}
-impl<'a> RocksObjectVersionIter<'a> {
-    fn new(
-        db: &'a rocksdb::DB,
-        obj_key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<RocksObjectVersionIter<'a>> {
-        let key = StorageKey {
-            obj_key: obj_key.clone(),
-            lsn,
-        };
-        let mut dbiter = db.raw_iterator();
-        dbiter.seek_for_prev(StorageKey::ser(&key)?); // locate last entry
-        Ok(RocksObjectVersionIter {
-            first_call: true,
-            obj_key: obj_key.clone(),
-            dbiter,
-        })
-    }
-}
-impl<'a> Iterator for RocksObjectVersionIter<'a> {
-    type Item = (Lsn, Vec<u8>);
-
-    fn next(&mut self) -> std::option::Option<Self::Item> {
-        if self.first_call {
-            self.first_call = false;
-        } else {
-            self.dbiter.prev(); // walk backwards
-        }
-
-        if !self.dbiter.valid() {
-            return None;
-        }
-        let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
-        if key.obj_key.tag != self.obj_key.tag {
-            return None;
-        }
-        let val = self.dbiter.value().unwrap();
-        let result = val.to_vec();
-
-        Some((key.lsn, result))
-    }
-}
-
-struct RocksObjects<'r> {
-    iter: rocksdb::DBIterator<'r>,
-    timeline: ZTimelineId,
-    lsn: Lsn,
-}
-
-impl<'r> Iterator for RocksObjects<'r> {
-    // TODO consider returning Box<[u8]>
-    type Item = Result<(ObjectTag, Lsn, Vec<u8>)>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.next_result().transpose()
-    }
-}
-
-impl<'r> RocksObjects<'r> {
-    fn next_result(&mut self) -> Result<Option<(ObjectTag, Lsn, Vec<u8>)>> {
-        for (key_bytes, v) in &mut self.iter {
-            let key = StorageKey::des(&key_bytes)?;
-
-            if key.obj_key.timeline != self.timeline {
-                return Ok(None);
-            }
-
-            if key.lsn > self.lsn {
-                // TODO can speed up by seeking iterator
-                continue;
-            }
-
-            return Ok(Some((key.obj_key.tag, key.lsn, v.to_vec())));
-        }
-
-        Ok(None)
-    }
-}
-
-///
-/// Iterator for `list_objects`. Returns all objects preceeding specified LSN
-///
-struct RocksObjectIter<'a> {
-    timeline: ZTimelineId,
-    key: StorageKey,
-    lsn: Lsn,
-    dbiter: rocksdb::DBRawIterator<'a>,
-}
-impl<'a> RocksObjectIter<'a> {
-    fn new(db: &'a rocksdb::DB, timeline: ZTimelineId, lsn: Lsn) -> Result<RocksObjectIter<'a>> {
-        let key = StorageKey {
-            obj_key: ObjectKey {
-                timeline,
-                tag: ObjectTag::FirstTag,
-            },
-            lsn: Lsn(0),
-        };
-        let dbiter = db.raw_iterator();
-        Ok(RocksObjectIter {
-            key,
-            timeline,
-            lsn,
-            dbiter,
-        })
-    }
-}
-impl<'a> Iterator for RocksObjectIter<'a> {
-    type Item = ObjectTag;
-
-    fn next(&mut self) -> std::option::Option<Self::Item> {
-        loop {
-            self.dbiter.seek(StorageKey::ser(&self.key).unwrap());
-            if !self.dbiter.valid() {
-                return None;
-            }
-            let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
-            if key.obj_key.timeline != self.timeline {
-                // End of this timeline
-                return None;
-            }
-            self.key = key.clone();
-            self.key.lsn = Lsn(u64::MAX); // next seek should skip all versions
-            if key.lsn <= self.lsn {
-                // visible in this snapshot
-                return Some(key.obj_key.tag);
-            }
-        }
-    }
-}
--- a/pageserver/src/tenant_mgr.rs
+++ b/pageserver/src/tenant_mgr.rs
@@ -3,11 +3,9 @@

 use crate::branches;
 use crate::layered_repository::LayeredRepository;
-use crate::object_repository::ObjectRepository;
 use crate::repository::Repository;
-use crate::rocksdb_storage::RocksObjectStore;
 use crate::walredo::PostgresRedoManager;
-use crate::{PageServerConf, RepositoryFormat, ZTenantId};
+use crate::PageServerConf;
 use anyhow::{anyhow, bail, Result};
 use lazy_static::lazy_static;
 use log::info;
@@ -15,6 +13,7 @@ use std::collections::HashMap;
 use std::fs;
 use std::str::FromStr;
 use std::sync::{Arc, Mutex};
+use zenith_utils::zid::ZTenantId;

 lazy_static! {
    pub static ref REPOSITORY: Mutex<HashMap<ZTenantId, Arc<dyn Repository>>> =
@@ -32,31 +31,12 @@ pub fn init(conf: &'static PageServerConf) {
        let walredo_mgr = PostgresRedoManager::new(conf, tenantid);

        // Set up an object repository, for actual data storage.
-        let repo: Arc<dyn Repository + Sync + Send> = match conf.repository_format {
-            RepositoryFormat::Layered => {
-				let repo = Arc::new(LayeredRepository::new(
-					conf,
-					Arc::new(walredo_mgr),
-					tenantid,
-				));
-				if conf.gc_horizon != 0 {
-					crate::layered_repository::LayeredRepository::launch_gc_thread(conf, repo.clone());
-				} else {
-					info!("Garbage collection is disabled");
-				}
-				repo
-			}
-            RepositoryFormat::RocksDb => {
-                let obj_store = RocksObjectStore::open(conf, &tenantid).unwrap();
-
-                Arc::new(ObjectRepository::new(
-                    conf,
-                    Arc::new(obj_store),
-                    Arc::new(walredo_mgr),
-                    tenantid,
-                ))
-            }
-        };
+        let repo = Arc::new(LayeredRepository::new(
+            conf,
+            Arc::new(walredo_mgr),
+            tenantid,
+        ));
+        LayeredRepository::launch_checkpointer_thread(conf, repo.clone());

        info!("initialized storage for tenant: {}", &tenantid);
        m.insert(tenantid, repo);
--- a/pageserver/src/waldecoder.rs
+++ b/pageserver/src/waldecoder.rs
@@ -1,8 +1,9 @@
 //!
 //! WAL decoder. For each WAL record, it decodes the record to figure out which data blocks
-//! the record affects, to add the records to the page cache.
+//! the record affects, so that they can be stored in repository.
 //!
 use bytes::{Buf, BufMut, Bytes, BytesMut};
+use crc32c::*;
 use log::*;
 use postgres_ffi::pg_constants;
 use postgres_ffi::xlog_utils::*;
@@ -166,6 +167,14 @@ impl WalStreamDecoder {
                    // XLOG_SWITCH records are special. If we see one, we need to skip
                    // to the next WAL segment.
                    let xlogrec = XLogRecord::from_bytes(&mut buf);
+                    let mut crc = crc32c_append(0, &recordbuf[XLOG_RECORD_CRC_OFFS + 4..]);
+                    crc = crc32c_append(crc, &recordbuf[0..XLOG_RECORD_CRC_OFFS]);
+                    if crc != xlogrec.xl_crc {
+                        return Err(WalDecodeError {
+                            msg: "WAL record crc mismatch".into(),
+                            lsn: self.lsn,
+                        });
+                    }
                    if xlogrec.is_xlog_switch_record() {
                        trace!("saw xlog switch record at {}", self.lsn);
                        self.padlen =
@@ -322,6 +331,31 @@ impl XlCreateDatabase {
    }
 }

+#[repr(C)]
+#[derive(Debug)]
+pub struct XlDropDatabase {
+    pub db_id: Oid,
+    pub n_tablespaces: Oid, /* number of tablespace IDs */
+    pub tablespace_ids: Vec<Oid>,
+}
+
+impl XlDropDatabase {
+    pub fn decode(buf: &mut Bytes) -> XlDropDatabase {
+        let mut rec = XlDropDatabase {
+            db_id: buf.get_u32_le(),
+            n_tablespaces: buf.get_u32_le(),
+            tablespace_ids: Vec::<Oid>::new(),
+        };
+
+        for _i in 0..rec.n_tablespaces {
+            let id = buf.get_u32_le();
+            rec.tablespace_ids.push(id);
+        }
+
+        rec
+    }
+}
+
 #[repr(C)]
 #[derive(Debug)]
 pub struct XlHeapInsert {
@@ -342,6 +376,7 @@ impl XlHeapInsert {
 #[derive(Debug)]
 pub struct XlHeapMultiInsert {
    pub flags: u8,
+    pub _padding: u8,
    pub ntuples: u16,
 }

@@ -349,6 +384,7 @@ impl XlHeapMultiInsert {
    pub fn decode(buf: &mut Bytes) -> XlHeapMultiInsert {
        XlHeapMultiInsert {
            flags: buf.get_u8(),
+            _padding: buf.get_u8(),
            ntuples: buf.get_u16_le(),
        }
    }
@@ -359,6 +395,8 @@ impl XlHeapMultiInsert {
 pub struct XlHeapDelete {
    pub xmax: TransactionId,
    pub offnum: OffsetNumber,
+    pub _padding: u16,
+    pub t_cid: u32,
    pub infobits_set: u8,
    pub flags: u8,
 }
@@ -368,6 +406,8 @@ impl XlHeapDelete {
        XlHeapDelete {
            xmax: buf.get_u32_le(),
            offnum: buf.get_u16_le(),
+            _padding: buf.get_u16_le(),
+            t_cid: buf.get_u32_le(),
            infobits_set: buf.get_u8(),
            flags: buf.get_u8(),
        }
@@ -381,6 +421,7 @@ pub struct XlHeapUpdate {
    pub old_offnum: OffsetNumber,
    pub old_infobits_set: u8,
    pub flags: u8,
+    pub t_cid: u32,
    pub new_xmax: TransactionId,
    pub new_offnum: OffsetNumber,
 }
@@ -392,6 +433,7 @@ impl XlHeapUpdate {
            old_offnum: buf.get_u16_le(),
            old_infobits_set: buf.get_u8(),
            flags: buf.get_u8(),
+            t_cid: buf.get_u32(),
            new_xmax: buf.get_u32_le(),
            new_offnum: buf.get_u16_le(),
        }
@@ -835,6 +877,7 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
        let blkno = blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32;
        if info == pg_constants::XLOG_HEAP_INSERT {
            let xlrec = XlHeapInsert::decode(&mut buf);
+            assert_eq!(0, buf.remaining());
            if (xlrec.flags
                & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
                    | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
@@ -850,6 +893,7 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
            }
        } else if info == pg_constants::XLOG_HEAP_DELETE {
            let xlrec = XlHeapDelete::decode(&mut buf);
+            assert_eq!(0, buf.remaining());
            if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 {
                let mut blk = DecodedBkpBlock::new();
                blk.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
@@ -863,6 +907,9 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
            || info == pg_constants::XLOG_HEAP_HOT_UPDATE
        {
            let xlrec = XlHeapUpdate::decode(&mut buf);
+            // the size of tuple data is inferred from the size of the record.
+            // we can't validate the remaining number of bytes without parsing
+            // the tuple data.
            if (xlrec.flags & pg_constants::XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0 {
                let mut blk = DecodedBkpBlock::new();
                blk.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
@@ -888,6 +935,15 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
        let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
        if info == pg_constants::XLOG_HEAP2_MULTI_INSERT {
            let xlrec = XlHeapMultiInsert::decode(&mut buf);
+
+            let offset_array_len = if xlogrec.xl_info & pg_constants::XLOG_HEAP_INIT_PAGE > 0 {
+                // the offsets array is omitted if XLOG_HEAP_INIT_PAGE is set
+                0
+            } else {
+                std::mem::size_of::<u16>() * xlrec.ntuples as usize
+            };
+            assert_eq!(offset_array_len, buf.remaining());
+
            if (xlrec.flags
                & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
                    | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
--- a/pageserver/src/walreceiver.rs
+++ b/pageserver/src/walreceiver.rs
@@ -1,16 +1,15 @@
 //!
-//! WAL receiver connects to the WAL safekeeper service,
-//! streams WAL, decodes records and saves them in page cache.
+//! WAL receiver connects to the WAL safekeeper service, streams WAL,
+//! decodes records and saves them in the repository for the correct
+//! timeline.
 //!
 //! We keep one WAL receiver active per timeline.

-use crate::page_cache;
 use crate::relish::*;
 use crate::restore_local_repo;
+use crate::tenant_mgr;
 use crate::waldecoder::*;
 use crate::PageServerConf;
-use crate::ZTenantId;
-use crate::ZTimelineId;
 use anyhow::{Error, Result};
 use lazy_static::lazy_static;
 use log::*;
@@ -32,6 +31,8 @@ use std::thread;
 use std::thread::sleep;
 use std::time::{Duration, SystemTime};
 use zenith_utils::lsn::Lsn;
+use zenith_utils::zid::ZTenantId;
+use zenith_utils::zid::ZTimelineId;

 //
 // We keep one WAL Receiver active per timeline.
@@ -131,12 +132,21 @@ fn walreceiver_main(
    let mut rclient = Client::connect(&connect_cfg, NoTls)?;
    info!("connected!");

+    // Immediately increment the gauge, then create a job to decrement it on thread exit.
+    // One of the pros of `defer!` is that this will *most probably*
+    // get called, even in presence of panics.
+    let gauge = crate::LIVE_CONNECTIONS_COUNT.with_label_values(&["wal_receiver"]);
+    gauge.inc();
+    scopeguard::defer! {
+        gauge.dec();
+    }
+
    let identify = identify_system(&mut rclient)?;
    info!("{:?}", identify);
    let end_of_wal = Lsn::from(u64::from(identify.xlogpos));
    let mut caught_up = false;

-    let repository = page_cache::get_repository_for_tenant(tenantid)?;
+    let repository = tenant_mgr::get_repository_for_tenant(tenantid)?;
    let timeline = repository.get_timeline(timelineid).unwrap();

    //
@@ -219,14 +229,13 @@ fn walreceiver_main(
                            0,
                            lsn,
                            new_checkpoint_bytes,
-                            false,
                        )?;
                    }
                }

-                // Update the last_valid LSN value in the page cache one more time. We updated
+                // Update the last_valid LSN value in the timeline one more time. We updated
                // it in the loop above, between each WAL record, but we might have received
-                // a partial record after the last completed record. Our page cache's value
+                // a partial record after the last completed record. Our timeline's value
                // better reflect that, because GetPage@LSN requests might also point in the
                // middle of a record, if the request LSN was taken from the server's current
                // flush ptr.
@@ -254,9 +263,7 @@ fn walreceiver_main(
                        tenantid,
                    )?;

-                    if newest_segno - oldest_segno >= 1 {
-                        timeline.checkpoint()?;
-
+                    if newest_segno - oldest_segno >= 10 {
                        // TODO: This is where we could remove WAL older than last_rec_lsn.
                        //remove_wal_files(timelineid, pg_constants::WAL_SEGMENT_SIZE, last_rec_lsn)?;
                    }
--- a/pageserver/src/walredo.rs
+++ b/pageserver/src/walredo.rs
@@ -1,6 +1,7 @@
 //!
 //! WAL redo. This service runs PostgreSQL in a special wal_redo mode
-//! to apply given WAL records over an old page image and return new page image.
+//! to apply given WAL records over an old page image and return new
+//! page image.
 //!
 //! We rely on Postgres to perform WAL redo for us. We launch a
 //! postgres process in special "wal redo" mode that's similar to
@@ -12,11 +13,14 @@
 //! See src/backend/tcop/zenith_wal_redo.c for the other side of
 //! this communication.
 //!
-//! TODO: Even though the postgres code runs in a separate process,
-//! it's not a secure sandbox.
+//! The Postgres process is assumed to be secure against malicious WAL
+//! records. It achieves it by dropping privileges before replaying
+//! any WAL records, so that even if an attacker hijacks the Postgres
+//! process, he cannot escape out of it.
 //!
 use byteorder::{ByteOrder, LittleEndian};
 use bytes::{Buf, BufMut, Bytes, BytesMut};
+use lazy_static::lazy_static;
 use log::*;
 use serde::{Deserialize, Serialize};
 use std::cell::RefCell;
@@ -26,7 +30,6 @@ use std::io::prelude::*;
 use std::io::Error;
 use std::path::PathBuf;
 use std::process::Stdio;
-use std::sync::mpsc;
 use std::sync::Mutex;
 use std::time::Duration;
 use std::time::Instant;
@@ -34,22 +37,22 @@ use tokio::io::AsyncBufReadExt;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::process::{ChildStdin, ChildStdout, Command};
 use tokio::time::timeout;
+use zenith_metrics::{register_histogram, register_int_counter, Histogram, IntCounter};
 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;
+use zenith_utils::zid::ZTenantId;

 use crate::relish::*;
 use crate::repository::WALRecord;
 use crate::waldecoder::XlXactParsedRecord;
 use crate::waldecoder::{MultiXactId, XlMultiXactCreate};
 use crate::PageServerConf;
-use crate::ZTenantId;
 use postgres_ffi::nonrelfile_utils::transaction_id_set_status;
 use postgres_ffi::pg_constants;
 use postgres_ffi::XLogRecord;

 ///
 /// `RelTag` + block number (`blknum`) gives us a unique id of the page in the cluster.
-/// This is used as a part of the key inside key-value storage (RocksDB currently).
 ///
 /// In Postgres `BufferTag` structure is used for exactly the same purpose.
 /// [See more related comments here](https://github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/buf_internals.h#L91).
@@ -102,22 +105,40 @@ impl crate::walredo::WalRedoManager for DummyRedoManager {

 static TIMEOUT: Duration = Duration::from_secs(20);

-///
-/// The implementation consists of two parts: PostgresRedoManager, and
-/// PostgresRedoManagerInternal. PostgresRedoManager is the public struct
-/// that can be used to send redo requests to the manager.
-/// PostgresRedoManagerInternal is used by the manager thread itself.
-///
-pub struct PostgresRedoManager {
-    request_tx: Mutex<mpsc::Sender<WalRedoRequest>>,
+// Metrics collected on WAL redo operations
+//
+// We collect the time spent in actual WAL redo ('redo'), and time waiting
+// for access to the postgres process ('wait') since there is only one for
+// each tenant.
+lazy_static! {
+    static ref WAL_REDO_TIME: Histogram =
+        register_histogram!("pageserver_wal_redo_time", "Time spent on WAL redo")
+            .expect("failed to define a metric");
+    static ref WAL_REDO_WAIT_TIME: Histogram = register_histogram!(
+        "pageserver_wal_redo_wait_time",
+        "Time spent waiting for access to the WAL redo process"
+    )
+    .expect("failed to define a metric");
+    static ref WAL_REDO_RECORD_COUNTER: IntCounter = register_int_counter!(
+        "pageserver_wal_records_replayed",
+        "Number of WAL records replayed"
+    )
+    .unwrap();
 }

-struct PostgresRedoManagerInternal {
+///
+/// This is the real implementation that uses a Postgres process to
+/// perform WAL replay. Only one thread can use the processs at a time,
+/// that is controlled by the Mutex. In the future, we might want to
+/// launch a pool of processes to allow concurrent replay of multiple
+/// records.
+///
+pub struct PostgresRedoManager {
+    tenantid: ZTenantId,
    conf: &'static PageServerConf,

-    request_rx: mpsc::Receiver<WalRedoRequest>,
-
-    tenantid: ZTenantId,
+    runtime: tokio::runtime::Runtime,
+    process: Mutex<Option<PostgresRedoProcess>>,
 }

 #[derive(Debug)]
@@ -128,8 +149,6 @@ struct WalRedoRequest {

    base_img: Option<Bytes>,
    records: Vec<WALRecord>,
-
-    response_channel: mpsc::Sender<Result<Bytes, WalRedoError>>,
 }

 /// An error happened in WAL redo
@@ -145,41 +164,6 @@ pub enum WalRedoError {
 ///
 /// Public interface of WAL redo manager
 ///
-impl PostgresRedoManager {
-    ///
-    /// Create a new PostgresRedoManager.
-    ///
-    /// This launches a new thread to handle the requests.
-    pub fn new(conf: &'static PageServerConf, tenantid: ZTenantId) -> PostgresRedoManager {
-        let (tx, rx) = mpsc::channel();
-
-        //
-        // Launch the WAL redo thread
-        //
-        // Get mutable references to the values that we need to pass to the
-        // thread.
-        let request_rx = rx;
-
-        // Currently, the join handle is not saved anywhere and we
-        // won't try restart the thread if it dies.
-        let _walredo_thread = std::thread::Builder::new()
-            .name("WAL redo thread".into())
-            .spawn(move || {
-                let mut internal = PostgresRedoManagerInternal {
-                    conf,
-                    request_rx,
-                    tenantid,
-                };
-                internal.wal_redo_main();
-            })
-            .unwrap();
-
-        PostgresRedoManager {
-            request_tx: Mutex::new(tx),
-        }
-    }
-}
-
 impl WalRedoManager for PostgresRedoManager {
    ///
    /// Request the WAL redo manager to apply some WAL records
@@ -195,8 +179,9 @@ impl WalRedoManager for PostgresRedoManager {
        base_img: Option<Bytes>,
        records: Vec<WALRecord>,
    ) -> Result<Bytes, WalRedoError> {
-        // Create a channel where to receive the response
-        let (tx, rx) = mpsc::channel::<Result<Bytes, WalRedoError>>();
+        let start_time;
+        let lock_time;
+        let end_time;

        let request = WalRedoRequest {
            rel,
@@ -204,17 +189,31 @@ impl WalRedoManager for PostgresRedoManager {
            lsn,
            base_img,
            records,
-            response_channel: tx,
        };

-        self.request_tx
-            .lock()
-            .unwrap()
-            .send(request)
-            .expect("could not send WAL redo request");
+        start_time = Instant::now();
+        let result = {
+            let mut process_guard = self.process.lock().unwrap();
+            lock_time = Instant::now();

-        rx.recv()
-            .expect("could not receive response to WAL redo request")
+            // launch the WAL redo process on first use
+            if process_guard.is_none() {
+                let p = self
+                    .runtime
+                    .block_on(PostgresRedoProcess::launch(self.conf, &self.tenantid))?;
+                *process_guard = Some(p);
+            }
+            let process = (*process_guard).as_ref().unwrap();
+
+            self.runtime
+                .block_on(self.handle_apply_request(&process, &request))
+        };
+        end_time = Instant::now();
+
+        WAL_REDO_WAIT_TIME.observe(lock_time.duration_since(start_time).as_secs_f64());
+        WAL_REDO_TIME.observe(end_time.duration_since(lock_time).as_secs_f64());
+
+        result
    }
 }

@@ -236,16 +235,11 @@ fn mx_offset_to_member_offset(xid: MultiXactId) -> usize {
            + (xid as u16 % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP) * 4) as usize
 }

-///
-/// WAL redo thread
-///
-impl PostgresRedoManagerInternal {
-    //
-    // Main entry point for the WAL applicator thread.
-    //
-    fn wal_redo_main(&mut self) {
-        info!("WAL redo thread started for tenant: {}", self.tenantid);
-
+impl PostgresRedoManager {
+    ///
+    /// Create a new PostgresRedoManager.
+    ///
+    pub fn new(conf: &'static PageServerConf, tenantid: ZTenantId) -> PostgresRedoManager {
        // We block on waiting for requests on the walredo request channel, but
        // use async I/O to communicate with the child process. Initialize the
        // runtime for the async part.
@@ -254,33 +248,12 @@ impl PostgresRedoManagerInternal {
            .build()
            .unwrap();

-        let process: PostgresRedoProcess;
-
-        info!(
-            "launching WAL redo postgres process for tenant: {}",
-            self.tenantid
-        );
-
-        process = runtime
-            .block_on(PostgresRedoProcess::launch(self.conf, &self.tenantid))
-            .unwrap();
-
-        // Loop forever, handling requests as they come.
-        loop {
-            let request = self
-                .request_rx
-                .recv()
-                .expect("WAL redo request channel was closed");
-
-            let result = runtime.block_on(self.handle_apply_request(&process, &request));
-            let result_ok = result.is_ok();
-
-            // Send the result to the requester
-            let _ = request.response_channel.send(result);
-
-            if !result_ok {
-                error!("wal-redo-postgres failed to apply request {:?}", request);
-            }
+        // The actual process is launched lazily, on first request.
+        PostgresRedoManager {
+            runtime,
+            tenantid,
+            conf,
+            process: Mutex::new(None),
        }
    }

@@ -308,6 +281,8 @@ impl PostgresRedoManagerInternal {
            let buf_tag = BufferTag { rel, blknum };
            apply_result = process.apply_wal_records(buf_tag, base_img, records).await;
        } else {
+            // Non-relational WAL records are handled here, with custom code that has the
+            // same effects as the corresponding Postgres WAL redo function.
            const ZERO_PAGE: [u8; 8192] = [0u8; 8192];
            let mut page = BytesMut::new();
            if let Some(fpi) = base_img {
@@ -321,6 +296,8 @@ impl PostgresRedoManagerInternal {
            for record in records {
                let mut buf = record.rec.clone();

+                WAL_REDO_RECORD_COUNTER.inc();
+
                // 1. Parse XLogRecord struct
                // FIXME: refactor to avoid code duplication.
                let xlogrec = XLogRecord::from_bytes(&mut buf);
@@ -335,7 +312,6 @@ impl PostgresRedoManagerInternal {

                if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
                    // Transaction manager stuff
-                    let info = xlogrec.xl_info & pg_constants::XLOG_XACT_OPMASK;
                    let rec_segno = match rel {
                        RelishTag::Slru { slru, segno } => {
                            if slru != SlruKind::Clog {
@@ -343,13 +319,6 @@ impl PostgresRedoManagerInternal {
                            }
                            segno
                        }
-                        RelishTag::TwoPhase { xid: _ } => {
-                            assert!(info == pg_constants::XLOG_XACT_PREPARE);
-                            trace!("Apply prepare {} record", xlogrec.xl_xid);
-                            page.clear();
-                            page.extend_from_slice(&buf[..]);
-                            continue;
-                        }
                        _ => panic!("Not valid XACT relish tag {:?}", rel),
                    };
                    let parsed_xact =
@@ -449,11 +418,6 @@ impl PostgresRedoManagerInternal {
                    } else {
                        panic!();
                    }
-                } else if xlogrec.xl_rmid == pg_constants::RM_RELMAP_ID {
-                    // Ralation map file has size 512 bytes
-                    page.clear();
-                    page.extend_from_slice(&buf[12..]); // skip xl_relmap_update
-                    assert!(page.len() == 512); // size of pg_filenode.map
                }
            }

@@ -485,6 +449,9 @@ impl PostgresRedoManagerInternal {
    }
 }

+///
+/// Handle to the Postgres WAL redo process
+///
 struct PostgresRedoProcess {
    stdin: RefCell<ChildStdin>,
    stdout: RefCell<ChildStdout>,
@@ -626,6 +593,8 @@ impl PostgresRedoProcess {
            for rec in records.iter() {
                let r = rec.clone();

+                WAL_REDO_RECORD_COUNTER.inc();
+
                stdin
                    .write_all(&build_apply_record_msg(r.lsn, r.rec))
                    .await?;
--- a/postgres_ffi/Cargo.toml
+++ b/postgres_ffi/Cargo.toml
@@ -19,7 +19,9 @@ lazy_static = "1.4"
 log = "0.4.14"
 memoffset = "0.6.2"
 thiserror = "1.0"
+serde = { version = "1.0", features = ["derive"] }
 workspace_hack = { path = "../workspace_hack" }
+zenith_utils = { path = "../zenith_utils" }

 [build-dependencies]
-bindgen = "0.57"
+bindgen = "0.59.1"
--- a/postgres_ffi/README
+++ b/postgres_ffi/README
@@ -13,11 +13,6 @@ in each major PostgreSQL version. Currently, this module is based on
 PostgreSQL v14, but in the future we will probably need a separate
 copy for each PostgreSQL version.

-To interact with the C structs, there is some unsafe code in this
-module. Do not copy-paste that to the rest of the codebase! Keep the
-amount of unsafe code to a minimum, and limited to this module only,
-and only where it's truly needed.
-
 TODO: Currently, there is also some code that deals with WAL records
 in pageserver/src/waldecoder.rs.  That should be moved into this
 module. The rest of the codebase should not have intimate knowledge of
--- a/postgres_ffi/build.rs
+++ b/postgres_ffi/build.rs
@@ -3,6 +3,44 @@ extern crate bindgen;
 use std::env;
 use std::path::PathBuf;

+use bindgen::callbacks::ParseCallbacks;
+
+#[derive(Debug)]
+struct PostgresFfiCallbacks;
+
+impl ParseCallbacks for PostgresFfiCallbacks {
+    fn include_file(&self, filename: &str) {
+        // This does the equivalent of passing bindgen::CargoCallbacks
+        // to the builder .parse_callbacks() method.
+        let cargo_callbacks = bindgen::CargoCallbacks;
+        cargo_callbacks.include_file(filename)
+    }
+
+    // Add any custom #[derive] attributes to the data structures that bindgen
+    // creates.
+    fn add_derives(&self, name: &str) -> Vec<String> {
+        // This is the list of data structures that we want to serialize/deserialize.
+        let serde_list = [
+            "XLogRecord",
+            "XLogPageHeaderData",
+            "XLogLongPageHeaderData",
+            "CheckPoint",
+            "FullTransactionId",
+            "ControlFileData",
+        ];
+
+        if serde_list.contains(&name) {
+            vec![
+                "Default".into(), // Default allows us to easily fill the padding fields with 0.
+                "Serialize".into(),
+                "Deserialize".into(),
+            ]
+        } else {
+            vec![]
+        }
+    }
+}
+
 fn main() {
    // Tell cargo to invalidate the built crate whenever the wrapper changes
    println!("cargo:rerun-if-changed=pg_control_ffi.h");
@@ -19,20 +57,23 @@ fn main() {
        // Tell cargo to invalidate the built crate whenever any of the
        // included header files changed.
        //
-        .parse_callbacks(Box::new(bindgen::CargoCallbacks))
+        .parse_callbacks(Box::new(PostgresFfiCallbacks))
        //
        // These are the types and constants that we want to generate bindings for
        //
-        .whitelist_type("ControlFileData")
-        .whitelist_type("CheckPoint")
-        .whitelist_type("FullTransactionId")
-        .whitelist_type("XLogRecord")
-        .whitelist_type("XLogPageHeaderData")
-        .whitelist_type("XLogLongPageHeaderData")
-        .whitelist_var("XLOG_PAGE_MAGIC")
-        .whitelist_var("PG_CONTROL_FILE_SIZE")
-        .whitelist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
-        .whitelist_type("DBState")
+        .allowlist_type("ControlFileData")
+        .allowlist_type("CheckPoint")
+        .allowlist_type("FullTransactionId")
+        .allowlist_type("XLogRecord")
+        .allowlist_type("XLogPageHeaderData")
+        .allowlist_type("XLogLongPageHeaderData")
+        .allowlist_var("XLOG_PAGE_MAGIC")
+        .allowlist_var("PG_CONTROL_FILE_SIZE")
+        .allowlist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
+        .allowlist_type("DBState")
+        // Because structs are used for serialization, tell bindgen to emit
+        // explicit padding fields.
+        .explicit_padding(true)
        //
        // Path the server include dir. It is in tmp_install/include/server, if you did
        // "configure --prefix=<path to tmp_install>". But if you used "configure --prefix=/",
--- a/postgres_ffi/src/controlfile_utils.rs
+++ b/postgres_ffi/src/controlfile_utils.rs
@@ -43,6 +43,8 @@ impl ControlFileData {
    /// Interpret a slice of bytes as a Postgres control file.
    ///
    pub fn decode(buf: &[u8]) -> Result<ControlFileData> {
+        use zenith_utils::bin_ser::LeSer;
+
        // Check that the slice has the expected size. The control file is
        // padded with zeros up to a 512 byte sector size, so accept a
        // larger size too, so that the caller can just the whole file
@@ -55,26 +57,8 @@ impl ControlFileData {
        let OFFSETOF_CRC = Self::pg_control_crc_offset();
        let expectedcrc = crc32c::crc32c(&buf[0..OFFSETOF_CRC]);

-        // Convert the slice into an array of the right size, and use `transmute` to
-        // reinterpret the raw bytes as a ControlFileData struct.
-        //
-        // NB: Ideally we would use 'zerocopy::FromBytes' for this, but bindgen doesn't
-        // derive FromBytes for us. The safety of this depends on the same constraints
-        // as for FromBytes, namely, all of its fields must implement FromBytes. That
-        // includes the primitive integer types, like `u8`, `u16`, `u32`, `u64` and their
-        // signed variants. But `bool` is not safe, because the contents of the high bits
-        // in a rust bool are undefined. In practice, PostgreSQL uses 1 to represent
-        // true and 0 for false, which is compatible with Rust bool, but let's try not to
-        // depend on it.
-        //
-        // FIXME: ControlFileData does contain 'bool's at the moment.
-        //
-        // See https://github.com/zenithdb/zenith/issues/207 for discussion on the safety
-        // of this.
-        let mut b: [u8; SIZEOF_CONTROLDATA] = [0u8; SIZEOF_CONTROLDATA];
-        b.copy_from_slice(&buf[0..SIZEOF_CONTROLDATA]);
-        let controlfile: ControlFileData =
-            unsafe { std::mem::transmute::<[u8; SIZEOF_CONTROLDATA], ControlFileData>(b) };
+        // Use serde to deserialize the input as a ControlFileData struct.
+        let controlfile = ControlFileData::des(buf)?;

        // Check the CRC
        if expectedcrc != controlfile.crc {
@@ -93,21 +77,10 @@ impl ControlFileData {
    ///
    /// The CRC is recomputed to match the contents of the fields.
    pub fn encode(&self) -> Bytes {
-        //
-        // Use `transmute` to reinterpret struct as raw bytes.
-        //
-        // FIXME: This triggers undefined behavior, because the contents
-        // of the padding bytes are undefined, and this leaks those
-        // undefined bytes into the resulting array. The Rust code won't
-        // care what's in those bytes, and PostgreSQL doesn't care
-        // either. HOWEVER, it is a potential security issue, because the
-        // bytes can contain arbitrary pieces of memory from the page
-        // server. In the worst case, that could be private keys or
-        // another tenant's data.
-        //
-        // See https://github.com/zenithdb/zenith/issues/207 for discussion.
-        let b: [u8; SIZEOF_CONTROLDATA] =
-            unsafe { std::mem::transmute::<ControlFileData, [u8; SIZEOF_CONTROLDATA]>(*self) };
+        use zenith_utils::bin_ser::LeSer;
+
+        // Serialize into a new buffer.
+        let b = self.ser().unwrap();

        // Recompute the CRC
        let OFFSETOF_CRC = Self::pg_control_crc_offset();
--- a/postgres_ffi/src/lib.rs
+++ b/postgres_ffi/src/lib.rs
@@ -4,6 +4,9 @@
 // suppress warnings on rust 1.53 due to bindgen unit tests.
 // https://github.com/rust-lang/rust-bindgen/issues/1651
 #![allow(deref_nullptr)]
+
+use serde::{Deserialize, Serialize};
+
 include!(concat!(env!("OUT_DIR"), "/bindings.rs"));

 pub mod controlfile_utils;
--- a/postgres_ffi/src/pg_constants.rs
+++ b/postgres_ffi/src/pg_constants.rs
@@ -46,7 +46,6 @@ pub const SIZE_OF_PAGE_HEADER: u16 = 24;
 pub const BITS_PER_HEAPBLOCK: u16 = 2;
 pub const HEAPBLOCKS_PER_PAGE: u16 = (BLCKSZ - SIZE_OF_PAGE_HEADER) * 8 / BITS_PER_HEAPBLOCK;

-pub const TRANSACTION_STATUS_IN_PROGRESS: u8 = 0x00;
 pub const TRANSACTION_STATUS_COMMITTED: u8 = 0x01;
 pub const TRANSACTION_STATUS_ABORTED: u8 = 0x02;
 pub const TRANSACTION_STATUS_SUB_COMMITTED: u8 = 0x03;
@@ -117,6 +116,7 @@ pub const XLOG_HEAP_INSERT: u8 = 0x00;
 pub const XLOG_HEAP_DELETE: u8 = 0x10;
 pub const XLOG_HEAP_UPDATE: u8 = 0x20;
 pub const XLOG_HEAP_HOT_UPDATE: u8 = 0x40;
+pub const XLOG_HEAP_INIT_PAGE: u8 = 0x80;
 pub const XLOG_HEAP2_VISIBLE: u8 = 0x40;
 pub const XLOG_HEAP2_MULTI_INSERT: u8 = 0x50;
 pub const XLH_INSERT_ALL_FROZEN_SET: u8 = (1 << 5) as u8;
--- a/postgres_ffi/src/xlog_utils.rs
+++ b/postgres_ffi/src/xlog_utils.rs
@@ -44,8 +44,6 @@ pub type TimeLineID = u32;
 pub type TimestampTz = i64;
 pub type XLogSegNo = u64;

-const XID_CHECKPOINT_INTERVAL: u32 = 1024;
-
 #[allow(non_snake_case)]
 pub fn XLogSegmentsPerXLogId(wal_segsz_bytes: usize) -> XLogSegNo {
    (0x100000000u64 / wal_segsz_bytes as u64) as XLogSegNo
@@ -121,6 +119,7 @@ fn find_end_of_wal_segment(
    let mut rec_hdr = [0u8; XLOG_RECORD_CRC_OFFS];

    while offs < wal_seg_size {
+        // we are at the beginning of the page; read it in
        if offs % XLOG_BLCKSZ == 0 {
            if let Ok(bytes_read) = file.read(&mut buf) {
                if bytes_read != buf.len() {
@@ -144,11 +143,12 @@ fn find_end_of_wal_segment(
            } else {
                offs += XLOG_SIZE_OF_XLOG_SHORT_PHD;
            }
+        // beginning of the next record
        } else if contlen == 0 {
            let page_offs = offs % XLOG_BLCKSZ;
            let xl_tot_len = LittleEndian::read_u32(&buf[page_offs..page_offs + 4]) as usize;
            if xl_tot_len == 0 {
-                break;
+                break; // zeros, reached the end
            }
            last_valid_rec_pos = offs;
            offs += 4;
@@ -156,12 +156,13 @@ fn find_end_of_wal_segment(
            contlen = xl_tot_len - 4;
            rec_hdr[0..4].copy_from_slice(&buf[page_offs..page_offs + 4]);
        } else {
-            let page_offs = offs % XLOG_BLCKSZ;
            // we're continuing a record, possibly from previous page.
+            let page_offs = offs % XLOG_BLCKSZ;
            let pageleft = XLOG_BLCKSZ - page_offs;

            // read the rest of the record, or as much as fits on this page.
            let n = min(contlen, pageleft);
+            // fill rec_hdr (header up to (but not including) xl_crc field)
            if rec_offs < XLOG_RECORD_CRC_OFFS {
                let len = min(XLOG_RECORD_CRC_OFFS - rec_offs, n);
                rec_hdr[rec_offs..rec_offs + len].copy_from_slice(&buf[page_offs..page_offs + len]);
@@ -185,6 +186,8 @@ fn find_end_of_wal_segment(
                crc = crc32c_append(crc, &rec_hdr);
                offs = (offs + 7) & !7; // pad on 8 bytes boundary */
                if crc == wal_crc {
+                    // record is valid, advance the result to its end (with
+                    // alignment to the next record taken into account)
                    last_valid_rec_pos = offs;
                } else {
                    info!(
@@ -201,6 +204,9 @@ fn find_end_of_wal_segment(

 ///
 /// Scan a directory that contains PostgreSQL WAL files, for the end of WAL.
+/// If precise, returns end LSN (next insertion point, basically);
+/// otherwise, start of the last segment.
+/// Returns (0, 0) if there is no WAL.
 ///
 pub fn find_end_of_wal(
    data_dir: &Path,
@@ -271,23 +277,13 @@ pub fn main() {

 impl XLogRecord {
    pub fn from_bytes(buf: &mut Bytes) -> XLogRecord {
-        XLogRecord {
-            xl_tot_len: buf.get_u32_le(),
-            xl_xid: buf.get_u32_le(),
-            xl_prev: buf.get_u64_le(),
-            xl_info: buf.get_u8(),
-            xl_rmid: buf.get_u8(),
-            xl_crc: {
-                buf.advance(2);
-                buf.get_u32_le()
-            },
-        }
+        use zenith_utils::bin_ser::LeSer;
+        XLogRecord::des_from(&mut buf.reader()).unwrap()
    }

    pub fn encode(&self) -> Bytes {
-        let b: [u8; XLOG_SIZE_OF_XLOG_RECORD];
-        b = unsafe { std::mem::transmute::<XLogRecord, [u8; XLOG_SIZE_OF_XLOG_RECORD]>(*self) };
-        Bytes::copy_from_slice(&b[..])
+        use zenith_utils::bin_ser::LeSer;
+        self.ser().unwrap().into()
    }

    // Is this record an XLOG_SWITCH record? They need some special processing,
@@ -298,34 +294,20 @@ impl XLogRecord {

 impl XLogPageHeaderData {
    pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogPageHeaderData {
-        let hdr: XLogPageHeaderData = XLogPageHeaderData {
-            xlp_magic: buf.get_u16_le(),
-            xlp_info: buf.get_u16_le(),
-            xlp_tli: buf.get_u32_le(),
-            xlp_pageaddr: buf.get_u64_le(),
-            xlp_rem_len: buf.get_u32_le(),
-        };
-        buf.get_u32_le(); //padding
-        hdr
+        use zenith_utils::bin_ser::LeSer;
+        XLogPageHeaderData::des_from(&mut buf.reader()).unwrap()
    }
 }

 impl XLogLongPageHeaderData {
    pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogLongPageHeaderData {
-        XLogLongPageHeaderData {
-            std: XLogPageHeaderData::from_bytes(buf),
-            xlp_sysid: buf.get_u64_le(),
-            xlp_seg_size: buf.get_u32_le(),
-            xlp_xlog_blcksz: buf.get_u32_le(),
-        }
+        use zenith_utils::bin_ser::LeSer;
+        XLogLongPageHeaderData::des_from(&mut buf.reader()).unwrap()
    }

    pub fn encode(&self) -> Bytes {
-        let b: [u8; XLOG_SIZE_OF_XLOG_LONG_PHD];
-        b = unsafe {
-            std::mem::transmute::<XLogLongPageHeaderData, [u8; XLOG_SIZE_OF_XLOG_LONG_PHD]>(*self)
-        };
-        Bytes::copy_from_slice(&b[..])
+        use zenith_utils::bin_ser::LeSer;
+        self.ser().unwrap().into()
    }
 }

@@ -333,26 +315,24 @@ pub const SIZEOF_CHECKPOINT: usize = std::mem::size_of::<CheckPoint>();

 impl CheckPoint {
    pub fn encode(&self) -> Bytes {
-        let b: [u8; SIZEOF_CHECKPOINT];
-        b = unsafe { std::mem::transmute::<CheckPoint, [u8; SIZEOF_CHECKPOINT]>(*self) };
-        Bytes::copy_from_slice(&b[..])
+        use zenith_utils::bin_ser::LeSer;
+        self.ser().unwrap().into()
    }

    pub fn decode(buf: &[u8]) -> Result<CheckPoint, anyhow::Error> {
-        let mut b = [0u8; SIZEOF_CHECKPOINT];
-        b.copy_from_slice(&buf[0..SIZEOF_CHECKPOINT]);
-        let checkpoint: CheckPoint;
-        checkpoint = unsafe { std::mem::transmute::<[u8; SIZEOF_CHECKPOINT], CheckPoint>(b) };
-        Ok(checkpoint)
+        use zenith_utils::bin_ser::LeSer;
+        Ok(CheckPoint::des(buf)?)
    }

    // Update next XID based on provided new_xid and stored epoch.
    // Next XID should be greater than new_xid.
    // Also take in account 32-bit wrap-around.
    pub fn update_next_xid(&mut self, xid: u32) {
-        let xid = xid.wrapping_add(XID_CHECKPOINT_INTERVAL - 1) & !(XID_CHECKPOINT_INTERVAL - 1);
        let full_xid = self.nextXid.value;
-        let new_xid = std::cmp::max(xid + 1, pg_constants::FIRST_NORMAL_TRANSACTION_ID);
+        let new_xid = std::cmp::max(
+            xid.wrapping_add(1),
+            pg_constants::FIRST_NORMAL_TRANSACTION_ID,
+        );
        let old_xid = full_xid as u32;
        if new_xid.wrapping_sub(old_xid) as i32 > 0 {
            let mut epoch = full_xid >> 32;
@@ -385,6 +365,7 @@ pub fn generate_wal_segment(pg_control: &ControlFileData) -> Bytes {
                xlp_tli: 1, // FIXME: always use Postgres timeline 1
                xlp_pageaddr: pg_control.checkPoint - XLOG_SIZE_OF_XLOG_LONG_PHD as u64,
                xlp_rem_len: 0,
+                ..Default::default() // Put 0 in padding fields.
            }
        },
        xlp_sysid: pg_control.system_identifier,
@@ -404,6 +385,7 @@ pub fn generate_wal_segment(pg_control: &ControlFileData) -> Bytes {
        xl_info: pg_constants::XLOG_CHECKPOINT_SHUTDOWN,
        xl_rmid: pg_constants::RM_XLOG_ID,
        xl_crc: 0,
+        ..Default::default() // Put 0 in padding fields.
    };

    let mut rec_shord_hdr_bytes = BytesMut::new();
@@ -428,3 +410,73 @@ pub fn generate_wal_segment(pg_control: &ControlFileData) -> Bytes {
    seg_buf.resize(pg_constants::WAL_SEGMENT_SIZE, 0);
    seg_buf.freeze()
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use regex::Regex;
+    use std::{env, process::Command, str::FromStr};
+    use zenith_utils::lsn::Lsn;
+
+    // Run find_end_of_wal against file in test_wal dir
+    // Ensure that it finds last record correctly
+    #[test]
+    pub fn test_find_end_of_wal() {
+        // 1. Run initdb to generate some WAL
+        let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
+        let data_dir = top_path.join("test_output/test_find_end_of_wal");
+        let initdb_path = top_path.join("tmp_install/bin/initdb");
+        let lib_path = top_path.join("tmp_install/lib");
+        if data_dir.exists() {
+            fs::remove_dir_all(&data_dir).unwrap();
+        }
+        println!("Using initdb from '{}'", initdb_path.display());
+        println!("Data directory '{}'", data_dir.display());
+        let initdb_output = Command::new(initdb_path)
+            .args(&["-D", data_dir.to_str().unwrap()])
+            .arg("--no-instructions")
+            .arg("--no-sync")
+            .env_clear()
+            .env("LD_LIBRARY_PATH", &lib_path)
+            .env("DYLD_LIBRARY_PATH", &lib_path)
+            .output()
+            .unwrap();
+        assert!(initdb_output.status.success());
+
+        // 2. Pick WAL generated by initdb
+        let wal_dir = data_dir.join("pg_wal");
+        let wal_seg_size = 16 * 1024 * 1024;
+
+        // 3. Check end_of_wal on non-partial WAL segment (we treat it as fully populated)
+        let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true);
+        let wal_end = Lsn(wal_end);
+        println!("wal_end={}, tli={}", wal_end, tli);
+        assert_eq!(wal_end, "0/2000000".parse::<Lsn>().unwrap());
+
+        // 4. Get the actual end of WAL by pg_waldump
+        let waldump_path = top_path.join("tmp_install/bin/pg_waldump");
+        let waldump_output = Command::new(waldump_path)
+            .arg(wal_dir.join("000000010000000000000001"))
+            .env_clear()
+            .env("LD_LIBRARY_PATH", &lib_path)
+            .env("DYLD_LIBRARY_PATH", &lib_path)
+            .output()
+            .unwrap();
+        let waldump_output = std::str::from_utf8(&waldump_output.stderr).unwrap();
+        println!("waldump_output = '{}'", &waldump_output);
+        let re = Regex::new(r"invalid record length at (.+):").unwrap();
+        let caps = re.captures(&waldump_output).unwrap();
+        let waldump_wal_end = Lsn::from_str(caps.get(1).unwrap().as_str()).unwrap();
+
+        // 5. Rename file to partial to actually find last valid lsn
+        fs::rename(
+            wal_dir.join("000000010000000000000001"),
+            wal_dir.join("000000010000000000000001.partial"),
+        )
+        .unwrap();
+        let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true);
+        let wal_end = Lsn(wal_end);
+        println!("wal_end={}, tli={}", wal_end, tli);
+        assert_eq!(wal_end, waldump_wal_end);
+    }
+}
--- a/pre-commit.py
+++ b/pre-commit.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+from typing import List
+import subprocess
+import sys
+import enum
+import argparse
+import os
+
+
+@enum.unique
+class Color(enum.Enum):
+    RED = "\033[0;31m"
+    GREEN = "\033[0;33m"
+    CYAN = "\033[0;36m"
+
+
+NC = "\033[0m"  # No Color
+
+
+def colorify(
+    s: str,
+    color: Color,
+    no_color: bool = False,
+):
+    if no_color:
+        return s
+    return f"{color.value}{s}{NC}"
+
+
+def rustfmt(fix_inplace: bool = False, no_color: bool = False) -> str:
+    cmd = "rustfmt --edition=2018"
+    if not fix_inplace:
+        cmd += " --check"
+    if no_color:
+        cmd += " --color=never"
+    return cmd
+
+
+def get_commit_files() -> List[str]:
+    files = subprocess.check_output(
+        "git diff --cached --name-only --diff-filter=ACM".split()
+    )
+    return files.decode().splitlines()
+
+
+def check(
+    name: str, suffix: str, cmd: str, changed_files: List[str], no_color: bool = False
+):
+    print(f"Checking: {name} ", end="")
+    applicable_files = list(
+        filter(lambda fname: fname.strip().endswith(suffix), changed_files)
+    )
+    if not applicable_files:
+        print(colorify("[NOT APPLICABLE]", Color.CYAN, no_color))
+        return
+
+    cmd = f'{cmd} {" ".join(applicable_files)}'
+    res = subprocess.run(cmd.split(), capture_output=True)
+    if res.returncode != 0:
+        print(colorify("[FAILED]", Color.RED, no_color))
+        print("Please inspect the output below and run make fmt to fix automatically\n")
+        print(res.stdout.decode())
+        exit(1)
+
+    print(colorify("[OK]", Color.GREEN, no_color))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--fix-inplace", action="store_true", help="apply fixes inplace"
+    )
+    parser.add_argument(
+        "--no-color", action="store_true", help="disable colored output", default=not sys.stdout.isatty()
+    )
+    args = parser.parse_args()
+
+    files = get_commit_files()
+    # we use rustfmt here because cargo fmt does not accept list of files
+    # it internally gathers project files and feeds them to rustfmt
+    # so because we want to check only files included in the commit we use rustfmt directly
+    check(
+        name="rustfmt",
+        suffix=".rs",
+        cmd=rustfmt(fix_inplace=args.fix_inplace, no_color=args.no_color),
+        changed_files=files,
+        no_color=args.no_color,
+    )
--- a/proxy/src/mgmt.rs
+++ b/proxy/src/mgmt.rs
@@ -7,7 +7,7 @@ use anyhow::bail;
 use bytes::Bytes;
 use serde::{Deserialize, Serialize};
 use zenith_utils::{
-    postgres_backend::{self, query_from_cstring, PostgresBackend},
+    postgres_backend::{self, query_from_cstring, AuthType, PostgresBackend},
    pq_proto::{BeMessage, SINGLE_COL_ROWDESC},
 };

@@ -34,7 +34,7 @@ pub fn thread_main(state: &'static ProxyState, listener: TcpListener) -> anyhow:

 pub fn mgmt_conn_main(state: &'static ProxyState, socket: TcpStream) -> anyhow::Result<()> {
    let mut conn_handler = MgmtHandler { state };
-    let mut pgbackend = PostgresBackend::new(socket, postgres_backend::AuthType::Trust)?;
+    let pgbackend = PostgresBackend::new(socket, AuthType::Trust)?;
    pgbackend.run(&mut conn_handler)
 }

--- a/proxy/src/proxy.rs
+++ b/proxy/src/proxy.rs
@@ -182,14 +182,14 @@ impl ProxyConnection {

        let hello_message = format!("☀️  Welcome to Zenith!

-To proceed with database creation open following link:
+To proceed with database creation, open the following link:

-    {}{}
+    {redirect_uri}{sess_id}

-It needed to be done once and we will send you '.pgpass' file which will allow you to access or create
+It needs to be done once and we will send you '.pgpass' file, which will allow you to access or create
 databases without opening the browser.

-", self.state.conf.redirect_uri,self.psql_session_id);
+", redirect_uri = self.state.conf.redirect_uri, sess_id = self.psql_session_id);

        self.pgb
            .write_message_noflush(&BeMessage::AuthenticationOk)?;
--- a/test_runner/Pipfile
+++ b/test_runner/Pipfile
@@ -7,6 +7,8 @@ name = "pypi"
 pytest = ">=6.0.0"
 psycopg2 = "*"
 typing-extensions = "*"
+pyjwt = {extras = ["crypto"], version = "*"}
+requests = "*"

 [dev-packages]
 yapf = "*"
--- a/test_runner/Pipfile.lock
+++ b/test_runner/Pipfile.lock
@@ -1,7 +1,7 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "4c20c05c20c50cf7e8f78ab461ab23841125345e63e00e2efa7661c165b6b364"
+            "sha256": "b666740289d9c82797e5c39b2a7f0074c865c9183ee878ce4fa5cda7928506ea"
        },
        "pipfile-spec": 6,
        "requires": {
@@ -24,13 +24,97 @@
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
            "version": "==21.2.0"
        },
-        "importlib-metadata": {
+        "certifi": {
            "hashes": [
-                "sha256:833b26fb89d5de469b24a390e9df088d4e52e4ba33b01dc5e0e4f41b81a16c00",
-                "sha256:b142cc1dd1342f31ff04bb7d022492b09920cb64fed867cd3ea6f80fe3ebd139"
+                "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee",
+                "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"
            ],
-            "markers": "python_version < '3.8'",
-            "version": "==4.5.0"
+            "version": "==2021.5.30"
+        },
+        "cffi": {
+            "hashes": [
+                "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d",
+                "sha256:0c0591bee64e438883b0c92a7bed78f6290d40bf02e54c5bf0978eaf36061771",
+                "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872",
+                "sha256:22b9c3c320171c108e903d61a3723b51e37aaa8c81255b5e7ce102775bd01e2c",
+                "sha256:26bb2549b72708c833f5abe62b756176022a7b9a7f689b571e74c8478ead51dc",
+                "sha256:33791e8a2dc2953f28b8d8d300dde42dd929ac28f974c4b4c6272cb2955cb762",
+                "sha256:3c8d896becff2fa653dc4438b54a5a25a971d1f4110b32bd3068db3722c80202",
+                "sha256:4373612d59c404baeb7cbd788a18b2b2a8331abcc84c3ba40051fcd18b17a4d5",
+                "sha256:487d63e1454627c8e47dd230025780e91869cfba4c753a74fda196a1f6ad6548",
+                "sha256:48916e459c54c4a70e52745639f1db524542140433599e13911b2f329834276a",
+                "sha256:4922cd707b25e623b902c86188aca466d3620892db76c0bdd7b99a3d5e61d35f",
+                "sha256:55af55e32ae468e9946f741a5d51f9896da6b9bf0bbdd326843fec05c730eb20",
+                "sha256:57e555a9feb4a8460415f1aac331a2dc833b1115284f7ded7278b54afc5bd218",
+                "sha256:5d4b68e216fc65e9fe4f524c177b54964af043dde734807586cf5435af84045c",
+                "sha256:64fda793737bc4037521d4899be780534b9aea552eb673b9833b01f945904c2e",
+                "sha256:6d6169cb3c6c2ad50db5b868db6491a790300ade1ed5d1da29289d73bbe40b56",
+                "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224",
+                "sha256:80b06212075346b5546b0417b9f2bf467fea3bfe7352f781ffc05a8ab24ba14a",
+                "sha256:818014c754cd3dba7229c0f5884396264d51ffb87ec86e927ef0be140bfdb0d2",
+                "sha256:8eb687582ed7cd8c4bdbff3df6c0da443eb89c3c72e6e5dcdd9c81729712791a",
+                "sha256:99f27fefe34c37ba9875f224a8f36e31d744d8083e00f520f133cab79ad5e819",
+                "sha256:9f3e33c28cd39d1b655ed1ba7247133b6f7fc16fa16887b120c0c670e35ce346",
+                "sha256:a8661b2ce9694ca01c529bfa204dbb144b275a31685a075ce123f12331be790b",
+                "sha256:a9da7010cec5a12193d1af9872a00888f396aba3dc79186604a09ea3ee7c029e",
+                "sha256:aedb15f0a5a5949ecb129a82b72b19df97bbbca024081ed2ef88bd5c0a610534",
+                "sha256:b315d709717a99f4b27b59b021e6207c64620790ca3e0bde636a6c7f14618abb",
+                "sha256:ba6f2b3f452e150945d58f4badd92310449876c4c954836cfb1803bdd7b422f0",
+                "sha256:c33d18eb6e6bc36f09d793c0dc58b0211fccc6ae5149b808da4a62660678b156",
+                "sha256:c9a875ce9d7fe32887784274dd533c57909b7b1dcadcc128a2ac21331a9765dd",
+                "sha256:c9e005e9bd57bc987764c32a1bee4364c44fdc11a3cc20a40b93b444984f2b87",
+                "sha256:d2ad4d668a5c0645d281dcd17aff2be3212bc109b33814bbb15c4939f44181cc",
+                "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195",
+                "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33",
+                "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f",
+                "sha256:e9dc245e3ac69c92ee4c167fbdd7428ec1956d4e754223124991ef29eb57a09d",
+                "sha256:eb687a11f0a7a1839719edd80f41e459cc5366857ecbed383ff376c4e3cc6afd",
+                "sha256:eb9e2a346c5238a30a746893f23a9535e700f8192a68c07c0258e7ece6ff3728",
+                "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7",
+                "sha256:f0010c6f9d1a4011e429109fda55a225921e3206e7f62a0c22a35344bfd13cca",
+                "sha256:f0c5d1acbfca6ebdd6b1e3eded8d261affb6ddcf2186205518f1428b8569bb99",
+                "sha256:f10afb1004f102c7868ebfe91c28f4a712227fe4cb24974350ace1f90e1febbf",
+                "sha256:f174135f5609428cc6e1b9090f9268f5c8935fddb1b25ccb8255a2d50de6789e",
+                "sha256:f3ebe6e73c319340830a9b2825d32eb6d8475c1dac020b4f0aa774ee3b898d1c",
+                "sha256:f627688813d0a4140153ff532537fbe4afea5a3dffce1f9deb7f91f848a832b5",
+                "sha256:fd4305f86f53dfd8cd3522269ed7fc34856a8ee3709a5e28b2836b2db9d4cd69"
+            ],
+            "version": "==1.14.6"
+        },
+        "charset-normalizer": {
+            "hashes": [
+                "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b",
+                "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==2.0.4"
+        },
+        "cryptography": {
+            "hashes": [
+                "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d",
+                "sha256:1e056c28420c072c5e3cb36e2b23ee55e260cb04eee08f702e0edfec3fb51959",
+                "sha256:240f5c21aef0b73f40bb9f78d2caff73186700bf1bc6b94285699aff98cc16c6",
+                "sha256:26965837447f9c82f1855e0bc8bc4fb910240b6e0d16a664bb722df3b5b06873",
+                "sha256:37340614f8a5d2fb9aeea67fd159bfe4f5f4ed535b1090ce8ec428b2f15a11f2",
+                "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713",
+                "sha256:3d8427734c781ea5f1b41d6589c293089704d4759e34597dce91014ac125aad1",
+                "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177",
+                "sha256:8e56e16617872b0957d1c9742a3f94b43533447fd78321514abbe7db216aa250",
+                "sha256:b01fd6f2737816cb1e08ed4807ae194404790eac7ad030b34f2ce72b332f5586",
+                "sha256:bf40af59ca2465b24e54f671b2de2c59257ddc4f7e5706dbd6930e26823668d3",
+                "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca",
+                "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d",
+                "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9"
+            ],
+            "version": "==3.4.7"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a",
+                "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==3.2"
        },
        "iniconfig": {
            "hashes": [
@@ -41,11 +125,11 @@
        },
        "packaging": {
            "hashes": [
-                "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5",
-                "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"
+                "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7",
+                "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"
            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==20.9"
+            "markers": "python_version >= '3.6'",
+            "version": "==21.0"
        },
        "pluggy": {
            "hashes": [
@@ -57,18 +141,18 @@
        },
        "psycopg2": {
            "hashes": [
-                "sha256:03a485bf71498870e38b535c0e6e7162d6ac06a91487edddc3b959894d65f79c",
-                "sha256:22102cfeb904898254f287b1a77360bf66c636858e7476593acd5267e5c24ff9",
-                "sha256:8f4c1800e57ad128d20b2e91d222ca238fffd316cef65be781361cdf35e37979",
-                "sha256:b12073fdf2002e828e5921be2c39ff9c6eab361c5c0bd6c529619fc23677accc",
-                "sha256:b6f47af317af8110818d255e693cfa80b7f1e435285be09778db7b66efd95789",
-                "sha256:d549db98fc0e6db41a2aa0d65f7434c4308a9f64012adb209b9e489f26fe87c6",
-                "sha256:e44e39a46af7c30566b7667fb27e701e652ab0a51e05c263a01d3ff0e223b765",
-                "sha256:e84c80be7a238d3c9c099b71f6890eaa35fc881146232cce888a88ab1bfb431e",
-                "sha256:f3d42bd42302293767b84206d9a446abc67ed4a133e4fe04dad8952de06c2091"
+                "sha256:079d97fc22de90da1d370c90583659a9f9a6ee4007355f5825e5f1c70dffc1fa",
+                "sha256:2087013c159a73e09713294a44d0c8008204d06326006b7f652bef5ace66eebb",
+                "sha256:2c992196719fadda59f72d44603ee1a2fdcc67de097eea38d41c7ad9ad246e62",
+                "sha256:7640e1e4d72444ef012e275e7b53204d7fab341fb22bc76057ede22fe6860b25",
+                "sha256:7f91312f065df517187134cce8e395ab37f5b601a42446bdc0f0d51773621854",
+                "sha256:830c8e8dddab6b6716a4bf73a09910c7954a92f40cf1d1e702fb93c8a919cc56",
+                "sha256:89409d369f4882c47f7ea20c42c5046879ce22c1e4ea20ef3b00a4dfc0a7f188",
+                "sha256:bf35a25f1aaa8a3781195595577fcbb59934856ee46b4f252f56ad12b8043bcf",
+                "sha256:de5303a6f1d0a7a34b9d40e4d3bef684ccc44a49bbe3eb85e3c0bffb4a131b7c"
            ],
            "index": "pypi",
-            "version": "==2.9"
+            "version": "==2.9.1"
        },
        "py": {
            "hashes": [
@@ -78,6 +162,25 @@
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
            "version": "==1.10.0"
        },
+        "pycparser": {
+            "hashes": [
+                "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",
+                "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.20"
+        },
+        "pyjwt": {
+            "extras": [
+                "crypto"
+            ],
+            "hashes": [
+                "sha256:934d73fbba91b0483d3857d1aff50e96b2a892384ee2c17417ed3203f173fca1",
+                "sha256:fba44e7898bbca160a2b2b501f492824fc8382485d3a6f11ba5d0c1937ce6130"
+            ],
+            "index": "pypi",
+            "version": "==2.1.0"
+        },
        "pyparsing": {
            "hashes": [
                "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
@@ -94,6 +197,14 @@
            "index": "pypi",
            "version": "==6.2.4"
        },
+        "requests": {
+            "hashes": [
+                "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24",
+                "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"
+            ],
+            "index": "pypi",
+            "version": "==2.26.0"
+        },
        "toml": {
            "hashes": [
                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
@@ -111,13 +222,13 @@
            "index": "pypi",
            "version": "==3.10.0.0"
        },
-        "zipp": {
+        "urllib3": {
            "hashes": [
-                "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76",
-                "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098"
+                "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4",
+                "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"
            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.4.1"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.26.6"
        }
    },
    "develop": {
@@ -129,14 +240,6 @@
            "index": "pypi",
            "version": "==3.9.2"
        },
-        "importlib-metadata": {
-            "hashes": [
-                "sha256:833b26fb89d5de469b24a390e9df088d4e52e4ba33b01dc5e0e4f41b81a16c00",
-                "sha256:b142cc1dd1342f31ff04bb7d022492b09920cb64fed867cd3ea6f80fe3ebd139"
-            ],
-            "markers": "python_version < '3.8'",
-            "version": "==4.5.0"
-        },
        "mccabe": {
            "hashes": [
                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
@@ -146,32 +249,32 @@
        },
        "mypy": {
            "hashes": [
-                "sha256:0190fb77e93ce971954c9e54ea61de2802065174e5e990c9d4c1d0f54fbeeca2",
-                "sha256:0756529da2dd4d53d26096b7969ce0a47997123261a5432b48cc6848a2cb0bd4",
-                "sha256:2f9fedc1f186697fda191e634ac1d02f03d4c260212ccb018fabbb6d4b03eee8",
-                "sha256:353aac2ce41ddeaf7599f1c73fed2b75750bef3b44b6ad12985a991bc002a0da",
-                "sha256:3f12705eabdd274b98f676e3e5a89f247ea86dc1af48a2d5a2b080abac4e1243",
-                "sha256:4efc67b9b3e2fddbe395700f91d5b8deb5980bfaaccb77b306310bd0b9e002eb",
-                "sha256:517e7528d1be7e187a5db7f0a3e479747307c1b897d9706b1c662014faba3116",
-                "sha256:68a098c104ae2b75e946b107ef69dd8398d54cb52ad57580dfb9fc78f7f997f0",
-                "sha256:746e0b0101b8efec34902810047f26a8c80e1efbb4fc554956d848c05ef85d76",
-                "sha256:8be7bbd091886bde9fcafed8dd089a766fa76eb223135fe5c9e9798f78023a20",
-                "sha256:9236c21194fde5df1b4d8ebc2ef2c1f2a5dc7f18bcbea54274937cae2e20a01c",
-                "sha256:9ef5355eaaf7a23ab157c21a44c614365238a7bdb3552ec3b80c393697d974e1",
-                "sha256:9f1d74eeb3f58c7bd3f3f92b8f63cb1678466a55e2c4612bf36909105d0724ab",
-                "sha256:a26d0e53e90815c765f91966442775cf03b8a7514a4e960de7b5320208b07269",
-                "sha256:ae94c31bb556ddb2310e4f913b706696ccbd43c62d3331cd3511caef466871d2",
-                "sha256:b5ba1f0d5f9087e03bf5958c28d421a03a4c1ad260bf81556195dffeccd979c4",
-                "sha256:b5dfcd22c6bab08dfeded8d5b44bdcb68c6f1ab261861e35c470b89074f78a70",
-                "sha256:cd01c599cf9f897b6b6c6b5d8b182557fb7d99326bcdf5d449a0fbbb4ccee4b9",
-                "sha256:e89880168c67cf4fde4506b80ee42f1537ad66ad366c101d388b3fd7d7ce2afd",
-                "sha256:ebe2bc9cb638475f5d39068d2dbe8ae1d605bb8d8d3ff281c695df1670ab3987",
-                "sha256:f89bfda7f0f66b789792ab64ce0978e4a991a0e4dd6197349d0767b0f1095b21",
-                "sha256:fc4d63da57ef0e8cd4ab45131f3fe5c286ce7dd7f032650d0fbc239c6190e167",
-                "sha256:fd634bc17b1e2d6ce716f0e43446d0d61cdadb1efcad5c56ca211c22b246ebc8"
+                "sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9",
+                "sha256:0aadfb2d3935988ec3815952e44058a3100499f5be5b28c34ac9d79f002a4a9a",
+                "sha256:119bed3832d961f3a880787bf621634ba042cb8dc850a7429f643508eeac97b9",
+                "sha256:1a85e280d4d217150ce8cb1a6dddffd14e753a4e0c3cf90baabb32cefa41b59e",
+                "sha256:3c4b8ca36877fc75339253721f69603a9c7fdb5d4d5a95a1a1b899d8b86a4de2",
+                "sha256:3e382b29f8e0ccf19a2df2b29a167591245df90c0b5a2542249873b5c1d78212",
+                "sha256:42c266ced41b65ed40a282c575705325fa7991af370036d3f134518336636f5b",
+                "sha256:53fd2eb27a8ee2892614370896956af2ff61254c275aaee4c230ae771cadd885",
+                "sha256:704098302473cb31a218f1775a873b376b30b4c18229421e9e9dc8916fd16150",
+                "sha256:7df1ead20c81371ccd6091fa3e2878559b5c4d4caadaf1a484cf88d93ca06703",
+                "sha256:866c41f28cee548475f146aa4d39a51cf3b6a84246969f3759cb3e9c742fc072",
+                "sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457",
+                "sha256:adaeee09bfde366d2c13fe6093a7df5df83c9a2ba98638c7d76b010694db760e",
+                "sha256:b6fb13123aeef4a3abbcfd7e71773ff3ff1526a7d3dc538f3929a49b42be03f0",
+                "sha256:b94e4b785e304a04ea0828759172a15add27088520dc7e49ceade7834275bedb",
+                "sha256:c0df2d30ed496a08de5daed2a9ea807d07c21ae0ab23acf541ab88c24b26ab97",
+                "sha256:c6c2602dffb74867498f86e6129fd52a2770c48b7cd3ece77ada4fa38f94eba8",
+                "sha256:ceb6e0a6e27fb364fb3853389607cf7eb3a126ad335790fa1e14ed02fba50811",
+                "sha256:d9dd839eb0dc1bbe866a288ba3c1afc33a202015d2ad83b31e875b5905a079b6",
+                "sha256:e4dab234478e3bd3ce83bac4193b2ecd9cf94e720ddd95ce69840273bf44f6de",
+                "sha256:ec4e0cd079db280b6bdabdc807047ff3e199f334050db5cbb91ba3e959a67504",
+                "sha256:ecd2c3fe726758037234c93df7e98deb257fd15c24c9180dacf1ef829da5f921",
+                "sha256:ef565033fa5a958e62796867b1df10c40263ea9ded87164d67572834e57a174d"
            ],
            "index": "pypi",
-            "version": "==0.902"
+            "version": "==0.910"
        },
        "mypy-extensions": {
            "hashes": [
@@ -204,42 +307,6 @@
            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
            "version": "==0.10.2"
        },
-        "typed-ast": {
-            "hashes": [
-                "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace",
-                "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff",
-                "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266",
-                "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528",
-                "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6",
-                "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808",
-                "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4",
-                "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363",
-                "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341",
-                "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04",
-                "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41",
-                "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e",
-                "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3",
-                "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899",
-                "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805",
-                "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c",
-                "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c",
-                "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39",
-                "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a",
-                "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3",
-                "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7",
-                "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f",
-                "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075",
-                "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0",
-                "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40",
-                "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428",
-                "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927",
-                "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3",
-                "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f",
-                "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"
-            ],
-            "markers": "python_version < '3.8'",
-            "version": "==1.4.3"
-        },
        "typing-extensions": {
            "hashes": [
                "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497",
@@ -256,14 +323,6 @@
            ],
            "index": "pypi",
            "version": "==0.31.0"
-        },
-        "zipp": {
-            "hashes": [
-                "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76",
-                "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.4.1"
        }
    }
 }
--- a/test_runner/README.md
+++ b/test_runner/README.md
@@ -7,7 +7,8 @@ Prerequisites:
 - Dependencies: install them via `pipenv install`. Note that Debian/Ubuntu
  packages are stale, as it commonly happens, so manual installation is not
  recommended.
-  Run `pipenv shell` to activate the venv.
+  Run `pipenv shell` to activate the venv or use `pipenv run` to run a single
+  command in the venv, e.g. `pipenv run pytests`.
 - Zenith and Postgres binaries
    - See the root README.md for build directions
    - Tests can be run from the git tree; or see the environment variables
@@ -19,10 +20,11 @@ Prerequisites:

 The tests are divided into a few batches, such that each batch takes roughly
 the same amount of time. The batches can be run in parallel, to minimize total
-runtime. Currently, there are only two batches:
+runtime.

- test_batch_pg_regress: Runs PostgreSQL regression tests
- test_others: All other tests
+- batch_pg_regress: Runs PostgreSQL regression tests
+- batch_wraparound: Runs xid wraparound tests.
+- batch_others: All other tests

 ### Running the tests

--- a/test_runner/batch_others/test_auth.py
+++ b/test_runner/batch_others/test_auth.py
@@ -0,0 +1,72 @@
+
+from contextlib import closing
+from uuid import uuid4
+import psycopg2
+from fixtures.zenith_fixtures import Postgres, ZenithCli, ZenithPageserver
+import pytest
+
+
+def test_pageserver_auth(pageserver_auth_enabled: ZenithPageserver):
+    ps = pageserver_auth_enabled
+
+    tenant_token = ps.auth_keys.generate_tenant_token(ps.initial_tenant)
+    invalid_tenant_token = ps.auth_keys.generate_tenant_token(uuid4().hex)
+    management_token = ps.auth_keys.generate_management_token()
+
+    # this does not invoke auth check and only decodes jwt and checks it for validity
+    # check both tokens
+    ps.safe_psql("status", password=tenant_token)
+    ps.safe_psql("status", password=management_token)
+
+    # tenant can create branches
+    ps.safe_psql(f"branch_create {ps.initial_tenant} new1 main", password=tenant_token)
+    # console can create branches for tenant
+    ps.safe_psql(f"branch_create {ps.initial_tenant} new2 main", password=management_token)
+
+    # fail to create branch using token with different tenantid
+    with pytest.raises(psycopg2.DatabaseError, match='Tenant id mismatch. Permission denied'):
+        ps.safe_psql(f"branch_create {ps.initial_tenant} new2 main", password=invalid_tenant_token)
+
+    # create tenant using management token
+    ps.safe_psql(f"tenant_create {uuid4().hex}", password=management_token)
+
+    # fail to create tenant using tenant token
+    with pytest.raises(psycopg2.DatabaseError, match='Attempt to access management api with tenant scope. Permission denied'):
+        ps.safe_psql(f"tenant_create {uuid4().hex}", password=tenant_token)
+
+
+@pytest.mark.parametrize('with_wal_acceptors', [False, True])
+def test_compute_auth_to_pageserver(
+    zenith_cli: ZenithCli,
+    wa_factory,
+    pageserver_auth_enabled: ZenithPageserver,
+    repo_dir: str,
+    with_wal_acceptors: bool,
+):
+    ps = pageserver_auth_enabled
+    # since we are in progress of refactoring protocols between compute safekeeper and page server
+    # use hardcoded management token in safekeeper
+    management_token = ps.auth_keys.generate_management_token()
+
+    branch = f"test_compute_auth_to_pageserver{with_wal_acceptors}"
+    zenith_cli.run(["branch", branch, "empty"])
+    if with_wal_acceptors:
+        wa_factory.start_n_new(3, management_token)
+
+    with Postgres(
+        zenith_cli=zenith_cli,
+        repo_dir=repo_dir,
+        tenant_id=ps.initial_tenant,
+        port=55432, # FIXME port distribution is hardcoded in tests and in cli
+    ).create_start(
+        branch,
+        wal_acceptors=wa_factory.get_connstrs() if with_wal_acceptors else None,
+    ) as pg:
+        with closing(pg.connect()) as conn:
+            with conn.cursor() as cur:
+                # we rely upon autocommit after each statement
+                # as waiting for acceptors happens there
+                cur.execute('CREATE TABLE t(key int primary key, value text)')
+                cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
+                cur.execute('SELECT sum(key) FROM t')
+                assert cur.fetchone() == (5000050000, )
--- a/test_runner/batch_others/test_clog_truncate.py
+++ b/test_runner/batch_others/test_clog_truncate.py
@@ -0,0 +1,72 @@
+import time
+import os
+
+from contextlib import closing
+
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+#
+# Test compute node start after clog truncation
+#
+def test_clog_truncate(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_clog_truncate", "empty"])
+
+    # set agressive autovacuum to make sure that truncation will happen
+    config = [
+        'autovacuum_max_workers=10', 'autovacuum_vacuum_threshold=0',
+        'autovacuum_vacuum_insert_threshold=0', 'autovacuum_vacuum_cost_delay=0',
+        'autovacuum_vacuum_cost_limit=10000', 'autovacuum_naptime =1s',
+        'autovacuum_freeze_max_age=100000'
+    ]
+
+    pg = postgres.create_start('test_clog_truncate', config_lines=config)
+    print('postgres is running on test_clog_truncate branch')
+
+    # Install extension containing function needed for test
+    pg.safe_psql('CREATE EXTENSION zenith_test_utils')
+
+    # Consume many xids to advance clog
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('select test_consume_xids(1000*1000*10);')
+            print('xids consumed')
+
+            # call a checkpoint to trigger TruncateSubtrans
+            cur.execute('CHECKPOINT;')
+
+            # ensure WAL flush
+            cur.execute('select txid_current()')
+            print(cur.fetchone())
+
+    # wait for autovacuum to truncate the pg_xact
+    # XXX Is it worth to add a timeout here?
+    pg_xact_0000_path = os.path.join(pg.pg_xact_dir_path(), '0000')
+    print("pg_xact_0000_path = " + pg_xact_0000_path)
+
+    while os.path.isfile(pg_xact_0000_path):
+        print("file exists. wait for truncation. " "pg_xact_0000_path = " + pg_xact_0000_path)
+        time.sleep(5)
+
+    # checkpoint to advance latest lsn
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('CHECKPOINT;')
+            cur.execute('select pg_current_wal_insert_lsn()')
+            lsn_after_truncation = cur.fetchone()[0]
+
+    # create new branch after clog truncation and start a compute node on it
+    print('create branch at lsn_after_truncation ' + lsn_after_truncation)
+    zenith_cli.run(
+        ["branch", "test_clog_truncate_new", "test_clog_truncate@" + lsn_after_truncation])
+
+    pg2 = postgres.create_start('test_clog_truncate_new')
+    print('postgres is running on test_clog_truncate_new branch')
+
+    # check that new node doesn't contain truncated segment
+    pg_xact_0000_path_new = os.path.join(pg2.pg_xact_dir_path(), '0000')
+    print("pg_xact_0000_path_new = " + pg_xact_0000_path_new)
+    assert os.path.isfile(pg_xact_0000_path_new) is False
--- a/test_runner/batch_others/test_createdb.py
+++ b/test_runner/batch_others/test_createdb.py
@@ -1,38 +0,0 @@
-from contextlib import closing
-from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory, ZenithCli
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-
-#
-# Test CREATE DATABASE when there have been relmapper changes
-#
-def test_createdb(
-    zenith_cli: ZenithCli,
-    pageserver: ZenithPageserver,
-    postgres: PostgresFactory,
-    pg_bin,
-):
-    zenith_cli.run(["branch", "test_createdb", "empty"])
-
-    pg = postgres.create_start('test_createdb')
-    print("postgres is running on 'test_createdb' branch")
-
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            # Cause a 'relmapper' change in the original branch
-            cur.execute('VACUUM FULL pg_class')
-
-            cur.execute('CREATE DATABASE foodb')
-
-            cur.execute('SELECT pg_current_wal_insert_lsn()')
-            lsn = cur.fetchone()[0]
-
-    # Create a branch
-    zenith_cli.run(["branch", "test_createdb2", "test_createdb@" + lsn])
-
-    pg2 = postgres.create_start('test_createdb2')
-
-    # Test that you can connect to the new database on both branches
-    for db in (pg, pg2):
-        db.connect(dbname='foodb').close()
--- a/test_runner/batch_others/test_createdropdb.py
+++ b/test_runner/batch_others/test_createdropdb.py
@@ -0,0 +1,96 @@
+import os
+import pathlib
+
+from contextlib import closing
+from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory, ZenithCli
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+#
+# Test CREATE DATABASE when there have been relmapper changes
+#
+def test_createdb(
+    zenith_cli: ZenithCli,
+    pageserver: ZenithPageserver,
+    postgres: PostgresFactory,
+    pg_bin,
+):
+    zenith_cli.run(["branch", "test_createdb", "empty"])
+
+    pg = postgres.create_start('test_createdb')
+    print("postgres is running on 'test_createdb' branch")
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            # Cause a 'relmapper' change in the original branch
+            cur.execute('VACUUM FULL pg_class')
+
+            cur.execute('CREATE DATABASE foodb')
+
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn = cur.fetchone()[0]
+
+    # Create a branch
+    zenith_cli.run(["branch", "test_createdb2", "test_createdb@" + lsn])
+
+    pg2 = postgres.create_start('test_createdb2')
+
+    # Test that you can connect to the new database on both branches
+    for db in (pg, pg2):
+        db.connect(dbname='foodb').close()
+
+#
+# Test DROP DATABASE
+#
+def test_dropdb(
+    zenith_cli: ZenithCli,
+    pageserver: ZenithPageserver,
+    postgres: PostgresFactory,
+    pg_bin,
+):
+    zenith_cli.run(["branch", "test_dropdb", "empty"])
+
+    pg = postgres.create_start('test_dropdb')
+    print("postgres is running on 'test_dropdb' branch")
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('CREATE DATABASE foodb')
+
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn_before_drop = cur.fetchone()[0]
+
+            cur.execute("SELECT oid FROM pg_database WHERE datname='foodb';")
+            dboid = cur.fetchone()[0]
+
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('DROP DATABASE foodb')
+
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn_after_drop = cur.fetchone()[0]
+
+
+    # Create two branches before and after database drop.
+    zenith_cli.run(["branch", "test_before_dropdb", "test_dropdb@" + lsn_before_drop])
+    pg_before = postgres.create_start('test_before_dropdb')
+
+    zenith_cli.run(["branch", "test_after_dropdb", "test_dropdb@" + lsn_after_drop])
+    pg_after = postgres.create_start('test_after_dropdb')
+
+    # Test that database exists on the branch before drop
+    pg_before.connect(dbname='foodb').close()
+
+    # Test that database subdir exists on the branch before drop
+    dbpath = pathlib.Path(pg_before.pgdata_dir) / 'base' / str(dboid)
+    print(dbpath)
+
+    assert os.path.isdir(dbpath) == True
+
+    # Test that database subdir doesn't exist on the branch after drop
+    dbpath = pathlib.Path(pg_after.pgdata_dir) / 'base' / str(dboid)
+    print(dbpath)
+
+    assert os.path.isdir(dbpath) == False
--- a/test_runner/batch_others/test_gc.py
+++ b/test_runner/batch_others/test_gc.py
@@ -1,100 +0,0 @@
-import pytest
-from contextlib import closing
-from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
-import psycopg2.extras
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-#
-# Test Garbage Collection of old page versions.
-#
-# This test is pretty tightly coupled with the current implementation of page version storage
-# and garbage collection in object_repository.rs.
-#
-@pytest.mark.skip(reason="This test only works with the RocksDB implementation")
-def test_gc(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
-    zenith_cli.run(["branch", "test_gc", "empty"])
-    pg = postgres.create_start('test_gc')
-
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            with closing(pageserver.connect()) as psconn:
-                with psconn.cursor(cursor_factory = psycopg2.extras.DictCursor) as pscur:
-
-                    # Get the timeline ID of our branch. We need it for the 'do_gc' command
-                    cur.execute("SHOW zenith.zenith_timeline")
-                    timeline = cur.fetchone()[0]
-
-                    # Create a test table
-                    cur.execute("CREATE TABLE foo(x integer)")
-
-                    # Run GC, to clear out any old page versions left behind in the catalogs by
-                    # the CREATE TABLE command. We want to have a clean slate with no garbage
-                    # before running the actual tests below, otherwise the counts won't match
-                    # what we expect.
-                    print("Running GC before test")
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    # remember the number of relations
-                    n_relations = row['n_relations']
-                    assert n_relations > 0
-
-                    # Insert a row. The first insert will also create a metadata entry for the
-                    # relation, with size == 1 block. Hence, bump up the expected relation count.
-                    n_relations += 1;
-                    print("Inserting one row and running GC")
-                    cur.execute("INSERT INTO foo VALUES (1)")
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    assert row['n_relations'] == n_relations
-                    assert row['dropped'] == 0
-                    assert row['truncated'] == 31
-                    assert row['deleted'] == 4
-
-                    # Insert two more rows and run GC.
-                    print("Inserting two more rows and running GC")
-                    cur.execute("INSERT INTO foo VALUES (2)")
-                    cur.execute("INSERT INTO foo VALUES (3)")
-
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    assert row['n_relations'] == n_relations
-                    assert row['dropped'] == 0
-                    assert row['truncated'] == 31
-                    assert row['deleted'] == 4
-
-                    # Insert one more row. It creates one more page version, but doesn't affect the
-                    # relation size.
-                    print("Inserting one more row")
-                    cur.execute("INSERT INTO foo VALUES (3)")
-
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    assert row['n_relations'] == n_relations
-                    assert row['dropped'] == 0
-                    assert row['truncated'] == 31
-                    assert row['deleted'] == 2
-
-                    # Run GC again, with no changes in the database. Should not remove anything.
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    assert row['n_relations'] == n_relations
-                    assert row['dropped'] == 0
-                    assert row['truncated'] == 31
-                    assert row['deleted'] == 0
-
-                    #
-                    # Test DROP TABLE checks that relation data and metadata was deleted by GC from object storage
-                    #
-                    cur.execute("DROP TABLE foo")
-
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    # Each relation fork is counted separately, hence 3.
-                    assert row['dropped'] == 3
--- a/test_runner/batch_others/test_pageserver_api.py
+++ b/test_runner/batch_others/test_pageserver_api.py
@@ -1,19 +1,20 @@
 import json
-import uuid
+from uuid import uuid4
 import pytest
 import psycopg2
-from fixtures.zenith_fixtures import ZenithPageserver
+import requests
+from fixtures.zenith_fixtures import ZenithPageserver, ZenithPageserverHttpClient

 pytest_plugins = ("fixtures.zenith_fixtures")


-def test_status(pageserver):
+def test_status_psql(pageserver):
    assert pageserver.safe_psql('status') == [
        ('hello world', ),
    ]


-def test_branch_list(pageserver: ZenithPageserver, zenith_cli):
+def test_branch_list_psql(pageserver: ZenithPageserver, zenith_cli):
    # Create a branch for us
    zenith_cli.run(["branch", "test_branch_list_main", "empty"])

@@ -52,7 +53,7 @@ def test_branch_list(pageserver: ZenithPageserver, zenith_cli):
    conn.close()


-def test_tenant_list(pageserver: ZenithPageserver, zenith_cli):
+def test_tenant_list_psql(pageserver: ZenithPageserver, zenith_cli):
    res = zenith_cli.run(["tenant", "list"])
    res.check_returncode()
    tenants = res.stdout.splitlines()
@@ -66,7 +67,7 @@ def test_tenant_list(pageserver: ZenithPageserver, zenith_cli):
        cur.execute(f'tenant_create {pageserver.initial_tenant}')

    # create one more tenant
-    tenant1 = uuid.uuid4().hex
+    tenant1 = uuid4().hex
    cur.execute(f'tenant_create {tenant1}')

    cur.execute('tenant_list')
@@ -74,3 +75,32 @@ def test_tenant_list(pageserver: ZenithPageserver, zenith_cli):
    # compare tenants list
    new_tenants = sorted(json.loads(cur.fetchone()[0]))
    assert sorted([pageserver.initial_tenant, tenant1]) == new_tenants
+
+
+def check_client(client: ZenithPageserverHttpClient, initial_tenant: str):
+    client.check_status()
+
+    # check initial tenant is there
+    assert initial_tenant in set(client.tenant_list())
+
+    # create new tenant and check it is also there
+    tenant_id = uuid4()
+    client.tenant_create(tenant_id)
+    assert tenant_id.hex in set(client.tenant_list())
+
+    # create branch
+    branch_name = uuid4().hex
+    client.branch_create(tenant_id, branch_name, "main")
+
+    # check it is there
+    assert branch_name in {b['name'] for b in client.branch_list(tenant_id)}
+
+
+def test_pageserver_http_api_client(pageserver: ZenithPageserver):
+    client = pageserver.http_client()
+    check_client(client, pageserver.initial_tenant)
+
+
+def test_pageserver_http_api_client_auth_enabled(pageserver_auth_enabled: ZenithPageserver):
+    client = pageserver_auth_enabled.http_client(auth_token=pageserver_auth_enabled.auth_keys.generate_management_token())
+    check_client(client, pageserver_auth_enabled.initial_tenant)
--- a/test_runner/batch_others/test_pageserver_restart.py
+++ b/test_runner/batch_others/test_pageserver_restart.py
@@ -0,0 +1,66 @@
+import pytest
+import random
+import time
+
+from contextlib import closing
+from multiprocessing import Process, Value
+from fixtures.zenith_fixtures import WalAcceptorFactory, ZenithPageserver, PostgresFactory
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+# Check that dead minority doesn't prevent the commits: execute insert n_inserts
+# times, with fault_probability chance of getting a wal acceptor down or up
+# along the way. 2 of 3 are always alive, so the work keeps going.
+def test_pageserver_restart(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory: WalAcceptorFactory):
+
+    # One safekeeper is enough for this test.
+    wa_factory.start_n_new(1)
+
+    zenith_cli.run(["branch", "test_pageserver_restart", "empty"])
+    pg = postgres.create_start('test_pageserver_restart',
+                               wal_acceptors=wa_factory.get_connstrs())
+
+    pg_conn = pg.connect()
+    cur = pg_conn.cursor()
+
+    # Create table, and insert some rows. Make it big enough that it doesn't fit in
+    # shared_buffers, otherwise the SELECT after restart will just return answer
+    # from shared_buffers without hitting the page server, which defeats the point
+    # of this test.
+    cur.execute('CREATE TABLE foo (t text)')
+    cur.execute('''
+        INSERT INTO foo
+            SELECT 'long string to consume some space' || g
+            FROM generate_series(1, 100000) g
+    ''')
+
+    # Verify that the table is larger than shared_buffers
+    cur.execute('''
+        select setting::int * pg_size_bytes(unit) as shared_buffers, pg_relation_size('foo') as tbl_ize
+        from pg_settings where name = 'shared_buffers'
+    ''')
+    row = cur.fetchone()
+    print("shared_buffers is {}, table size {}", row[0], row[1]);
+    assert int(row[0]) < int(row[1])
+
+    # Stop and restart pageserver. This is a more or less graceful shutdown, although
+    # the page server doesn't currently have a shutdown routine so there's no difference
+    # between stopping and crashing.
+    pageserver.stop();
+    pageserver.start();
+
+    # Stopping the pageserver breaks the connection from the postgres backend to
+    # the page server, and causes the next query on the connection to fail. Start a new
+    # postgres connection too, to avoid that error. (Ideally, the compute node would
+    # handle that and retry internally, without propagating the error to the user, but
+    # currently it doesn't...)
+    pg_conn = pg.connect()
+    cur = pg_conn.cursor()
+
+    cur.execute("SELECT count(*) FROM foo")
+    assert cur.fetchone() == (100000, )
+
+    # Stop the page server by force, and restart it
+    pageserver.stop();
+    pageserver.start();
+
--- a/test_runner/batch_others/test_restart_compute.py
+++ b/test_runner/batch_others/test_restart_compute.py
@@ -1,3 +1,5 @@
+import pytest
+
 from contextlib import closing
 from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory

@@ -7,10 +9,24 @@ pytest_plugins = ("fixtures.zenith_fixtures")
 #
 # Test restarting and recreating a postgres instance
 #
-def test_restart_compute(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+@pytest.mark.parametrize('with_wal_acceptors', [False, True])
+def test_restart_compute(
+        zenith_cli,
+        pageserver: ZenithPageserver,
+        postgres: PostgresFactory,
+        pg_bin,
+        wa_factory,
+        with_wal_acceptors: bool,
+    ):
+    wal_acceptor_connstrs = None
    zenith_cli.run(["branch", "test_restart_compute", "empty"])

-    pg = postgres.create_start('test_restart_compute')
+    if with_wal_acceptors:
+        wa_factory.start_n_new(3)
+        wal_acceptor_connstrs = wa_factory.get_connstrs()
+
+    pg = postgres.create_start('test_restart_compute',
+                               wal_acceptors=wal_acceptor_connstrs)
    print("postgres is running on 'test_restart_compute' branch")

    with closing(pg.connect()) as conn:
@@ -20,7 +36,8 @@ def test_restart_compute(zenith_cli, pageserver: ZenithPageserver, postgres: Pos
            cur.execute("INSERT INTO foo VALUES ('bar')")

    # Stop and restart the Postgres instance
-    pg.stop_and_destroy().create_start('test_restart_compute')
+    pg.stop_and_destroy().create_start('test_restart_compute',
+                                       wal_acceptors=wal_acceptor_connstrs)

    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
@@ -34,7 +51,8 @@ def test_restart_compute(zenith_cli, pageserver: ZenithPageserver, postgres: Pos
            assert cur.fetchone() == (2, )

    # Stop, and destroy the Postgres instance. Then recreate and restart it.
-    pg.stop_and_destroy().create_start('test_restart_compute')
+    pg.stop_and_destroy().create_start('test_restart_compute',
+                                       wal_acceptors=wal_acceptor_connstrs)

    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
--- a/test_runner/batch_others/test_snapfiles_gc.py
+++ b/test_runner/batch_others/test_snapfiles_gc.py
@@ -31,8 +31,8 @@ def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin):

                    # Create a test table
                    cur.execute("CREATE TABLE foo(x integer)")
+                    cur.execute("INSERT INTO foo VALUES (1)")

-                    print("Inserting two more rows and running GC")
                    cur.execute("select relfilenode from pg_class where oid = 'foo'::regclass");
                    row = cur.fetchone();
                    print("relfilenode is {}", row[0]);
@@ -46,6 +46,10 @@ def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin):
                    # kicks in and confuses our numbers.
                    cur.execute("VACUUM")

+                    # delete the row, to update the Visibility Map. We don't want the VM
+                    # update to confuse our numbers either.
+                    cur.execute("DELETE FROM foo")
+
                    print("Running GC before test")
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
@@ -54,16 +58,14 @@ def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin):
                    snapshot_relfiles_remain = row['snapshot_relfiles_total'] - row['snapshot_relfiles_removed']
                    assert snapshot_relfiles_remain > 0

-                    # Insert a row. The first insert will also create a metadata entry for the
-                    # relation, with size == 1 block. Hence, bump up the expected relation count.
-                    snapshot_relfiles_remain += 1;
+                    # Insert a row.
                    print("Inserting one row and running GC")
                    cur.execute("INSERT INTO foo VALUES (1)")
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print_gc_result(row);
-                    assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain
-                    assert row['snapshot_relfiles_removed'] == 0
+                    assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain + 1
+                    assert row['snapshot_relfiles_removed'] == 1
                    assert row['snapshot_relfiles_dropped'] == 0

                    # Insert two more rows and run GC.
--- a/test_runner/batch_others/test_twophase.py
+++ b/test_runner/batch_others/test_twophase.py
@@ -1,3 +1,5 @@
+import os
+
 from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver


@@ -28,24 +30,59 @@ def test_twophase(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFa
    cur.execute("INSERT INTO foo VALUES ('two')")
    cur.execute("PREPARE TRANSACTION 'insert_two'")

+    # Prepare a transaction that will insert a row
+    cur.execute('BEGIN')
+    cur.execute("INSERT INTO foo VALUES ('three')")
+    cur.execute("PREPARE TRANSACTION 'insert_three'")
+
+    # Prepare another transaction that will insert a row
+    cur.execute('BEGIN')
+    cur.execute("INSERT INTO foo VALUES ('four')")
+    cur.execute("PREPARE TRANSACTION 'insert_four'")
+
+    # On checkpoint state data copied to files in
+    # pg_twophase directory and fsynced
+    cur.execute('CHECKPOINT')
+
+    twophase_files = os.listdir(pg.pg_twophase_dir_path())
+    print(twophase_files)
+    assert len(twophase_files) == 4
+
+    cur.execute("COMMIT PREPARED 'insert_three'")
+    cur.execute("ROLLBACK PREPARED 'insert_four'")
+    cur.execute('CHECKPOINT')
+
+    twophase_files = os.listdir(pg.pg_twophase_dir_path())
+    print(twophase_files)
+    assert len(twophase_files) == 2
+
    # Create a branch with the transaction in prepared state
    zenith_cli.run(["branch", "test_twophase_prepared", "test_twophase"])

-    pg2 = postgres.create_start(
+    # Create compute node, but don't start.
+    # We want to observe pgdata before postgres starts
+    pg2 = postgres.create(
        'test_twophase_prepared',
        config_lines=['max_prepared_transactions=5'],
    )
+
+    # Check that we restored only needed twophase files
+    twophase_files2 = os.listdir(pg2.pg_twophase_dir_path())
+    print(twophase_files2)
+    assert twophase_files2.sort() == twophase_files.sort()
+
+    pg2 = pg2.start()
    conn2 = pg2.connect()
    cur2 = conn2.cursor()

-    # On the new branch, commit one of the prepared transactions, abort the other one.
+    # On the new branch, commit one of the prepared transactions,
+    # abort the other one.
    cur2.execute("COMMIT PREPARED 'insert_one'")
    cur2.execute("ROLLBACK PREPARED 'insert_two'")

    cur2.execute('SELECT * FROM foo')
-    assert cur2.fetchall() == [('one', )]
+    assert cur2.fetchall() == [('one',), ('three',)]

-    # Neither insert is visible on the original branch, the transactions are still
-    # in prepared state there.
+    # Only one committed insert is visible on the original branch
    cur.execute('SELECT * FROM foo')
-    assert cur.fetchall() == []
+    assert cur.fetchall() == [('three',)]
--- a/test_runner/batch_others/test_vm_bits.py
+++ b/test_runner/batch_others/test_vm_bits.py
@@ -0,0 +1,79 @@
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+#
+# Test that the VM bit is cleared correctly at a HEAP_DELETE and
+# HEAP_UPDATE record.
+#
+def test_vm_bit_clear(pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin, zenith_cli, base_dir):
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_vm_bit_clear", "empty"])
+    pg = postgres.create_start('test_vm_bit_clear')
+
+    print("postgres is running on 'test_vm_bit_clear' branch")
+    pg_conn = pg.connect()
+    cur = pg_conn.cursor()
+
+    # Install extension containing function needed for test
+    cur.execute('CREATE EXTENSION zenith_test_utils')
+
+    # Create a test table and freeze it to set the VM bit.
+    cur.execute('CREATE TABLE vmtest_delete (id integer PRIMARY KEY)')
+    cur.execute('INSERT INTO vmtest_delete VALUES (1)')
+    cur.execute('VACUUM FREEZE vmtest_delete')
+
+    cur.execute('CREATE TABLE vmtest_update (id integer PRIMARY KEY)')
+    cur.execute('INSERT INTO vmtest_update SELECT g FROM generate_series(1, 1000) g')
+    cur.execute('VACUUM FREEZE vmtest_update')
+
+    # DELETE and UDPATE the rows.
+    cur.execute('DELETE FROM vmtest_delete WHERE id = 1')
+    cur.execute('UPDATE vmtest_update SET id = 5000 WHERE id = 1')
+
+    # Branch at this point, to test that later
+    zenith_cli.run(["branch", "test_vm_bit_clear_new", "test_vm_bit_clear"])
+
+    # Clear the buffer cache, to force the VM page to be re-fetched from
+    # the page server
+    cur.execute('SELECT clear_buffer_cache()')
+
+    # Check that an index-only scan doesn't see the deleted row. If the
+    # clearing of the VM bit was not replayed correctly, this would incorrectly
+    # return deleted row.
+    cur.execute('''
+    set enable_seqscan=off;
+    set enable_indexscan=on;
+    set enable_bitmapscan=off;
+    ''')
+
+    cur.execute('SELECT * FROM vmtest_delete WHERE id = 1')
+    assert(cur.fetchall() == []);
+    cur.execute('SELECT * FROM vmtest_update WHERE id = 1')
+    assert(cur.fetchall() == []);
+
+    cur.close()
+
+
+    # Check the same thing on the branch that we created right after the DELETE
+    #
+    # As of this writing, the code in smgrwrite() creates a full-page image whenever
+    # a dirty VM page is evicted. If the VM bit was not correctly cleared by the
+    # earlier WAL record, the full-page image hides the problem. Starting a new
+    # server at the right point-in-time avoids that full-page image.
+    pg_new = postgres.create_start('test_vm_bit_clear_new')
+
+    print("postgres is running on 'test_vm_bit_clear_new' branch")
+    pg_new_conn = pg_new.connect()
+    cur_new = pg_new_conn.cursor()
+
+    cur_new.execute('''
+    set enable_seqscan=off;
+    set enable_indexscan=on;
+    set enable_bitmapscan=off;
+    ''')
+
+    cur_new.execute('SELECT * FROM vmtest_delete WHERE id = 1')
+    assert(cur_new.fetchall() == []);
+    cur_new.execute('SELECT * FROM vmtest_update WHERE id = 1')
+    assert(cur_new.fetchall() == []);
--- a/test_runner/batch_others/test_wal_acceptor.py
+++ b/test_runner/batch_others/test_wal_acceptor.py
@@ -4,7 +4,7 @@ import time

 from contextlib import closing
 from multiprocessing import Process, Value
-from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory
+from fixtures.zenith_fixtures import WalAcceptorFactory, ZenithPageserver, PostgresFactory

 pytest_plugins = ("fixtures.zenith_fixtures")

@@ -61,7 +61,7 @@ def test_many_timelines(zenith_cli, pageserver: ZenithPageserver, postgres: Post
 # Check that dead minority doesn't prevent the commits: execute insert n_inserts
 # times, with fault_probability chance of getting a wal acceptor down or up
 # along the way. 2 of 3 are always alive, so the work keeps going.
-def test_restarts(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory):
+def test_restarts(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory: WalAcceptorFactory):
    fault_probability = 0.01
    n_inserts = 1000
    n_acceptors = 3
--- a/test_runner/batch_wraparound/test_xid_wraparound.py
+++ b/test_runner/batch_wraparound/test_xid_wraparound.py
@@ -0,0 +1,139 @@
+import time
+import os
+
+from contextlib import closing
+
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+#
+# Test compute node start after xid wraparound
+# Note: this is a long test (> 15 min)
+#
+def test_xid_wraparound(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory,
+                        pg_bin):
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_xid_wraparound", "empty"])
+
+    # set agressive autovacuum
+    config = [
+        'autovacuum_max_workers=10', 'autovacuum_vacuum_threshold=0',
+        'autovacuum_vacuum_insert_threshold=0', 'autovacuum_vacuum_cost_delay=0',
+        'autovacuum_vacuum_cost_limit=10000', 'autovacuum_naptime =1s'
+    ]
+
+    pg = postgres.create_start('test_xid_wraparound', config_lines=config)
+    print('postgres is running on test_xid_wraparound branch')
+
+    # Install extension containing function needed for test
+    pg.safe_psql('CREATE EXTENSION zenith_test_utils')
+
+    # Consume many xids
+    for i in range(4):
+        with closing(pg.connect()) as conn:
+            with conn.cursor() as cur:
+                cur.execute('select test_consume_xids(1000*1000*1000);')
+                print('xids consumed')
+
+                # This query is just for debugging
+                cur.execute('select datname, datfrozenxid from pg_database;')
+                res = cur.fetchall()
+                print(res)
+
+                # call a checkpoint to trigger TruncateSubtrans
+                cur.execute('CHECKPOINT;')
+
+                # ensure WAL flush
+                cur.execute('select txid_current();')
+
+                # This query is just for debugging
+                cur.execute('select pg_current_wal_insert_lsn(),'
+                            'next_xid, oldest_xid, oldest_xid_dbid,'
+                            'oldest_active_xid from pg_control_checkpoint();')
+                res = cur.fetchone()
+                print(res)
+
+    # This run triggers xid wraparound
+    pg.safe_psql('select test_consume_xids(1000*1000*1000);')
+
+    print('xids consumed. expect wraparound')
+
+    # call a checkpoint to trigger TruncateSubtrans
+    pg.safe_psql('CHECKPOINT;')
+    # ensure WAL flush
+    txid = pg.safe_psql('select txid_current();')
+    print(txid)
+
+    # wait for autovacuum to truncate the pg_xact
+    i = 0
+    pg_xact_0FFF_path = os.path.join(pg.pg_xact_dir_path(), '0FFF')
+    while os.path.isfile(pg_xact_0FFF_path) and i < 10:
+        print("file exists. wait for truncation. pg_xact_0FFF_path = " + pg_xact_0FFF_path)
+        time.sleep(5)
+        i += 1
+
+    # check that compute node doesn't contain the truncated segment
+    assert os.path.isfile(pg_xact_0FFF_path) == False
+
+    # A bunch of sanity checks for pg_control_checkpoint() fields
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('select pg_current_wal_insert_lsn(),'
+                        'next_xid, oldest_xid, oldest_xid_dbid,'
+                        'oldest_active_xid from pg_control_checkpoint();')
+            res = cur.fetchone()
+            print(res)
+
+    next_full_xid = res[1].split(':')
+    next_xid_epoch = int(next_full_xid[0])
+    next_xid_xid = int(next_full_xid[1])
+    oldest_xid = int(res[2])
+    oldest_xid_dbid = int(res[3])
+
+    # ensure that wraparound happened
+    assert next_xid_xid < oldest_xid
+    # ensure that wraparound happened. check that next_xid epoch > 0
+    assert next_xid_epoch == 1
+
+    # create new branch after xid wraparound and start a compute node on it
+    lsn_after_wraparound = res[0]
+    print('create branch at lsn_after_wraparound ' + lsn_after_wraparound)
+    zenith_cli.run(
+        ["branch", "test_xid_wraparound_new", "test_xid_wraparound@" + lsn_after_wraparound])
+
+    pg2 = postgres.create_start('test_xid_wraparound_new')
+    print('postgres is running on test_xid_wraparound_new branch')
+
+    with closing(pg2.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('select pg_current_wal_insert_lsn(),'
+                        'next_xid, oldest_xid, oldest_xid_dbid,'
+                        'oldest_active_xid from pg_control_checkpoint();')
+            res2 = cur.fetchone()
+            print(res2)
+
+    next_full_xid2 = res2[1].split(':')
+    next_xid_epoch2 = int(next_full_xid2[0])
+    next_xid_xid2 = int(next_full_xid2[1])
+    oldest_xid2 = int(res2[2])
+    oldest_xid_dbid2 = int(res2[3])
+
+    # ensure that wraparound happened
+    assert next_xid_epoch2 == 1
+    assert next_xid_xid2 < oldest_xid2
+
+    # check that new compute node doesn't contain truncated clog segment
+    pg_xact_0FFF_path_new = os.path.join(pg2.pg_xact_dir_path(), '0FFF')
+    print("pg_xact_0FFF_path_new = " + pg_xact_0FFF_path_new)
+    assert os.path.isfile(pg_xact_0FFF_path_new) == False
+
+    # ensure that pg_control_checkpoint() fields are restored correctly
+    assert next_xid_xid <= next_xid_xid2
+    assert oldest_xid == oldest_xid2
+    assert oldest_xid_dbid == oldest_xid_dbid2
+    #TODO check oldest_active_xid?
+
+    # TODO Test transaction visibility like in vendor/postgres/src/test/regress/sql/xid.sql
+    # Transactions from the future shouldn't be visible
--- a/test_runner/fixtures/benchmark_fixture.py
+++ b/test_runner/fixtures/benchmark_fixture.py
@@ -0,0 +1,162 @@
+from pprint import pprint
+
+import os
+import timeit
+import pathlib
+import uuid
+import psycopg2
+import pytest
+from _pytest.config import Config
+from _pytest.runner import CallInfo
+from _pytest.terminal import TerminalReporter
+import shutil
+import signal
+import subprocess
+import time
+
+from contextlib import contextmanager
+from contextlib import closing
+from pathlib import Path
+from dataclasses import dataclass
+
+# Type-related stuff
+from psycopg2.extensions import connection as PgConnection
+from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar, cast
+from typing_extensions import Literal
+
+from .utils import (get_self_dir, mkdir_if_needed, subprocess_capture)
+
+"""
+This file contains fixtures for micro-benchmarks.
+
+To use, declare the 'zenbenchmark' fixture in the test function. Run the
+bencmark, and then record the result by calling zenbenchmark.record. For example:
+
+import timeit
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
+
+def test_mybench(postgres: PostgresFactory, pageserver: ZenithPageserver, zenbenchmark):
+
+    # Initialize the test
+    ...
+    
+    # Run the test, timing how long it takes
+    with zenbenchmark.record_duration('test_query'):
+        cur.execute('SELECT test_query(...)')
+
+    # Record another measurement
+    zenbenchmark.record('speed_of_light', 300000, 'km/s')
+
+
+You can measure multiple things in one test, and record each one with a separate
+call to zenbenchmark. For example, you could time the bulk loading that happens
+in the test initialization, or measure disk usage after the test query.
+
+"""
+
+
+# All the results are collected in this list, as a tuple:
+# (test_name: str, metric_name: str, metric_value: float, unit: str)
+#
+# TODO: It would perhaps be better to store the results as additional
+# properties in the pytest TestReport objects, to make them visible to
+# other pytest tools.
+global zenbenchmark_results
+zenbenchmark_results = []
+
+class ZenithBenchmarkResults:
+    """ An object for recording benchmark results. """
+    def __init__(self):
+        self.results = []
+
+    def record(self, test_name: str, metric_name: str, metric_value: float, unit: str):
+        """
+        Record a benchmark result.
+        """
+
+        self.results.append((test_name, metric_name, metric_value, unit))
+
+# Sesssion scope fixture that initializes the results object
+@pytest.fixture(autouse=True, scope='session')
+def zenbenchmark_global(request) -> Iterator[ZenithBenchmarkResults]:
+    """
+    This is a python decorator for benchmark fixtures
+    """
+    global zenbenchmark_results
+    zenbenchmark_results = ZenithBenchmarkResults()
+
+    yield zenbenchmark_results
+
+class ZenithBenchmarker:
+    """
+    An object for recording benchmark results. This is created for each test
+    function by the zenbenchmark fixture
+    """
+    def __init__(self, results, request):
+        self.results = results
+        self.request = request
+
+    def record(self, metric_name: str, metric_value: float, unit: str):
+        """
+        Record a benchmark result.
+        """
+        self.results.record(self.request.node.name, metric_name, metric_value, unit)
+
+
+    @contextmanager
+    def record_duration(self, metric_name):
+        """
+        Record a duration. Usage:
+        
+        with zenbenchmark.record_duration('foobar_runtime'):
+            foobar()   # measure this
+        
+        """
+        start = timeit.default_timer()
+        yield
+        end = timeit.default_timer()
+
+        self.results.record(self.request.node.name, metric_name, end - start, 's')
+
+@pytest.fixture(scope='function')
+def zenbenchmark(zenbenchmark_global, request) -> Iterator[ZenithBenchmarker]:
+    """
+    This is a python decorator for benchmark fixtures. It contains functions for
+    recording measurements, and prints them out at the end.
+    """
+    benchmarker = ZenithBenchmarker(zenbenchmark_global, request)
+    yield benchmarker
+
+
+# Hook to print the results at the end
+@pytest.hookimpl(hookwrapper=True)
+def pytest_terminal_summary(
+    terminalreporter: TerminalReporter, exitstatus: int, config: Config
+):
+    yield
+
+    global zenbenchmark_results
+
+    if not zenbenchmark_results:
+        return
+
+    terminalreporter.section('Benchmark results', "-")
+
+    for result in zenbenchmark_results.results:
+        func = result[0]
+        metric_name = result[1]
+        metric_value = result[2]
+        unit = result[3]
+
+        terminalreporter.write("{}.{}: ".format(func, metric_name))
+
+        if unit == 'MB':
+            terminalreporter.write("{0:,.0f}".format(metric_value), green=True)
+        elif unit == 's':
+            terminalreporter.write("{0:,.3f}".format(metric_value), green=True)
+        else:
+            terminalreporter.write("{0:,.4f}".format(metric_value), green=True)
+
+        terminalreporter.line(" {}".format(unit))
--- a/test_runner/fixtures/zenith_fixtures.py
+++ b/test_runner/fixtures/zenith_fixtures.py
@@ -1,21 +1,27 @@
-import getpass
+from dataclasses import dataclass
+from functools import cached_property
 import os
 import pathlib
 import uuid
+import jwt
 import psycopg2
 import pytest
 import shutil
 import signal
 import subprocess
+import time

 from contextlib import closing
 from pathlib import Path
+from dataclasses import dataclass

 # Type-related stuff
 from psycopg2.extensions import connection as PgConnection
 from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar, cast
 from typing_extensions import Literal

+import requests
+
 from .utils import (get_self_dir, mkdir_if_needed, subprocess_capture)
 """
 This file contains pytest fixtures. A fixture is a test resource that can be
@@ -41,7 +47,8 @@ Fn = TypeVar('Fn', bound=Callable[..., Any])
 DEFAULT_OUTPUT_DIR = 'test_output'
 DEFAULT_POSTGRES_DIR = 'tmp_install'

-DEFAULT_PAGESERVER_PORT = 64000
+DEFAULT_PAGESERVER_PG_PORT = 64000
+DEFAULT_PAGESERVER_HTTP_PORT = 9898


 def determine_scope(fixture_name: str, config: Any) -> str:
@@ -87,23 +94,30 @@ class PgProtocol:
        self.port = port
        self.username = username or "zenith_admin"

-    def connstr(self, *, dbname: str = 'postgres', username: Optional[str] = None) -> str:
+    def connstr(self, *, dbname: str = 'postgres', username: Optional[str] = None, password: Optional[str] = None) -> str:
        """
        Build a libpq connection string for the Postgres instance.
        """

        username = username or self.username
-        return f'host={self.host} port={self.port} user={username} dbname={dbname}'
+        res = f'host={self.host} port={self.port} user={username} dbname={dbname}'
+        if not password:
+            return res
+        return f'{res} password={password}'

    # autocommit=True here by default because that's what we need most of the time
-    def connect(self, *, autocommit=True, **kwargs: Any) -> PgConnection:
+    def connect(self, *, autocommit=True, dbname: str = 'postgres', username: Optional[str] = None, password: Optional[str] = None) -> PgConnection:
        """
        Connect to the node.
        Returns psycopg2's connection object.
        This method passes all extra params to connstr.
        """

-        conn = psycopg2.connect(self.connstr(**kwargs))
+        conn = psycopg2.connect(self.connstr(
+            dbname=dbname,
+            username=username,
+            password=password,
+        ))
        # WARNING: this setting affects *all* tests!
        conn.autocommit = autocommit
        return conn
@@ -133,6 +147,7 @@ class ZenithCli:
        assert os.path.isdir(binpath)
        self.binpath = binpath
        self.bin_zenith = os.path.join(binpath, 'zenith')
+        self.repo_dir = repo_dir
        self.env = os.environ.copy()
        self.env['ZENITH_REPO_DIR'] = repo_dir
        self.env['POSTGRES_DISTRIB_DIR'] = pg_distrib_dir
@@ -167,22 +182,105 @@ def zenith_cli(zenith_binpath: str, repo_dir: str, pg_distrib_dir: str) -> Zenit
    return ZenithCli(zenith_binpath, repo_dir, pg_distrib_dir)


+class ZenithPageserverHttpClient(requests.Session):
+    def __init__(self, port: int, auth_token: Optional[str] = None) -> None:
+        super().__init__()
+        self.port = port
+        self.auth_token = auth_token
+
+        if auth_token is not None:
+            self.headers['Authorization'] = f'Bearer {auth_token}'
+
+    def check_status(self):
+        self.get(f"http://localhost:{self.port}/v1/status").raise_for_status()
+
+    def branch_list(self, tenant_id: uuid.UUID) -> List[Dict]:
+        res = self.get(f"http://localhost:{self.port}/v1/branch/{tenant_id.hex}")
+        res.raise_for_status()
+        return res.json()
+
+    def branch_create(self, tenant_id: uuid.UUID, name: str, start_point: str) -> Dict:
+        res = self.post(
+            f"http://localhost:{self.port}/v1/branch",
+            json={
+                'tenant_id': tenant_id.hex,
+                'name': name,
+                'start_point': start_point,
+            }
+        )
+        res.raise_for_status()
+        return res.json()
+
+    def tenant_list(self) -> List[str]:
+        res = self.get(f"http://localhost:{self.port}/v1/tenant")
+        res.raise_for_status()
+        return res.json()
+
+    def tenant_create(self, tenant_id: uuid.UUID):
+        res = self.post(
+            f"http://localhost:{self.port}/v1/tenant",
+            json={
+                'tenant_id': tenant_id.hex,
+            },
+        )
+        res.raise_for_status()
+        return res.json()
+
+
+@dataclass
+class AuthKeys:
+    pub: bytes
+    priv: bytes
+
+    def generate_management_token(self):
+        token = jwt.encode({"scope": "pageserverapi"}, self.priv, algorithm="RS256")
+
+        # jwt.encode can return 'bytes' or 'str', depending on Python version or type
+        # hinting or something (not sure what). If it returned 'bytes', convert it to 'str'
+        # explicitly.
+        if isinstance(token, bytes):
+            token = token.decode()
+
+        return token
+
+    def generate_tenant_token(self, tenant_id):
+        token = jwt.encode({"scope": "tenant", "tenant_id": tenant_id}, self.priv, algorithm="RS256")
+
+        if isinstance(token, bytes):
+            token = token.decode()
+
+        return token
+
+
+
+
+
 class ZenithPageserver(PgProtocol):
    """ An object representing a running pageserver. """
-    def __init__(self, zenith_cli: ZenithCli):
-        super().__init__(host='localhost', port=DEFAULT_PAGESERVER_PORT)
+    def __init__(self, zenith_cli: ZenithCli, repo_dir: str):
+        super().__init__(host='localhost', port=DEFAULT_PAGESERVER_PG_PORT)
        self.zenith_cli = zenith_cli
        self.running = False
        self.initial_tenant = None
+        self.repo_dir = repo_dir

-    def init(self) -> 'ZenithPageserver':
+    def init(self, enable_auth: bool = False) -> 'ZenithPageserver':
        """
        Initialize the repository, i.e. run "zenith init".
        Returns self.
        """
-        self.zenith_cli.run(['init'])
+        cmd = ['init']
+        if enable_auth:
+            cmd.append('--enable-auth')
+        self.zenith_cli.run(cmd)
        return self

+    def repo_dir(self) -> str:
+        """
+        Return path to repository dir
+        """
+        return self.zenith_cli.repo_dir
+
    def start(self) -> 'ZenithPageserver':
        """
        Start the page server.
@@ -207,9 +305,27 @@ class ZenithPageserver(PgProtocol):

        return self

+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        self.stop()
+
+    @cached_property
+    def auth_keys(self) -> AuthKeys:
+        pub = (Path(self.repo_dir) / 'auth_public_key.pem').read_bytes()
+        priv = (Path(self.repo_dir) / 'auth_private_key.pem').read_bytes()
+        return AuthKeys(pub=pub, priv=priv)
+
+    def http_client(self, auth_token: Optional[str] = None):
+        return ZenithPageserverHttpClient(
+            port=DEFAULT_PAGESERVER_HTTP_PORT,
+            auth_token=auth_token,
+        )
+

@zenfixture
-def pageserver(zenith_cli: ZenithCli) -> Iterator[ZenithPageserver]:
+def pageserver(zenith_cli: ZenithCli, repo_dir: str) -> Iterator[ZenithPageserver]:
    """
    The 'pageserver' fixture provides a Page Server that's up and running.

@@ -222,7 +338,7 @@ def pageserver(zenith_cli: ZenithCli) -> Iterator[ZenithPageserver]:
    test called 'test_foo' would create and use branches with the 'test_foo' prefix.
    """

-    ps = ZenithPageserver(zenith_cli).init().start()
+    ps = ZenithPageserver(zenith_cli, repo_dir).init().start()
    # For convenience in tests, create a branch from the freshly-initialized cluster.
    zenith_cli.run(["branch", "empty", "main"])

@@ -233,24 +349,32 @@ def pageserver(zenith_cli: ZenithCli) -> Iterator[ZenithPageserver]:
    ps.stop()


+@pytest.fixture
+def pageserver_auth_enabled(zenith_cli: ZenithCli, repo_dir: str):
+    with ZenithPageserver(zenith_cli, repo_dir).init(enable_auth=True).start() as ps:
+        # For convenience in tests, create a branch from the freshly-initialized cluster.
+        zenith_cli.run(["branch", "empty", "main"])
+        yield ps
+
+
 class Postgres(PgProtocol):
    """ An object representing a running postgres daemon. """
-    def __init__(self, zenith_cli: ZenithCli, repo_dir: str, instance_num: int, tenant_id: str):
-        super().__init__(host='localhost', port=55431 + instance_num)
+    def __init__(self, zenith_cli: ZenithCli, repo_dir: str, tenant_id: str, port: int):
+        super().__init__(host='localhost', port=port)

        self.zenith_cli = zenith_cli
-        self.instance_num = instance_num
        self.running = False
        self.repo_dir = repo_dir
        self.branch: Optional[str] = None  # dubious, see asserts below
+        self.pgdata_dir: Optional[str] = None # Path to computenode PGDATA
        self.tenant_id = tenant_id
-        # path to conf is <repo_dir>/pgdatadirs/tenants/<tenant_id>/<branch_name>/postgresql.conf

    def create(
        self,
        branch: str,
        wal_acceptors: Optional[str] = None,
        config_lines: Optional[List[str]] = None,
+        config_only: bool = False,
    ) -> 'Postgres':
        """
        Create the pg data directory.
@@ -262,8 +386,14 @@ class Postgres(PgProtocol):
        if not config_lines:
            config_lines = []

-        self.zenith_cli.run(['pg', 'create', branch, f'--tenantid={self.tenant_id}'])
+        if config_only:
+            self.zenith_cli.run(['pg', 'create', '--config-only', branch, f'--tenantid={self.tenant_id}'])
+        else:
+            self.zenith_cli.run(['pg', 'create', branch, f'--tenantid={self.tenant_id}'])
        self.branch = branch
+        path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch
+        self.pgdata_dir = os.path.join(self.repo_dir, path)
+
        if wal_acceptors is not None:
            self.adjust_for_wal_acceptors(wal_acceptors)
        if config_lines is None:
@@ -284,10 +414,20 @@ class Postgres(PgProtocol):

        return self

+    def pg_xact_dir_path(self) -> str:
+        """ Path to pg_xact dir """
+        return os.path.join(self.pgdata_dir, 'pg_xact')
+
+    def pg_twophase_dir_path(self) -> str:
+        """ Path to pg_twophase dir """
+        print(self.tenant_id)
+        print(self.branch)
+        path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch / 'pg_twophase'
+        return os.path.join(self.repo_dir, path)
+
    def config_file_path(self) -> str:
        """ Path to postgresql.conf """
-        filename = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch / 'postgresql.conf'
-        return os.path.join(self.repo_dir, filename)
+        return os.path.join(self.pgdata_dir, 'postgresql.conf')

    def adjust_for_wal_acceptors(self, wal_acceptors: str) -> 'Postgres':
        """
@@ -356,7 +496,8 @@ class Postgres(PgProtocol):
        config_lines: Optional[List[str]] = None,
    ) -> 'Postgres':
        """
-        Create a Postgres instance, then start it.
+        Create a Postgres instance, apply config
+        and then start it.
        Returns self.
        """

@@ -364,19 +505,27 @@ class Postgres(PgProtocol):
            branch=branch,
            wal_acceptors=wal_acceptors,
            config_lines=config_lines,
+            config_only=True,
        ).start()

        return self

+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        self.stop()
+

 class PostgresFactory:
    """ An object representing multiple running postgres daemons. """
-    def __init__(self, zenith_cli: ZenithCli, repo_dir: str, initial_tenant: str):
+    def __init__(self, zenith_cli: ZenithCli, repo_dir: str, initial_tenant: str, base_port: int = 55431):
        self.zenith_cli = zenith_cli
        self.repo_dir = repo_dir
        self.num_instances = 0
        self.instances: List[Postgres] = []
        self.initial_tenant: str = initial_tenant
+        self.base_port = base_port

    def create_start(
        self,
@@ -386,7 +535,13 @@ class PostgresFactory:
        config_lines: Optional[List[str]] = None
    ) -> Postgres:

-        pg = Postgres(self.zenith_cli, self.repo_dir, self.num_instances + 1, tenant_id=tenant_id or self.initial_tenant)
+        pg = Postgres(
+            zenith_cli=self.zenith_cli,
+            repo_dir=self.repo_dir,
+            tenant_id=tenant_id or self.initial_tenant,
+            port=self.base_port + self.num_instances + 1,
+        )
+
        self.num_instances += 1
        self.instances.append(pg)

@@ -396,6 +551,54 @@ class PostgresFactory:
            config_lines=config_lines,
        )

+    def create(
+        self,
+        branch: str = "main",
+        tenant_id: Optional[str] = None,
+        wal_acceptors: Optional[str] = None,
+        config_lines: Optional[List[str]] = None
+    ) -> Postgres:
+
+        pg = Postgres(
+            zenith_cli=self.zenith_cli,
+            repo_dir=self.repo_dir,
+            tenant_id=tenant_id or self.initial_tenant,
+            port=self.base_port + self.num_instances + 1,
+        )
+
+        self.num_instances += 1
+        self.instances.append(pg)
+
+        return pg.create(
+            branch=branch,
+            wal_acceptors=wal_acceptors,
+            config_lines=config_lines,
+        )
+
+    def config(
+        self,
+        branch: str = "main",
+        tenant_id: Optional[str] = None,
+        wal_acceptors: Optional[str] = None,
+        config_lines: Optional[List[str]] = None
+    ) -> Postgres:
+
+        pg = Postgres(
+            zenith_cli=self.zenith_cli,
+            repo_dir=self.repo_dir,
+            tenant_id=tenant_id or self.initial_tenant,
+            port=self.base_port + self.num_instances + 1,
+        )
+
+        self.num_instances += 1
+        self.instances.append(pg)
+
+        return pg.config(
+            branch=branch,
+            wal_acceptors=wal_acceptors,
+            config_lines=config_lines,
+        )
+
    def stop_all(self) -> 'PostgresFactory':
        for pg in self.instances:
            pg.stop()
@@ -478,88 +681,108 @@ def pg_bin(test_output_dir: str, pg_distrib_dir: str) -> PgBin:
    return PgBin(test_output_dir, pg_distrib_dir)


-def read_pid(path):
+def read_pid(path: Path):
    """ Read content of file into number """
-    return int(Path(path).read_text())
+    return int(path.read_text())


+@dataclass
 class WalAcceptor:
    """ An object representing a running wal acceptor daemon. """
-    def __init__(self, wa_binpath, data_dir, port, num):
-        self.wa_binpath = wa_binpath
-        self.data_dir = data_dir
-        self.port = port
-        self.num = num  # identifier for logging
+    bin_path: Path
+    data_dir: Path
+    port: int
+    num: int # identifier for logging
+    auth_token: Optional[str] = None

    def start(self) -> 'WalAcceptor':
        # create data directory if not exists
-        Path(self.data_dir).mkdir(parents=True, exist_ok=True)
+        self.data_dir.mkdir(parents=True, exist_ok=True)
+        self.pidfile.unlink(missing_ok=True)

-        cmd = [self.wa_binpath]
-        cmd.extend(["-D", self.data_dir])
+        cmd = [str(self.bin_path)]
+        cmd.extend(["-D", str(self.data_dir)])
        cmd.extend(["-l", "localhost:{}".format(self.port)])
        cmd.append("--daemonize")
        cmd.append("--no-sync")
        # Tell page server it can receive WAL from this WAL safekeeper
-        cmd.extend(["--pageserver", "localhost:{}".format(DEFAULT_PAGESERVER_PORT)])
+        cmd.extend(["--pageserver", "localhost:{}".format(DEFAULT_PAGESERVER_PG_PORT)])
        cmd.extend(["--recall", "1 second"])
        print('Running command "{}"'.format(' '.join(cmd)))
-        subprocess.run(cmd, check=True)
+        env = {'PAGESERVER_AUTH_TOKEN': self.auth_token} if self.auth_token else None
+        subprocess.run(cmd, check=True, env=env)

-        return self
+        # wait for wal acceptor start by checkking that pid is readable
+        for _ in range(3):
+            pid = self.get_pid()
+            if pid is not None:
+                return self
+            time.sleep(0.5)
+
+        raise RuntimeError("cannot get wal acceptor pid")
+
+    @property
+    def pidfile(self) -> Path:
+        return self.data_dir / "wal_acceptor.pid"
+
+    def get_pid(self) -> Optional[int]:
+        if not self.pidfile.exists():
+            return None
+
+        try:
+            pid = read_pid(self.pidfile)
+        except ValueError:
+            return None
+
+        return pid

    def stop(self) -> 'WalAcceptor':
        print('Stopping wal acceptor {}'.format(self.num))
-        pidfile_path = os.path.join(self.data_dir, "wal_acceptor.pid")
-        try:
-            pid = read_pid(pidfile_path)
-            try:
-                os.kill(pid, signal.SIGTERM)
-            except Exception:
-                pass  # pidfile might be obsolete
-            # TODO: cleanup pid file on exit in wal acceptor
-            return self
-            # for _ in range(5):
-            # print('waiting wal acceptor {} (pid {}) to stop...', self.num, pid)
-            # try:
-            # read_pid(pidfile_path)
-            # except FileNotFoundError:
-            # return  # done
-            # time.sleep(1)
-            # raise Exception('Failed to wait for wal acceptor {} shutdown'.format(self.num))
-        except FileNotFoundError:
+        pid = self.get_pid()
+        if pid is None:
            print("Wal acceptor {} is not running".format(self.num))
            return self

+        try:
+            os.kill(pid, signal.SIGTERM)
+        except Exception:
+            # TODO: cleanup pid file on exit in wal acceptor
+            pass # pidfile might be obsolete
+        return self
+

 class WalAcceptorFactory:
    """ An object representing multiple running wal acceptors. """
-    def __init__(self, zenith_binpath, data_dir):
-        self.wa_binpath = os.path.join(zenith_binpath, 'wal_acceptor')
+    def __init__(self, zenith_binpath: Path, data_dir: Path):
+        self.wa_binpath = zenith_binpath / 'wal_acceptor'
        self.data_dir = data_dir
-        self.instances = []
+        self.instances: List[WalAcceptor] = []
        self.initial_port = 54321

-    def start_new(self) -> WalAcceptor:
+    def start_new(self, auth_token: Optional[str] = None) -> WalAcceptor:
        """
        Start new wal acceptor.
        """

        wa_num = len(self.instances)
-        wa = WalAcceptor(self.wa_binpath,
-                         os.path.join(self.data_dir, "wal_acceptor_{}".format(wa_num)),
-                         self.initial_port + wa_num, wa_num)
+        wa = WalAcceptor(
+            self.wa_binpath,
+            self.data_dir / "wal_acceptor_{}".format(wa_num),
+            self.initial_port + wa_num,
+            wa_num,
+            auth_token,
+        )
        wa.start()
        self.instances.append(wa)
        return wa

-    def start_n_new(self, n: int) -> None:
+    def start_n_new(self, n: int, auth_token: Optional[str] = None) -> None:
        """
        Start n new wal acceptors.
        """

        for _ in range(n):
-            self.start_new()
+            self.start_new(auth_token)

    def stop_all(self) -> 'WalAcceptorFactory':
        for wa in self.instances:
@@ -574,7 +797,7 @@ class WalAcceptorFactory:
@zenfixture
 def wa_factory(zenith_binpath: str, repo_dir: str) -> Iterator[WalAcceptorFactory]:
    """ Gives WalAcceptorFactory providing wal acceptors. """
-    wafactory = WalAcceptorFactory(zenith_binpath, os.path.join(repo_dir, "wal_acceptors"))
+    wafactory = WalAcceptorFactory(Path(zenith_binpath), Path(repo_dir) / "wal_acceptors")
    yield wafactory
    # After the yield comes any cleanup code we need.
    print('Starting wal acceptors cleanup')
--- a/test_runner/performance/test_perf_pgbench.py
+++ b/test_runner/performance/test_perf_pgbench.py
@@ -0,0 +1,67 @@
+import os
+from contextlib import closing
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
+
+def get_timeline_size(repo_dir: str, tenantid: str, timelineid: str):
+    path = "{}/tenants/{}/timelines/{}".format(repo_dir, tenantid, timelineid)
+
+    totalbytes = 0
+    for root, dirs, files in os.walk(path):
+        for name in files:
+            totalbytes += os.path.getsize(os.path.join(root, name))
+
+        if 'wal' in dirs:
+            dirs.remove('wal')  # don't visit 'wal' subdirectory
+
+    return totalbytes
+
+#
+# Run a very short pgbench test.
+#
+# Collects three metrics:
+#
+# 1. Time to initialize the pgbench database (pgbench -s5 -i)
+# 2. Time to run 5000 pgbench transactions
+# 3. Disk space used
+#
+def test_pgbench(postgres: PostgresFactory, pageserver: ZenithPageserver, pg_bin, zenith_cli, zenbenchmark, repo_dir: str):
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_pgbench_perf", "empty"])
+
+    pg = postgres.create_start('test_pgbench_perf')
+    print("postgres is running on 'test_pgbench_perf' branch")
+
+    # Open a connection directly to the page server that we'll use to force
+    # flushing the layers to disk
+    psconn = pageserver.connect();
+    pscur = psconn.cursor()
+
+    # Get the timeline ID of our branch. We need it for the 'do_gc' command
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute("SHOW zenith.zenith_timeline")
+            timeline = cur.fetchone()[0]
+
+    connstr = pg.connstr()
+
+    # Initialize pgbench database
+    with zenbenchmark.record_duration('init'):
+        pg_bin.run_capture(['pgbench', '-s5', '-i', connstr])
+
+        # Flush the layers from memory to disk. The time to do that is included in the
+        # reported init time.
+        pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+
+    # Run pgbench for 5000 transactions
+    with zenbenchmark.record_duration('5000_xacts'):
+        pg_bin.run_capture(['pgbench', '-c1', '-t5000', connstr])
+
+    # Flush the layers to disk again. This is *not' included in the reported time,
+    # though.
+    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+
+    # Report disk space used by the repository
+    timeline_size = get_timeline_size(repo_dir, pageserver.initial_tenant, timeline)
+    zenbenchmark.record('size', timeline_size / (1024*1024), 'MB')
--- a/vendor/postgres
+++ b/vendor/postgres
--- a/walkeeper/Cargo.toml
+++ b/walkeeper/Cargo.toml
@@ -8,6 +8,7 @@ edition = "2018"

 [dependencies]
 regex = "1.4.5"
+bincode = "1.3"
 bytes = "1.0.1"
 byteorder = "1.4.3"
 fs2 = "0.4.3"
--- a/walkeeper/README
+++ b/walkeeper/README
@@ -1,24 +1,64 @@
-# WAL safekeeper
+# WAL service

-Also know as the WAL service, WAL keeper or WAL acceptor.
-
-The WAL safekeeper acts as a holding area and redistribution center
-for recently generated WAL. The primary Postgres server streams the
-WAL to the WAL safekeeper, and treats it like a (synchronous)
+The zenith WAL service acts as a holding area and redistribution
+center for recently generated WAL. The primary Postgres server streams
+the WAL to the WAL safekeeper, and treats it like a (synchronous)
 replica. A replication slot is used in the primary to prevent the
-primary from discarding WAL that hasn't been streamed to the
-safekeeper yet.
+primary from discarding WAL that hasn't been streamed to the WAL
+service yet.

-The primary connects to the WAL safekeeper, so it works in a "push"
+--------------+              +------------------+
+|              |     WAL      |                  |
+| Compute node |  ----------> |   WAL Service    |
+|              |              |                  |
+--------------+              +------------------+
+                                     |
+                                     |
+                                     | WAL
+                                     |
+                                     |
+                                     V
+                              +--------------+
+                              |              |
+                              | Pageservers  |
+                              |              |
+                              +--------------+
+
+
+
+The WAL service consists of multiple WAL safekeepers that all store a
+copy of the WAL. A WAL record is considered durable when the majority
+of safekeepers have received and stored the WAL to local disk. A
+consensus algorithm based on Paxos is used to manage the quorum.
+
+  +-------------------------------------------+
+  | WAL Service                               |
+  |                                           |
+  |                                           |
+  |  +------------+                           |
+  |  | safekeeper |                           |
+  |  +------------+                           |
+  |                                           |
+  |  +------------+                           |
+  |  | safekeeper |                           |
+  |  +------------+                           |
+  |                                           |
+  |  +------------+                           |
+  |  | safekeeper |                           |
+  |  +------------+                           |
+  |                                           |
+  +-------------------------------------------+
+
+
+The primary connects to the WAL safekeepers, so it works in a "push"
 fashion.  That's different from how streaming replication usually
 works, where the replica initiates the connection. To do that, there
 is a component called the "WAL proposer". The WAL proposer is a
 background worker that runs in the primary Postgres server. It
-connects to the WAL safekeeper, and
-sends all the WAL. (PostgreSQL's archive_commands works in the
-"push" style, but it operates on a WAL segment granularity. If
-PostgreSQL had a push style API for streaming, WAL propose could be
-implemented using it.)
+connects to the WAL safekeeper, and sends all the WAL. (PostgreSQL's
+archive_commands works in the "push" style, but it operates on a WAL
+segment granularity. If PostgreSQL had a push style API for streaming,
+WAL propose could be implemented using it.)

 The Page Server connects to the WAL safekeeper, using the same
 streaming replication protocol that's used between Postgres primary
@@ -33,5 +73,17 @@ safekeepers. The Paxos and crash recovery algorithm ensures that only
 one primary node can be actively streaming WAL to the quorum of
 safekeepers.

-See README_PROTO.md for a more detailed desription of the consensus protocol. spec/
-contains TLA+ specification of it.
+See README_PROTO.md for a more detailed desription of the consensus
+protocol. spec/ contains TLA+ specification of it.
+
+
+# Terminology
+
+WAL service - The service as whole that ensures that WAL is stored durably.
+
+WAL safekeeper - One node that participates in the quorum. All the safekeepers
+together form the WAL service.
+
+WAL acceptor, WAL proposer - In the context of the consensus algorithm, the Postgres
+compute node is also known as the WAL proposer, and the safekeeper is also known
+as the acceptor. Those are the standard terms in the Paxos algorithm.
--- a/walkeeper/src/bin/wal_acceptor.rs
+++ b/walkeeper/src/bin/wal_acceptor.rs
@@ -6,9 +6,9 @@ use clap::{App, Arg};
 use daemonize::Daemonize;
 use log::*;
 use slog::Drain;
-use std::io;
 use std::path::{Path, PathBuf};
 use std::thread;
+use std::{env, io};
 use std::{fs::File, fs::OpenOptions};

 use walkeeper::s3_offload;
@@ -74,6 +74,7 @@ fn main() -> Result<()> {
        listen_addr: "localhost:5454".to_string(),
        ttl: None,
        recall_period: None,
+        pageserver_auth_token: env::var("PAGESERVER_AUTH_TOKEN").ok(),
    };

    if let Some(dir) = arg_matches.value_of("datadir") {
--- a/walkeeper/src/lib.rs
+++ b/walkeeper/src/lib.rs
@@ -5,6 +5,7 @@ use std::time::Duration;
 pub mod receive_wal;
 pub mod replication;
 pub mod s3_offload;
+pub mod safekeeper;
 pub mod send_wal;
 pub mod timeline;
 pub mod wal_service;
@@ -16,6 +17,8 @@ pub struct WalAcceptorConf {
    pub no_sync: bool,
    pub listen_addr: String,
    pub pageserver_addr: Option<String>,
+    // TODO (create issue) this is temporary, until protocol between PG<->SK<->PS rework
+    pub pageserver_auth_token: Option<String>,
    pub ttl: Option<Duration>,
    pub recall_period: Option<Duration>,
 }
--- a/walkeeper/src/receive_wal.rs
+++ b/walkeeper/src/receive_wal.rs
@@ -1,150 +1,32 @@
-//! This implements the Safekeeper protocol.
-//!
-//! FIXME: better description needed here
+//! Safekeeper communication endpoint to wal proposer (compute node).
+//! Gets messages from the network, passes them down to consensus module and
+//! sends replies back.

-use anyhow::{bail, Result};
+use anyhow::{bail, Context, Result};
+use bytes::Bytes;
 use log::*;
 use postgres::{Client, Config, NoTls};
-use serde::{Deserialize, Serialize};
-use std::cmp::{max, min};
-use std::fs::{self, File, OpenOptions};
-use std::io::{BufReader, Read, Seek, SeekFrom, Write};
-use std::net::{SocketAddr, TcpStream};
-use std::str;
-use std::sync::Arc;
+
+use std::net::SocketAddr;
 use std::thread;
 use std::thread::sleep;
-use zenith_utils::bin_ser::LeSer;
-use zenith_utils::connstring::connection_host_port;
-use zenith_utils::lsn::Lsn;

-use crate::replication::HotStandbyFeedback;
-use crate::timeline::{Timeline, TimelineTools};
+use crate::safekeeper::AcceptorProposerMessage;
+use crate::safekeeper::ProposerAcceptorMessage;
+
+use crate::send_wal::SendWalHandler;
+use crate::timeline::TimelineTools;
 use crate::WalAcceptorConf;
-use pageserver::{ZTenantId, ZTimelineId};
-use postgres_ffi::xlog_utils::{TimeLineID, XLogFileName, MAX_SEND_SIZE, XLOG_BLCKSZ};
-use zenith_utils::pq_proto::SystemId;
+use zenith_utils::connstring::connection_host_port;
+use zenith_utils::postgres_backend::PostgresBackend;
+use zenith_utils::pq_proto::{BeMessage, FeMessage};
+use zenith_utils::zid::{ZTenantId, ZTimelineId};

-pub const SK_MAGIC: u32 = 0xcafeceefu32;
-pub const SK_FORMAT_VERSION: u32 = 1;
-const SK_PROTOCOL_VERSION: u32 = 1;
-const UNKNOWN_SERVER_VERSION: u32 = 0;
-const END_OF_STREAM: Lsn = Lsn(0);
-pub const CONTROL_FILE_NAME: &str = "safekeeper.control";
-
-/// Unique node identifier used by Paxos
-#[derive(Debug, Clone, Copy, Ord, PartialOrd, PartialEq, Eq, Serialize, Deserialize)]
-pub struct NodeId {
-    term: u64,
-    uuid: [u8; 16],
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub struct ServerInfo {
-    /// proposer-safekeeper protocol version
-    pub protocol_version: u32,
-    /// Postgres server version
-    pub pg_version: u32,
-    pub node_id: NodeId,
-    pub system_id: SystemId,
-    /// Zenith timelineid
-    pub timeline_id: ZTimelineId,
-    pub wal_end: Lsn,
-    pub timeline: TimeLineID,
-    pub wal_seg_size: u32,
-    pub tenant_id: ZTenantId,
-}
-
-/// Vote request sent from proposer to safekeepers
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-struct RequestVote {
-    node_id: NodeId,
-    /// volume commit LSN
-    vcl: Lsn,
-    /// new epoch when safekeeper reaches vcl
-    epoch: u64,
-}
-
-/// Information of about storage node
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub struct SafeKeeperInfo {
-    /// magic for verifying content the control file
-    pub magic: u32,
-    /// safekeeper format version
-    pub format_version: u32,
-    /// safekeeper's epoch
-    pub epoch: u64,
-    /// information about server
-    pub server: ServerInfo,
-    /// part of WAL acknowledged by quorum
-    pub commit_lsn: Lsn,
-    /// locally flushed part of WAL
-    pub flush_lsn: Lsn,
-    /// minimal LSN which may be needed for recovery of some safekeeper: min(commit_lsn) for all safekeepers
-    pub restart_lsn: Lsn,
-}
-
-impl SafeKeeperInfo {
-    pub fn new() -> SafeKeeperInfo {
-        SafeKeeperInfo {
-            magic: SK_MAGIC,
-            format_version: SK_FORMAT_VERSION,
-            epoch: 0,
-            server: ServerInfo {
-                protocol_version: SK_PROTOCOL_VERSION, /* proposer-safekeeper protocol version */
-                pg_version: UNKNOWN_SERVER_VERSION,    /* Postgres server version */
-                node_id: NodeId {
-                    term: 0,
-                    uuid: [0; 16],
-                },
-                system_id: 0, /* Postgres system identifier */
-                timeline_id: ZTimelineId::from([0u8; 16]),
-                wal_end: Lsn(0),
-                timeline: 0,
-                wal_seg_size: 0,
-                tenant_id: ZTenantId::from([0u8; 16]),
-            },
-            commit_lsn: Lsn(0),  /* part of WAL acknowledged by quorum */
-            flush_lsn: Lsn(0),   /* locally flushed part of WAL */
-            restart_lsn: Lsn(0), /* minimal LSN which may be needed for recovery of some safekeeper */
-        }
-    }
-}
-
-/// Request with WAL message sent from proposer to safekeeper.
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-struct SafeKeeperRequest {
-    /// Sender's node identifier (looks like we do not need it for TCP streaming connection)
-    sender_id: NodeId,
-    /// start position of message in WAL
-    begin_lsn: Lsn,
-    /// end position of message in WAL
-    end_lsn: Lsn,
-    /// restart LSN position  (minimal LSN which may be needed by proposer to perform recovery)
-    restart_lsn: Lsn,
-    /// LSN committed by quorum of safekeepers
-    commit_lsn: Lsn,
-}
-
-/// Report safekeeper state to proposer
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-struct SafeKeeperResponse {
-    epoch: u64,
-    flush_lsn: Lsn,
-    hs_feedback: HotStandbyFeedback,
-}
-
-#[derive(Debug)]
-pub struct ReceiveWalConn {
-    pub timeline: Option<Arc<Timeline>>,
-    /// Postgres connection, buffered input
-    pub stream_in: BufReader<TcpStream>,
-    /// Postgres connection, output
-    pub stream_out: TcpStream,
-    /// The cached result of socket.peer_addr()
-    pub peer_addr: SocketAddr,
-    /// wal acceptor configuration
-    pub conf: WalAcceptorConf,
+pub struct ReceiveWalConn<'pg> {
+    /// Postgres connection
+    pg_backend: &'pg mut PostgresBackend,
+    /// The cached result of `pg_backend.socket().peer_addr()` (roughly)
+    peer_addr: SocketAddr,
 }

 ///
@@ -153,7 +35,11 @@ pub struct ReceiveWalConn {
 ///
 fn request_callback(conf: WalAcceptorConf, timelineid: ZTimelineId, tenantid: ZTenantId) {
    let ps_addr = conf.pageserver_addr.unwrap();
-    let ps_connstr = format!("postgresql://no_user@{}/no_db", ps_addr);
+    let ps_connstr = format!(
+        "postgresql://no_user:{}@{}/no_db",
+        &conf.pageserver_auth_token.unwrap_or_default(),
+        ps_addr
+    );

    // use Config parsing because SockAddr parsing doesnt allow to use host names instead of ip addresses
    let me_connstr = format!("postgresql://no_user@{}/no_db", conf.listen_addr);
@@ -186,283 +72,76 @@ fn request_callback(conf: WalAcceptorConf, timelineid: ZTimelineId, tenantid: ZT
    }
 }

-impl ReceiveWalConn {
-    pub fn new(socket: TcpStream, conf: WalAcceptorConf) -> Result<ReceiveWalConn> {
-        let peer_addr = socket.peer_addr()?;
-        let conn = ReceiveWalConn {
-            timeline: None,
-            stream_in: BufReader::new(socket.try_clone()?),
-            stream_out: socket,
+impl<'pg> ReceiveWalConn<'pg> {
+    pub fn new(pg: &'pg mut PostgresBackend) -> Result<ReceiveWalConn<'pg>> {
+        let peer_addr = pg.get_peer_addr()?;
+        Ok(ReceiveWalConn {
+            pg_backend: pg,
            peer_addr,
-            conf,
-        };
-        Ok(conn)
+        })
    }

-    fn read_req<T: LeSer>(&mut self) -> Result<T> {
-        // As the trait bound implies, this always encodes little-endian.
-        Ok(T::des_from(&mut self.stream_in)?)
+    // Read and extract the bytes of a `CopyData` message from the postgres instance
+    fn read_msg_bytes(&mut self) -> Result<Bytes> {
+        match self.pg_backend.read_message()? {
+            Some(FeMessage::CopyData(bytes)) => Ok(bytes),
+            Some(msg) => bail!("expected `CopyData` message, found {:?}", msg),
+            None => bail!("connection closed unexpectedly"),
+        }
+    }
+
+    // Read and parse message sent from the postgres instance
+    fn read_msg(&mut self) -> Result<ProposerAcceptorMessage> {
+        let data = self.read_msg_bytes()?;
+        ProposerAcceptorMessage::parse(data)
+    }
+
+    // Send message to the postgres
+    fn write_msg(&mut self, msg: &AcceptorProposerMessage) -> Result<()> {
+        let mut buf = Vec::new();
+        msg.serialize(&mut buf)?;
+        self.pg_backend.write_message(&BeMessage::CopyData(&buf))?;
+        Ok(())
    }

    /// Receive WAL from wal_proposer
-    pub fn run(&mut self) -> Result<()> {
+    pub fn run(&mut self, swh: &mut SendWalHandler) -> Result<()> {
+        // Notify the libpq client that it's allowed to send `CopyData` messages
+        self.pg_backend
+            .write_message(&BeMessage::CopyBothResponse)?;
+
        // Receive information about server
-        let server_info = self.read_req::<ServerInfo>()?;
-        info!(
-            "Start handshake with wal_proposer {} sysid {} timeline {} tenant {}",
-            self.peer_addr, server_info.system_id, server_info.timeline_id, server_info.tenant_id,
-        );
-        // FIXME: also check that the system identifier matches
-        self.timeline.set(server_info.timeline_id)?;
-        self.timeline.get().load_control_file(&self.conf)?;
-
-        let mut my_info = self.timeline.get().get_info();
-
-        /* Check protocol compatibility */
-        if server_info.protocol_version != SK_PROTOCOL_VERSION {
-            bail!(
-                "Incompatible protocol version {}, expected {}",
-                server_info.protocol_version,
-                SK_PROTOCOL_VERSION
-            );
-        }
-        /* Postgres upgrade is not treated as fatal error */
-        if server_info.pg_version != my_info.server.pg_version
-            && my_info.server.pg_version != UNKNOWN_SERVER_VERSION
-        {
-            info!(
-                "Incompatible server version {}, expected {}",
-                server_info.pg_version, my_info.server.pg_version
-            );
+        let mut msg = self
+            .read_msg()
+            .context("failed to receive proposer greeting")?;
+        let tenant_id: ZTenantId;
+        match msg {
+            ProposerAcceptorMessage::Greeting(ref greeting) => {
+                info!(
+                    "start handshake with wal proposer {} sysid {} timeline {}",
+                    self.peer_addr, greeting.system_id, greeting.tli,
+                );
+                tenant_id = greeting.tenant_id;
+            }
+            _ => bail!("unexpected message {:?} instead of greeting", msg),
        }

-        /* Update information about server, but preserve locally stored node_id */
-        let node_id = my_info.server.node_id;
-        my_info.server = server_info.clone();
-        my_info.server.node_id = node_id;
-
-        /* Calculate WAL end based on local data */
-        let (flush_lsn, timeline) = self.timeline.find_end_of_wal(&self.conf.data_dir, true);
-        my_info.flush_lsn = flush_lsn;
-        my_info.server.timeline = timeline;
-
-        /* Report my identifier to proposer */
-        my_info.ser_into(&mut self.stream_out)?;
-
-        /* Wait for vote request */
-        let prop = self.read_req::<RequestVote>()?;
-        /* This is Paxos check which should ensure that only one master can perform commits */
-        if prop.node_id < my_info.server.node_id {
-            /* Send my node-id to inform proposer that it's candidate was rejected */
-            my_info.server.node_id.ser_into(&mut self.stream_out)?;
-            bail!(
-                "Reject connection attempt with term {} because my term is {}",
-                prop.node_id.term,
-                my_info.server.node_id.term,
-            );
-        }
-        my_info.server.node_id = prop.node_id;
-        self.timeline.get().set_info(&my_info);
-        /* Need to persist our vote first */
-        self.timeline.get().save_control_file(true)?;
-
-        let mut flushed_restart_lsn = Lsn(0);
-        let wal_seg_size = server_info.wal_seg_size as usize;
-
-        /* Acknowledge the proposed candidate by returning it to the proposer */
-        prop.node_id.ser_into(&mut self.stream_out)?;
-
-        if self.conf.pageserver_addr.is_some() {
+        // if requested, ask pageserver to fetch wal from us
+        // xxx: this place seems not really fitting
+        if swh.conf.pageserver_addr.is_some() {
            // Need to establish replication channel with page server.
            // Add far as replication in postgres is initiated by receiver, we should use callme mechanism
-            let conf = self.conf.clone();
-            let timelineid = self.timeline.get().timelineid;
-            let tenantid = server_info.tenant_id;
+            let conf = swh.conf.clone();
+            let timelineid = swh.timeline.get().timelineid;
            thread::spawn(move || {
-                request_callback(conf, timelineid, tenantid);
+                request_callback(conf, timelineid, tenant_id);
            });
        }

-        info!(
-            "Start streaming from timeline {} tenant {} address {:?}",
-            server_info.timeline_id, server_info.tenant_id, self.peer_addr,
-        );
-
-        // Main loop
        loop {
-            let mut sync_control_file = false;
-
-            /* Receive message header */
-            let req = self.read_req::<SafeKeeperRequest>()?;
-            if req.sender_id != my_info.server.node_id {
-                bail!("Sender NodeId is changed");
-            }
-            if req.begin_lsn == END_OF_STREAM {
-                info!("Server stops streaming");
-                break;
-            }
-            let start_pos = req.begin_lsn;
-            let end_pos = req.end_lsn;
-            let rec_size = end_pos.checked_sub(start_pos).unwrap().0 as usize;
-            assert!(rec_size <= MAX_SEND_SIZE);
-
-            debug!(
-                "received for {} bytes between {} and {}",
-                rec_size, start_pos, end_pos,
-            );
-
-            /* Receive message body */
-            let mut inbuf = vec![0u8; rec_size];
-            self.stream_in.read_exact(&mut inbuf)?;
-
-            /* Save message in file */
-            self.write_wal_file(start_pos, timeline, wal_seg_size, &inbuf)?;
-
-            my_info.restart_lsn = req.restart_lsn;
-            my_info.commit_lsn = req.commit_lsn;
-
-            /*
-             * Epoch switch happen when written WAL record cross the boundary.
-             * The boundary is maximum of last WAL position at this node (FlushLSN) and global
-             * maximum (vcl) determined by WAL proposer during handshake.
-             * Switching epoch means that node completes recovery and start writing in the WAL new data.
-             */
-            if my_info.epoch < prop.epoch && end_pos > max(my_info.flush_lsn, prop.vcl) {
-                info!("Switch to new epoch {}", prop.epoch);
-                my_info.epoch = prop.epoch; /* bump epoch */
-                sync_control_file = true;
-            }
-            if end_pos > my_info.flush_lsn {
-                my_info.flush_lsn = end_pos;
-            }
-            /*
-             * Update restart LSN in control file.
-             * To avoid negative impact on performance of extra fsync, do it only
-             * when restart_lsn delta exceeds WAL segment size.
-             */
-            sync_control_file |= flushed_restart_lsn + (wal_seg_size as u64) < my_info.restart_lsn;
-            self.timeline.get().save_control_file(sync_control_file)?;
-
-            if sync_control_file {
-                flushed_restart_lsn = my_info.restart_lsn;
-            }
-
-            /* Report flush position */
-            //info!("Confirm LSN: {:X}/{:>08X}", (end_pos>>32) as u32, end_pos as u32);
-            let resp = SafeKeeperResponse {
-                epoch: my_info.epoch,
-                flush_lsn: end_pos,
-                hs_feedback: self.timeline.get().get_hs_feedback(),
-            };
-            resp.ser_into(&mut self.stream_out)?;
-
-            /*
-             * Ping wal sender that new data is available.
-             * FlushLSN (end_pos) can be smaller than commitLSN in case we are at catching-up safekeeper.
-             */
-            self.timeline
-                .get()
-                .notify_wal_senders(min(req.commit_lsn, end_pos));
+            let reply = swh.timeline.get().process_msg(&msg)?;
+            self.write_msg(&reply)?;
+            msg = self.read_msg()?;
        }
-        Ok(())
-    }
-
-    fn write_wal_file(
-        &self,
-        startpos: Lsn,
-        timeline: TimeLineID,
-        wal_seg_size: usize,
-        buf: &[u8],
-    ) -> Result<()> {
-        let mut bytes_left: usize = buf.len();
-        let mut bytes_written: usize = 0;
-        let mut partial;
-        let mut start_pos = startpos;
-        const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ];
-
-        /* Extract WAL location for this block */
-        let mut xlogoff = start_pos.segment_offset(wal_seg_size) as usize;
-
-        while bytes_left != 0 {
-            let bytes_to_write;
-
-            /*
-             * If crossing a WAL boundary, only write up until we reach wal
-             * segment size.
-             */
-            if xlogoff + bytes_left > wal_seg_size {
-                bytes_to_write = wal_seg_size - xlogoff;
-            } else {
-                bytes_to_write = bytes_left;
-            }
-
-            /* Open file */
-            let segno = start_pos.segment_number(wal_seg_size);
-            let wal_file_name = XLogFileName(timeline, segno, wal_seg_size);
-            let wal_file_path = self
-                .conf
-                .data_dir
-                .join(self.timeline.get().timelineid.to_string())
-                .join(wal_file_name.clone());
-            let wal_file_partial_path = self
-                .conf
-                .data_dir
-                .join(self.timeline.get().timelineid.to_string())
-                .join(wal_file_name.clone() + ".partial");
-
-            {
-                let mut wal_file: File;
-                /* Try to open already completed segment */
-                if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_path) {
-                    wal_file = file;
-                    partial = false;
-                } else if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_partial_path)
-                {
-                    /* Try to open existed partial file */
-                    wal_file = file;
-                    partial = true;
-                } else {
-                    /* Create and fill new partial file */
-                    partial = true;
-                    match OpenOptions::new()
-                        .create(true)
-                        .write(true)
-                        .open(&wal_file_partial_path)
-                    {
-                        Ok(mut file) => {
-                            for _ in 0..(wal_seg_size / XLOG_BLCKSZ) {
-                                file.write_all(&ZERO_BLOCK)?;
-                            }
-                            wal_file = file;
-                        }
-                        Err(e) => {
-                            error!("Failed to open log file {:?}: {}", &wal_file_path, e);
-                            return Err(e.into());
-                        }
-                    }
-                }
-                wal_file.seek(SeekFrom::Start(xlogoff as u64))?;
-                wal_file.write_all(&buf[bytes_written..(bytes_written + bytes_to_write)])?;
-
-                // Flush file is not prohibited
-                if !self.conf.no_sync {
-                    wal_file.sync_all()?;
-                }
-            }
-            /* Write was successful, advance our position */
-            bytes_written += bytes_to_write;
-            bytes_left -= bytes_to_write;
-            start_pos += bytes_to_write as u64;
-            xlogoff += bytes_to_write;
-
-            /* Did we reach the end of a WAL segment? */
-            if start_pos.segment_offset(wal_seg_size) == 0 {
-                xlogoff = 0;
-                if partial {
-                    fs::rename(&wal_file_partial_path, &wal_file_path)?;
-                }
-            }
-        }
-        Ok(())
    }
 }
--- a/walkeeper/src/replication.rs
+++ b/walkeeper/src/replication.rs
@@ -1,5 +1,5 @@
 //! This module implements the streaming side of replication protocol, starting
-//! with the "START REPLICATION" message.
+//! with the "START_REPLICATION" message.

 use crate::send_wal::SendWalHandler;
 use crate::timeline::{Timeline, TimelineTools};
@@ -35,6 +35,16 @@ pub struct HotStandbyFeedback {
    pub catalog_xmin: FullTransactionId,
 }

+impl HotStandbyFeedback {
+    pub fn empty() -> HotStandbyFeedback {
+        HotStandbyFeedback {
+            ts: 0,
+            xmin: 0,
+            catalog_xmin: 0,
+        }
+    }
+}
+
 /// A network connection that's speaking the replication protocol.
 pub struct ReplicationConn {
    /// This is an `Option` because we will spawn a background thread that will
@@ -76,8 +86,10 @@ impl ReplicationConn {
                    let feedback = HotStandbyFeedback::des(&m)?;
                    subscriber.add_hs_feedback(feedback);
                }
+                FeMessage::Sync => {}
+                FeMessage::CopyFail => return Err(anyhow!("Copy failed")),
                _ => {
-                    // We only handle `CopyData` messages. Anything else is ignored.
+                    // We only handle `CopyData`, 'Sync', 'CopyFail' messages. Anything else is ignored.
                    info!("unexpected message {:?}", msg);
                }
            }
@@ -148,7 +160,7 @@ impl ReplicationConn {
                break;
            }
        }
-        let (wal_end, timeline) = swh.timeline.find_end_of_wal(&swh.conf.data_dir, false);
+        let (wal_end, timeline) = swh.timeline.get().get_end_of_wal();
        if start_pos == Lsn(0) {
            start_pos = wal_end;
        }
--- a/walkeeper/src/safekeeper.rs
+++ b/walkeeper/src/safekeeper.rs
@@ -0,0 +1,550 @@
+//! Acceptor part of proposer-acceptor consensus algorithm.
+
+use anyhow::{anyhow, bail, Result};
+use byteorder::LittleEndian;
+use byteorder::ReadBytesExt;
+use byteorder::WriteBytesExt;
+use bytes::Buf;
+use bytes::Bytes;
+use log::*;
+use postgres_ffi::xlog_utils::TimeLineID;
+use serde::{Deserialize, Serialize};
+use std::cmp::max;
+use std::io;
+use std::io::Read;
+
+use crate::replication::HotStandbyFeedback;
+use postgres_ffi::xlog_utils::MAX_SEND_SIZE;
+use zenith_utils::bin_ser::LeSer;
+use zenith_utils::lsn::Lsn;
+use zenith_utils::pq_proto::SystemId;
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
+
+pub const SK_MAGIC: u32 = 0xcafeceefu32;
+pub const SK_FORMAT_VERSION: u32 = 1;
+const SK_PROTOCOL_VERSION: u32 = 1;
+const UNKNOWN_SERVER_VERSION: u32 = 0;
+
+/// Consensus logical timestamp.
+type Term = u64;
+
+/// Unique id of proposer. Not needed for correctness, used for monitoring.
+type PgUuid = [u8; 16];
+
+/// Persistent consensus state of the acceptor.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AcceptorState {
+    /// acceptor's last term it voted for (advanced in 1 phase)
+    pub term: Term,
+    /// acceptor's epoch (advanced, i.e. bumped to 'term' when VCL is reached).
+    pub epoch: Term,
+}
+
+/// Information about Postgres. Safekeeper gets it once and then verifies
+/// all further connections from computes match.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct ServerInfo {
+    /// Postgres server version
+    pub pg_version: u32,
+    pub system_id: SystemId,
+    /// Zenith timelineid
+    pub ztli: ZTimelineId,
+    pub tli: TimeLineID,
+    pub wal_seg_size: u32,
+}
+
+/// Persistent information stored on safekeeper node
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SafeKeeperState {
+    /// magic for verifying content the control file
+    pub magic: u32,
+    /// safekeeper format version
+    pub format_version: u32,
+    /// persistent acceptor state
+    pub acceptor_state: AcceptorState,
+    /// information about server
+    pub server: ServerInfo,
+    /// Unique id of the last *elected* proposer we dealed with. Not needed
+    /// correctness, exists for monitoring purposes.
+    pub proposer_uuid: PgUuid,
+    /// part of WAL acknowledged by quorum (note that we might not have wal to
+    /// up this point locally)
+    pub commit_lsn: Lsn,
+    /// minimal LSN which may be needed for recovery of some safekeeper (end lsn
+    /// + 1 of last record streamed to everyone)
+    pub truncate_lsn: Lsn,
+}
+
+impl SafeKeeperState {
+    pub fn new() -> SafeKeeperState {
+        SafeKeeperState {
+            magic: SK_MAGIC,
+            format_version: SK_FORMAT_VERSION,
+            acceptor_state: AcceptorState { term: 0, epoch: 0 },
+            server: ServerInfo {
+                pg_version: UNKNOWN_SERVER_VERSION, /* Postgres server version */
+                system_id: 0,                       /* Postgres system identifier */
+                ztli: ZTimelineId::from([0u8; 16]),
+                tli: 0,
+                wal_seg_size: 0,
+            },
+            proposer_uuid: [0; 16],
+            commit_lsn: Lsn(0),   /* part of WAL acknowledged by quorum */
+            truncate_lsn: Lsn(0), /* minimal LSN which may be needed for recovery of some safekeeper */
+        }
+    }
+}
+
+// protocol messages
+
+/// Initial Proposer -> Acceptor message
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ProposerGreeting {
+    /// proposer-acceptor protocol version
+    pub protocol_version: u32,
+    /// Postgres server version
+    pub pg_version: u32,
+    pub proposer_id: PgUuid,
+    pub system_id: SystemId,
+    /// Zenith timelineid
+    pub ztli: ZTimelineId,
+    pub tenant_id: ZTenantId,
+    pub tli: TimeLineID,
+    pub wal_seg_size: u32,
+}
+
+/// Acceptor -> Proposer initial response: the highest term known to me
+/// (acceptor voted for).
+#[derive(Debug, Serialize, Deserialize)]
+pub struct AcceptorGreeting {
+    term: u64,
+}
+
+/// Vote request sent from proposer to safekeepers
+#[derive(Debug, Serialize, Deserialize)]
+pub struct VoteRequest {
+    term: Term,
+}
+
+/// Vote itself, sent from safekeeper to proposer
+#[derive(Debug, Serialize, Deserialize)]
+pub struct VoteResponse {
+    term: Term,      // not really needed, just a sanity check
+    vote_given: u64, // fixme u64 due to padding
+    /// Safekeeper's log position, to let proposer choose the most advanced one
+    epoch: Term,
+    flush_lsn: Lsn,
+    restart_lsn: Lsn,
+}
+
+/// Request with WAL message sent from proposer to safekeeper. Along the way it
+/// announces 1) successful election (with epoch_start_lsn); 2) commit_lsn.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct AppendRequest {
+    h: AppendRequestHeader,
+    wal_data: Bytes,
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AppendRequestHeader {
+    term: Term,
+    // LSN since the proposer appends WAL; determines epoch switch point.
+    epoch_start_lsn: Lsn,
+    /// start position of message in WAL
+    begin_lsn: Lsn,
+    /// end position of message in WAL
+    end_lsn: Lsn,
+    /// LSN committed by quorum of safekeepers
+    commit_lsn: Lsn,
+    /// restart LSN position  (minimal LSN which may be needed by proposer to perform recovery)
+    restart_lsn: Lsn,
+    // only for logging/debugging
+    proposer_uuid: PgUuid,
+}
+
+/// Report safekeeper state to proposer
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct AppendResponse {
+    // Current term of the safekeeper; if it is higher than proposer's, the
+    // compute is out of date.
+    pub term: Term,
+    pub epoch: Term,
+    // NOTE: this is physical end of wal on safekeeper; currently it doesn't
+    // make much sense without taking epoch into account, as history can be
+    // diverged.
+    pub flush_lsn: Lsn,
+    pub hs_feedback: HotStandbyFeedback,
+}
+
+/// Proposer -> Acceptor messages
+#[derive(Debug)]
+pub enum ProposerAcceptorMessage {
+    Greeting(ProposerGreeting),
+    VoteRequest(VoteRequest),
+    AppendRequest(AppendRequest),
+}
+
+impl ProposerAcceptorMessage {
+    /// Parse proposer message.
+    pub fn parse(msg: Bytes) -> Result<ProposerAcceptorMessage> {
+        // xxx using Reader is inefficient but easy to work with bincode
+        let mut stream = msg.reader();
+        // u64 is here to avoid padding; it will be removed once we stop packing C structs into the wire as is
+        let tag = stream.read_u64::<LittleEndian>()? as u8 as char;
+        match tag {
+            'g' => {
+                let msg = ProposerGreeting::des_from(&mut stream)?;
+                Ok(ProposerAcceptorMessage::Greeting(msg))
+            }
+            'v' => {
+                let msg = VoteRequest::des_from(&mut stream)?;
+                Ok(ProposerAcceptorMessage::VoteRequest(msg))
+            }
+            'a' => {
+                // read header followed by wal data
+                let hdr = AppendRequestHeader::des_from(&mut stream)?;
+                let rec_size = hdr
+                    .end_lsn
+                    .checked_sub(hdr.begin_lsn)
+                    .ok_or(anyhow!("begin_lsn > end_lsn in AppendRequest"))?
+                    .0 as usize;
+                if rec_size > MAX_SEND_SIZE {
+                    bail!(
+                        "AppendRequest is longer than MAX_SEND_SIZE ({})",
+                        MAX_SEND_SIZE
+                    );
+                }
+
+                let mut wal_data_vec: Vec<u8> = vec![0; rec_size];
+                stream.read_exact(&mut wal_data_vec)?;
+                let wal_data = Bytes::from(wal_data_vec);
+                let msg = AppendRequest {
+                    h: hdr,
+                    wal_data: wal_data,
+                };
+
+                Ok(ProposerAcceptorMessage::AppendRequest(msg))
+            }
+            _ => Err(anyhow!("unknown proposer-acceptor message tag: {}", tag,)),
+        }
+    }
+}
+
+/// Acceptor -> Proposer messages
+#[derive(Debug)]
+pub enum AcceptorProposerMessage {
+    Greeting(AcceptorGreeting),
+    VoteResponse(VoteResponse),
+    AppendResponse(AppendResponse),
+}
+
+impl AcceptorProposerMessage {
+    /// Serialize acceptor -> proposer message.
+    pub fn serialize(&self, stream: &mut impl io::Write) -> Result<()> {
+        match self {
+            AcceptorProposerMessage::Greeting(msg) => {
+                stream.write_u64::<LittleEndian>('g' as u64)?;
+                msg.ser_into(stream)?;
+            }
+            AcceptorProposerMessage::VoteResponse(msg) => {
+                stream.write_u64::<LittleEndian>('v' as u64)?;
+                msg.ser_into(stream)?;
+            }
+            AcceptorProposerMessage::AppendResponse(msg) => {
+                stream.write_u64::<LittleEndian>('a' as u64)?;
+                msg.ser_into(stream)?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
+pub trait Storage {
+    /// Persist safekeeper state on disk, optionally syncing it.
+    fn persist(&mut self, s: &SafeKeeperState, sync: bool) -> Result<()>;
+    /// Write piece of wal in buf to disk.
+    fn write_wal(&mut self, s: &SafeKeeperState, startpos: Lsn, buf: &[u8]) -> Result<()>;
+}
+
+/// SafeKeeper which consumes events (messages from compute) and provides
+/// replies.
+#[derive(Debug)]
+pub struct SafeKeeper<ST: Storage> {
+    /// Locally flushed part of WAL (end_lsn of last record). Established by
+    /// reading wal.
+    pub flush_lsn: Lsn,
+    pub tli: u32,
+    pub flushed_truncate_lsn: Lsn,
+    pub storage: ST,
+    pub s: SafeKeeperState,          // persistent part
+    pub elected_proposer_term: Term, // for monitoring/debugging
+}
+
+impl<ST> SafeKeeper<ST>
+where
+    ST: Storage,
+{
+    // constructor
+    pub fn new(flush_lsn: Lsn, tli: u32, storage: ST, state: SafeKeeperState) -> SafeKeeper<ST> {
+        SafeKeeper {
+            flush_lsn,
+            tli,
+            flushed_truncate_lsn: Lsn(0),
+            storage,
+            s: state,
+            elected_proposer_term: 0,
+        }
+    }
+
+    /// Process message from proposer and possibly form reply. Concurrent
+    /// callers must exclude each other.
+    pub fn process_msg(
+        &mut self,
+        msg: &ProposerAcceptorMessage,
+    ) -> Result<AcceptorProposerMessage> {
+        match msg {
+            ProposerAcceptorMessage::Greeting(msg) => self.handle_greeting(msg),
+            ProposerAcceptorMessage::VoteRequest(msg) => self.handle_vote_request(msg),
+            ProposerAcceptorMessage::AppendRequest(msg) => self.handle_append_request(msg),
+        }
+    }
+
+    /// Handle initial message from proposer: check its sanity and send my
+    /// current term.
+    fn handle_greeting(&mut self, msg: &ProposerGreeting) -> Result<AcceptorProposerMessage> {
+        /* Check protocol compatibility */
+        if msg.protocol_version != SK_PROTOCOL_VERSION {
+            bail!(
+                "incompatible protocol version {}, expected {}",
+                msg.protocol_version,
+                SK_PROTOCOL_VERSION
+            );
+        }
+        if self.s.server.system_id != 0 && self.s.server.system_id != msg.system_id {
+            bail!(
+                "system identifier changed: got {}, expected {}",
+                msg.system_id,
+                self.s.server.system_id,
+            );
+        }
+        /* Postgres upgrade is not treated as fatal error */
+        if msg.pg_version != self.s.server.pg_version
+            && self.s.server.pg_version != UNKNOWN_SERVER_VERSION
+        {
+            info!(
+                "incompatible server version {}, expected {}",
+                msg.pg_version, self.s.server.pg_version
+            );
+        }
+
+        // set basic info about server, if not yet
+        self.s.server.system_id = msg.system_id;
+        self.s.server.ztli = msg.ztli;
+        self.s.server.tli = msg.tli;
+        self.s.server.wal_seg_size = msg.wal_seg_size;
+        self.s.proposer_uuid = msg.proposer_id;
+        self.storage.persist(&self.s, true)?;
+
+        info!(
+            "processed greeting from proposer {:?}, sending term {:?}",
+            msg.proposer_id, self.s.acceptor_state.term
+        );
+        Ok(AcceptorProposerMessage::Greeting(AcceptorGreeting {
+            term: self.s.acceptor_state.term,
+        }))
+    }
+
+    /// Give vote for the given term, if we haven't done that previously.
+    fn handle_vote_request(&mut self, msg: &VoteRequest) -> Result<AcceptorProposerMessage> {
+        // initialize with refusal
+        let mut resp = VoteResponse {
+            term: msg.term,
+            vote_given: false as u64,
+            epoch: 0,
+            flush_lsn: Lsn(0),
+            restart_lsn: Lsn(0),
+        };
+        if self.s.acceptor_state.term < msg.term {
+            self.s.acceptor_state.term = msg.term;
+            // persist vote before sending it out
+            self.storage.persist(&self.s, true)?;
+            resp.vote_given = true as u64;
+            resp.epoch = self.s.acceptor_state.epoch;
+            resp.flush_lsn = self.flush_lsn;
+            resp.restart_lsn = self.s.truncate_lsn;
+        }
+        info!("processed VoteRequest for term {}: {:?}", msg.term, &resp);
+        Ok(AcceptorProposerMessage::VoteResponse(resp))
+    }
+
+    /// Handle request to append WAL.
+    fn handle_append_request(&mut self, msg: &AppendRequest) -> Result<AcceptorProposerMessage> {
+        // log first AppendRequest from this proposer
+        if self.elected_proposer_term < msg.h.term {
+            info!(
+                "start receiving WAL from timeline {} term {}",
+                self.s.server.ztli, msg.h.term,
+            );
+            self.elected_proposer_term = msg.h.term;
+        }
+
+        // If our term is lower than elected proposer one, bump it.
+        if self.s.acceptor_state.term < msg.h.term {
+            self.s.acceptor_state.term = msg.h.term;
+            self.storage.persist(&self.s, true)?;
+        }
+        // OTOH, if it is higher, immediately refuse the message.
+        else if self.s.acceptor_state.term > msg.h.term {
+            let resp = AppendResponse {
+                term: self.s.acceptor_state.term,
+                epoch: self.s.acceptor_state.epoch,
+                flush_lsn: Lsn(0),
+                hs_feedback: HotStandbyFeedback::empty(),
+            };
+            return Ok(AcceptorProposerMessage::AppendResponse(resp));
+        }
+
+        // do the job
+        self.storage
+            .write_wal(&self.s, msg.h.begin_lsn, &msg.wal_data)?;
+        let mut sync_control_file = false;
+        /*
+         * Epoch switch happen when written WAL record cross the boundary.
+         * The boundary is maximum of last WAL position at this node (FlushLSN) and global
+         * maximum (vcl) determined by WAL proposer during handshake.
+         * Switching epoch means that node completes recovery and start writing in the WAL new data.
+         * XXX: this is wrong, we must actively truncate not matching part of log.
+         */
+        if self.s.acceptor_state.epoch < msg.h.term
+            && msg.h.end_lsn > max(self.flush_lsn, msg.h.epoch_start_lsn)
+        {
+            info!("switched to new epoch {}", msg.h.term);
+            self.s.acceptor_state.epoch = msg.h.term; /* bump epoch */
+            sync_control_file = true;
+        }
+        if msg.h.end_lsn > self.flush_lsn {
+            self.flush_lsn = msg.h.end_lsn;
+        }
+
+        self.s.proposer_uuid = msg.h.proposer_uuid;
+        self.s.commit_lsn = msg.h.commit_lsn;
+        self.s.truncate_lsn = msg.h.restart_lsn;
+
+        /*
+         * Update restart LSN in control file.
+         * To avoid negative impact on performance of extra fsync, do it only
+         * when restart_lsn delta exceeds WAL segment size.
+         */
+        sync_control_file |=
+            self.flushed_truncate_lsn + (self.s.server.wal_seg_size as u64) < self.s.truncate_lsn;
+        self.storage.persist(&self.s, sync_control_file)?;
+        if sync_control_file {
+            self.flushed_truncate_lsn = self.s.truncate_lsn;
+        }
+
+        let resp = AppendResponse {
+            term: self.s.acceptor_state.term,
+            epoch: self.s.acceptor_state.epoch,
+            flush_lsn: self.flush_lsn,
+            // will be filled by caller code to avoid bothering safekeeper
+            hs_feedback: HotStandbyFeedback::empty(),
+        };
+        trace!(
+            "processed AppendRequest of len {}, flush_lsn={:X}/{:>08X}, resp {:?}",
+            msg.wal_data.len(),
+            (self.flush_lsn.0 >> 32) as u32,
+            self.flush_lsn.0 as u32,
+            &resp,
+        );
+        Ok(AcceptorProposerMessage::AppendResponse(resp))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // fake storage for tests
+    struct InMemoryStorage {
+        persisted_state: SafeKeeperState,
+    }
+
+    impl Storage for InMemoryStorage {
+        fn persist(&mut self, s: &SafeKeeperState, _sync: bool) -> Result<()> {
+            self.persisted_state = s.clone();
+            Ok(())
+        }
+
+        fn write_wal(&mut self, _s: &SafeKeeperState, _startpos: Lsn, _buf: &[u8]) -> Result<()> {
+            Ok(())
+        }
+    }
+
+    #[test]
+    fn test_voting() {
+        let storage = InMemoryStorage {
+            persisted_state: SafeKeeperState::new(),
+        };
+        let mut sk = SafeKeeper::new(Lsn(0), 0, storage, SafeKeeperState::new());
+
+        // check voting for 1 is ok
+        let vote_request = ProposerAcceptorMessage::VoteRequest(VoteRequest { term: 1 });
+        let mut vote_resp = sk.process_msg(&vote_request);
+        match vote_resp.unwrap() {
+            AcceptorProposerMessage::VoteResponse(resp) => assert!(resp.vote_given != 0),
+            _ => assert!(false),
+        }
+
+        // reboot...
+        let state = sk.storage.persisted_state.clone();
+        let storage = InMemoryStorage {
+            persisted_state: state.clone(),
+        };
+        sk = SafeKeeper::new(Lsn(0), 0, storage, state);
+
+        // and ensure voting second time for 1 is not ok
+        vote_resp = sk.process_msg(&vote_request);
+        match vote_resp.unwrap() {
+            AcceptorProposerMessage::VoteResponse(resp) => assert!(resp.vote_given == 0),
+            _ => assert!(false),
+        }
+    }
+
+    #[test]
+    fn test_epoch_switch() {
+        let storage = InMemoryStorage {
+            persisted_state: SafeKeeperState::new(),
+        };
+        let mut sk = SafeKeeper::new(Lsn(0), 0, storage, SafeKeeperState::new());
+
+        let mut ar_hdr = AppendRequestHeader {
+            term: 1,
+            epoch_start_lsn: Lsn(2),
+            begin_lsn: Lsn(1),
+            end_lsn: Lsn(2),
+            commit_lsn: Lsn(0),
+            restart_lsn: Lsn(0),
+            proposer_uuid: [0; 16],
+        };
+        let mut append_request = AppendRequest {
+            h: ar_hdr.clone(),
+            wal_data: Bytes::from_static(b"b"),
+        };
+
+        // check that AppendRequest before VCL doesn't switch epoch
+        let resp = sk.process_msg(&ProposerAcceptorMessage::AppendRequest(append_request));
+        assert!(resp.is_ok());
+        assert!(sk.storage.persisted_state.acceptor_state.epoch == 0);
+
+        // but record after VCL does the switch
+        ar_hdr.begin_lsn = Lsn(2);
+        ar_hdr.end_lsn = Lsn(3);
+        append_request = AppendRequest {
+            h: ar_hdr.clone(),
+            wal_data: Bytes::from_static(b"b"),
+        };
+        let resp = sk.process_msg(&ProposerAcceptorMessage::AppendRequest(append_request));
+        assert!(resp.is_ok());
+        assert!(sk.storage.persisted_state.acceptor_state.epoch == 1);
+    }
+}
--- a/walkeeper/src/send_wal.rs
+++ b/walkeeper/src/send_wal.rs
@@ -2,17 +2,20 @@
 //! pageserver/any other consumer.
 //!

+use crate::receive_wal::ReceiveWalConn;
 use crate::replication::ReplicationConn;
 use crate::timeline::{Timeline, TimelineTools};
 use crate::WalAcceptorConf;
 use anyhow::{bail, Result};
 use bytes::Bytes;
-use pageserver::ZTimelineId;
 use std::str::FromStr;
 use std::sync::Arc;
 use zenith_utils::postgres_backend;
 use zenith_utils::postgres_backend::PostgresBackend;
 use zenith_utils::pq_proto::{BeMessage, FeStartupMessage, RowDescriptor};
+use zenith_utils::zid::ZTimelineId;
+
+use crate::timeline::CreateControlFile;

 /// Handler for streaming WAL from acceptor
 pub struct SendWalHandler {
@@ -20,6 +23,7 @@ pub struct SendWalHandler {
    pub conf: WalAcceptorConf,
    /// assigned application name
    pub appname: Option<String>,
+    pub timelineid: Option<ZTimelineId>,
    pub timeline: Option<Arc<Timeline>>,
 }

@@ -28,7 +32,7 @@ impl postgres_backend::Handler for SendWalHandler {
        match sm.params.get("ztimelineid") {
            Some(ref ztimelineid) => {
                let ztlid = ZTimelineId::from_str(ztimelineid)?;
-                self.timeline.set(ztlid)?;
+                self.timelineid = Some(ztlid);
            }
            _ => bail!("timelineid is required"),
        }
@@ -39,12 +43,31 @@ impl postgres_backend::Handler for SendWalHandler {
    }

    fn process_query(&mut self, pgb: &mut PostgresBackend, query_string: Bytes) -> Result<()> {
+        // START_WAL_PUSH is the only command that initializes the timeline
+        if self.timeline.is_none() {
+            if query_string.starts_with(b"START_WAL_PUSH") {
+                self.timeline.set(
+                    &self.conf,
+                    self.timelineid.unwrap(),
+                    CreateControlFile::True,
+                )?;
+            } else {
+                self.timeline.set(
+                    &self.conf,
+                    self.timelineid.unwrap(),
+                    CreateControlFile::False,
+                )?;
+            }
+        }
        if query_string.starts_with(b"IDENTIFY_SYSTEM") {
            self.handle_identify_system(pgb)?;
            Ok(())
        } else if query_string.starts_with(b"START_REPLICATION") {
            ReplicationConn::new(pgb).run(self, pgb, &query_string)?;
            Ok(())
+        } else if query_string.starts_with(b"START_WAL_PUSH") {
+            ReceiveWalConn::new(pgb)?.run(self)?;
+            Ok(())
        } else {
            bail!("Unexpected command {:?}", query_string);
        }
@@ -56,6 +79,7 @@ impl SendWalHandler {
        SendWalHandler {
            conf,
            appname: None,
+            timelineid: None,
            timeline: None,
        }
    }
@@ -64,7 +88,7 @@ impl SendWalHandler {
    /// Handle IDENTIFY_SYSTEM replication command
    ///
    fn handle_identify_system(&mut self, pgb: &mut PostgresBackend) -> Result<()> {
-        let (start_pos, timeline) = self.timeline.find_end_of_wal(&self.conf.data_dir, false);
+        let (start_pos, timeline) = self.timeline.get().get_end_of_wal();
        let lsn = start_pos.to_string();
        let tli = timeline.to_string();
        let sysid = self.timeline.get().get_info().server.system_id.to_string();
--- a/walkeeper/src/timeline.rs
+++ b/walkeeper/src/timeline.rs
@@ -1,50 +1,84 @@
-//! This module contains tools for managing timelines.
-//!
+//! This module contains timeline id -> safekeeper state map with file-backed
+//! persistence and support for interaction between sending and receiving wal.

 use anyhow::{bail, Result};
 use fs2::FileExt;
 use lazy_static::lazy_static;
 use log::*;
-use pageserver::ZTimelineId;
-use postgres_ffi::xlog_utils::{find_end_of_wal, TimeLineID};
+use postgres_ffi::xlog_utils::find_end_of_wal;
 use std::cmp::{max, min};
 use std::collections::HashMap;
 use std::fs::{self, File, OpenOptions};
-use std::io::{Seek, SeekFrom};
-use std::path::Path;
+use std::io::{Seek, SeekFrom, Write};
 use std::sync::{Arc, Condvar, Mutex};
 use zenith_utils::bin_ser::LeSer;
 use zenith_utils::lsn::Lsn;

-use crate::receive_wal::{SafeKeeperInfo, CONTROL_FILE_NAME, SK_FORMAT_VERSION, SK_MAGIC};
+use zenith_utils::zid::ZTimelineId;
+
 use crate::replication::{HotStandbyFeedback, END_REPLICATION_MARKER};
+use crate::safekeeper::{
+    AcceptorProposerMessage, ProposerAcceptorMessage, SafeKeeper, SafeKeeperState, Storage,
+    SK_FORMAT_VERSION, SK_MAGIC,
+};
 use crate::WalAcceptorConf;
+use postgres_ffi::xlog_utils::{XLogFileName, XLOG_BLCKSZ};
+
+const CONTROL_FILE_NAME: &str = "safekeeper.control";

 /// Shared state associated with database instance (tenant)
 #[derive(Debug)]
 struct SharedState {
-    /// quorum commit LSN
+    /// Safekeeper object
+    sk: SafeKeeper<FileStorage>,
+    /// opened file control file handle (needed to hold exlusive file lock)
+    control_file: File,
+    /// For receiving-sending wal cooperation
+    /// quorum commit LSN we've notified walsenders about
    commit_lsn: Lsn,
-    /// information about this safekeeper
-    info: SafeKeeperInfo,
-    /// opened file control file handle (needed to hold exlusive file lock
-    control_file: Option<File>,
    /// combined hot standby feedback from all replicas
    hs_feedback: HotStandbyFeedback,
 }

+// A named boolean.
+#[derive(Debug)]
+pub enum CreateControlFile {
+    True,
+    False,
+}
+
 impl SharedState {
-    fn new() -> Self {
-        Self {
+    /// Restore SharedState from control file. Locks the control file along the
+    /// way to prevent running more than one instance of safekeeper on the same
+    /// data dir.
+    /// If create=false and file doesn't exist, bails out.
+    fn create_restore(
+        conf: &WalAcceptorConf,
+        timelineid: ZTimelineId,
+        create: CreateControlFile,
+    ) -> Result<Self> {
+        let (cf, state) = SharedState::load_control_file(conf, timelineid, create)?;
+        let storage = FileStorage {
+            control_file: cf.try_clone()?,
+            conf: conf.clone(),
+        };
+        let (flush_lsn, tli) = if state.server.wal_seg_size != 0 {
+            let wal_dir = conf.data_dir.join(format!("{}", timelineid));
+            find_end_of_wal(&wal_dir, state.server.wal_seg_size as usize, true)
+        } else {
+            (0, 0)
+        };
+
+        Ok(Self {
            commit_lsn: Lsn(0),
-            info: SafeKeeperInfo::new(),
-            control_file: None,
+            sk: SafeKeeper::new(Lsn(flush_lsn), tli, storage, state),
+            control_file: cf,
            hs_feedback: HotStandbyFeedback {
                ts: 0,
                xmin: u64::MAX,
                catalog_xmin: u64::MAX,
            },
-        }
+        })
    }

    /// Accumulate hot standby feedbacks from replicas
@@ -54,80 +88,75 @@ impl SharedState {
        self.hs_feedback.ts = max(self.hs_feedback.ts, feedback.ts);
    }

-    /// Load and lock control file (prevent running more than one instance of safekeeper)
-    pub fn load_control_file(
-        &mut self,
+    /// Fetch and lock control file (prevent running more than one instance of safekeeper)
+    /// If create=false and file doesn't exist, bails out.
+    fn load_control_file(
        conf: &WalAcceptorConf,
        timelineid: ZTimelineId,
-    ) -> Result<()> {
-        if self.control_file.is_some() {
-            info!("control file for timeline {} is already open", timelineid);
-            return Ok(());
-        }
-
+        create: CreateControlFile,
+    ) -> Result<(File, SafeKeeperState)> {
        let control_file_path = conf
            .data_dir
            .join(timelineid.to_string())
            .join(CONTROL_FILE_NAME);
-        info!("loading control file {}", control_file_path.display());
-        match OpenOptions::new()
-            .read(true)
-            .write(true)
-            .create(true)
-            .open(&control_file_path)
-        {
-            Ok(file) => {
+        info!(
+            "loading control file {}, create={:?}",
+            control_file_path.display(),
+            create
+        );
+        let mut opts = OpenOptions::new();
+        opts.read(true).write(true);
+        if let CreateControlFile::True = create {
+            opts.create(true);
+        }
+        match opts.open(&control_file_path) {
+            Ok(mut file) => {
                // Lock file to prevent two or more active wal_acceptors
                match file.try_lock_exclusive() {
                    Ok(()) => {}
                    Err(e) => {
                        bail!(
-                            "Control file {:?} is locked by some other process: {}",
+                            "control file {:?} is locked by some other process: {}",
                            &control_file_path,
                            e
                        );
                    }
                }
-                self.control_file = Some(file);
-
-                let cfile_ref = self.control_file.as_mut().unwrap();
-                match SafeKeeperInfo::des_from(cfile_ref) {
-                    Err(e) => {
-                        warn!("read from {:?} failed: {}", control_file_path, e);
+                // Empty file is legit on 'create', don't try to deser from it.
+                if file.metadata().unwrap().len() == 0 {
+                    if let CreateControlFile::False = create {
+                        bail!("control file is empty");
                    }
-                    Ok(info) => {
-                        if info.magic != SK_MAGIC {
-                            bail!("Invalid control file magic: {}", info.magic);
+                    return Ok((file, SafeKeeperState::new()));
+                } else {
+                    match SafeKeeperState::des_from(&mut file) {
+                        Err(e) => {
+                            bail!("failed to read control file {:?}: {}", control_file_path, e);
                        }
-                        if info.format_version != SK_FORMAT_VERSION {
-                            bail!(
-                                "Incompatible format version: {} vs. {}",
-                                info.format_version,
-                                SK_FORMAT_VERSION
-                            );
+                        Ok(s) => {
+                            if s.magic != SK_MAGIC {
+                                bail!("bad control file magic: {}", s.magic);
+                            }
+                            if s.format_version != SK_FORMAT_VERSION {
+                                bail!(
+                                    "incompatible format version: {} vs. {}",
+                                    s.format_version,
+                                    SK_FORMAT_VERSION
+                                );
+                            }
+                            return Ok((file, s));
                        }
-                        self.info = info;
                    }
                }
            }
            Err(e) => {
-                panic!(
-                    "Failed to open control file {:?}: {}",
-                    &control_file_path, e
+                bail!(
+                    "failed to open control file {:?}: {}",
+                    &control_file_path,
+                    e
                );
            }
        }
-        Ok(())
-    }
-
-    pub fn save_control_file(&mut self, sync: bool) -> Result<()> {
-        let file = self.control_file.as_mut().unwrap();
-        file.seek(SeekFrom::Start(0))?;
-        self.info.ser_into(file)?;
-        if sync {
-            file.sync_all()?;
-        }
-        Ok(())
    }
 }

@@ -179,12 +208,32 @@ impl Timeline {
        self.notify_wal_senders(END_REPLICATION_MARKER);
    }

-    pub fn get_info(&self) -> SafeKeeperInfo {
-        return self.mutex.lock().unwrap().info.clone();
+    /// Pass arrived message to the safekeeper.
+    pub fn process_msg(&self, msg: &ProposerAcceptorMessage) -> Result<AcceptorProposerMessage> {
+        let mut rmsg: AcceptorProposerMessage;
+        let commit_lsn: Lsn;
+        {
+            let mut shared_state = self.mutex.lock().unwrap();
+            rmsg = shared_state.sk.process_msg(msg)?;
+            // locally available commit lsn. flush_lsn can be smaller than
+            // commit_lsn if we are catching up safekeeper.
+            commit_lsn = min(shared_state.sk.flush_lsn, shared_state.sk.s.commit_lsn);
+
+            // if this is AppendResponse, fill in proper hot standby feedback
+            match rmsg {
+                AcceptorProposerMessage::AppendResponse(ref mut resp) => {
+                    resp.hs_feedback = shared_state.hs_feedback.clone();
+                }
+                _ => (),
+            }
+        }
+        // Ping wal sender that new data might be available.
+        self.notify_wal_senders(commit_lsn);
+        Ok(rmsg)
    }

-    pub fn set_info(&self, info: &SafeKeeperInfo) {
-        self.mutex.lock().unwrap().info = info.clone();
+    pub fn get_info(&self) -> SafeKeeperState {
+        self.mutex.lock().unwrap().sk.s.clone()
    }

    // Accumulate hot standby feedbacks from replicas
@@ -198,43 +247,40 @@ impl Timeline {
        shared_state.hs_feedback.clone()
    }

-    pub fn load_control_file(&self, conf: &WalAcceptorConf) -> Result<()> {
-        let mut shared_state = self.mutex.lock().unwrap();
-        shared_state.load_control_file(conf, self.timelineid)
-    }
-
-    pub fn save_control_file(&self, sync: bool) -> Result<()> {
-        let mut shared_state = self.mutex.lock().unwrap();
-        shared_state.save_control_file(sync)
+    pub fn get_end_of_wal(&self) -> (Lsn, u32) {
+        let shared_state = self.mutex.lock().unwrap();
+        (shared_state.sk.flush_lsn, shared_state.sk.tli)
    }
 }

 // Utilities needed by various Connection-like objects
 pub trait TimelineTools {
-    fn set(&mut self, timeline_id: ZTimelineId) -> Result<()>;
+    fn set(
+        &mut self,
+        conf: &WalAcceptorConf,
+        timeline_id: ZTimelineId,
+        create: CreateControlFile,
+    ) -> Result<()>;
    fn get(&self) -> &Arc<Timeline>;
-    fn find_end_of_wal(&self, data_dir: &Path, precise: bool) -> (Lsn, TimeLineID);
 }

 impl TimelineTools for Option<Arc<Timeline>> {
-    fn set(&mut self, timeline_id: ZTimelineId) -> Result<()> {
+    fn set(
+        &mut self,
+        conf: &WalAcceptorConf,
+        timeline_id: ZTimelineId,
+        create: CreateControlFile,
+    ) -> Result<()> {
        // We will only set the timeline once. If it were to ever change,
        // anyone who cloned the Arc would be out of date.
        assert!(self.is_none());
-        *self = Some(GlobalTimelines::store(timeline_id)?);
+        *self = Some(GlobalTimelines::get(conf, timeline_id, create)?);
        Ok(())
    }

    fn get(&self) -> &Arc<Timeline> {
        self.as_ref().unwrap()
    }
-
-    /// Find last WAL record. If "precise" is false then just locate last partial segment
-    fn find_end_of_wal(&self, data_dir: &Path, precise: bool) -> (Lsn, TimeLineID) {
-        let seg_size = self.get().get_info().server.wal_seg_size as usize;
-        let (lsn, timeline) = find_end_of_wal(data_dir, seg_size, precise);
-        (Lsn(lsn), timeline)
-    }
 }

 lazy_static! {
@@ -246,22 +292,143 @@ lazy_static! {
 struct GlobalTimelines;

 impl GlobalTimelines {
-    /// Store a new timeline into the global TIMELINES map.
-    fn store(timeline_id: ZTimelineId) -> Result<Arc<Timeline>> {
+    /// Get a timeline with control file loaded from the global TIMELINES map.
+    /// If control file doesn't exist and create=false, bails out.
+    pub fn get(
+        conf: &WalAcceptorConf,
+        timeline_id: ZTimelineId,
+        create: CreateControlFile,
+    ) -> Result<Arc<Timeline>> {
        let mut timelines = TIMELINES.lock().unwrap();

        match timelines.get(&timeline_id) {
            Some(result) => Ok(Arc::clone(result)),
            None => {
-                info!("creating timeline dir {}", timeline_id);
+                info!(
+                    "creating timeline dir {}, create is {:?}",
+                    timeline_id, create
+                );
                fs::create_dir_all(timeline_id.to_string())?;

-                let shared_state = SharedState::new();
+                let shared_state = SharedState::create_restore(conf, timeline_id, create)?;

-                let new_tid = Arc::new(Timeline::new(timeline_id, shared_state));
-                timelines.insert(timeline_id, Arc::clone(&new_tid));
-                Ok(new_tid)
+                let new_tli = Arc::new(Timeline::new(timeline_id, shared_state));
+                timelines.insert(timeline_id, Arc::clone(&new_tli));
+                Ok(new_tli)
            }
        }
    }
 }
+
+#[derive(Debug)]
+struct FileStorage {
+    control_file: File,
+    conf: WalAcceptorConf,
+}
+
+impl Storage for FileStorage {
+    fn persist(&mut self, s: &SafeKeeperState, sync: bool) -> Result<()> {
+        self.control_file.seek(SeekFrom::Start(0))?;
+        s.ser_into(&mut self.control_file)?;
+        if sync {
+            self.control_file.sync_all()?;
+        }
+        Ok(())
+    }
+
+    fn write_wal(&mut self, s: &SafeKeeperState, startpos: Lsn, buf: &[u8]) -> Result<()> {
+        let mut bytes_left: usize = buf.len();
+        let mut bytes_written: usize = 0;
+        let mut partial;
+        let mut start_pos = startpos;
+        const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ];
+        let wal_seg_size = s.server.wal_seg_size as usize;
+        let ztli = s.server.ztli;
+
+        /* Extract WAL location for this block */
+        let mut xlogoff = start_pos.segment_offset(wal_seg_size) as usize;
+
+        while bytes_left != 0 {
+            let bytes_to_write;
+
+            /*
+             * If crossing a WAL boundary, only write up until we reach wal
+             * segment size.
+             */
+            if xlogoff + bytes_left > wal_seg_size {
+                bytes_to_write = wal_seg_size - xlogoff;
+            } else {
+                bytes_to_write = bytes_left;
+            }
+
+            /* Open file */
+            let segno = start_pos.segment_number(wal_seg_size);
+            // note: we basically don't support changing pg timeline
+            let wal_file_name = XLogFileName(s.server.tli, segno, wal_seg_size);
+            let wal_file_path = self
+                .conf
+                .data_dir
+                .join(ztli.to_string())
+                .join(wal_file_name.clone());
+            let wal_file_partial_path = self
+                .conf
+                .data_dir
+                .join(ztli.to_string())
+                .join(wal_file_name.clone() + ".partial");
+
+            {
+                let mut wal_file: File;
+                /* Try to open already completed segment */
+                if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_path) {
+                    wal_file = file;
+                    partial = false;
+                } else if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_partial_path)
+                {
+                    /* Try to open existed partial file */
+                    wal_file = file;
+                    partial = true;
+                } else {
+                    /* Create and fill new partial file */
+                    partial = true;
+                    match OpenOptions::new()
+                        .create(true)
+                        .write(true)
+                        .open(&wal_file_partial_path)
+                    {
+                        Ok(mut file) => {
+                            for _ in 0..(wal_seg_size / XLOG_BLCKSZ) {
+                                file.write_all(&ZERO_BLOCK)?;
+                            }
+                            wal_file = file;
+                        }
+                        Err(e) => {
+                            error!("Failed to open log file {:?}: {}", &wal_file_path, e);
+                            return Err(e.into());
+                        }
+                    }
+                }
+                wal_file.seek(SeekFrom::Start(xlogoff as u64))?;
+                wal_file.write_all(&buf[bytes_written..(bytes_written + bytes_to_write)])?;
+
+                // Flush file, if not said otherwise
+                if !self.conf.no_sync {
+                    wal_file.sync_all()?;
+                }
+            }
+            /* Write was successful, advance our position */
+            bytes_written += bytes_to_write;
+            bytes_left -= bytes_to_write;
+            start_pos += bytes_to_write as u64;
+            xlogoff += bytes_to_write;
+
+            /* Did we reach the end of a WAL segment? */
+            if start_pos.segment_offset(wal_seg_size) == 0 {
+                xlogoff = 0;
+                if partial {
+                    fs::rename(&wal_file_partial_path, &wal_file_path)?;
+                }
+            }
+        }
+        Ok(())
+    }
+}
--- a/walkeeper/src/wal_service.rs
+++ b/walkeeper/src/wal_service.rs
@@ -4,11 +4,9 @@
 //!
 use anyhow::Result;
 use log::*;
-use std::io::Read;
 use std::net::{TcpListener, TcpStream};
 use std::thread;

-use crate::receive_wal::ReceiveWalConn;
 use crate::send_wal::SendWalHandler;
 use crate::WalAcceptorConf;
 use zenith_utils::postgres_backend::{AuthType, PostgresBackend};
@@ -37,35 +35,15 @@ pub fn thread_main(conf: WalAcceptorConf) -> Result<()> {
    }
 }

-/// This is run by main_loop, inside a background thread.
+/// This is run by `thread_main` above, inside a background thread.
 ///
-fn handle_socket(mut socket: TcpStream, conf: WalAcceptorConf) -> Result<()> {
+fn handle_socket(socket: TcpStream, conf: WalAcceptorConf) -> Result<()> {
    socket.set_nodelay(true)?;

-    // Peek at the incoming data to see what protocol is being sent.
-    let peeked = peek_u32(&mut socket)?;
-    if peeked == 0 {
-        // Consume the 4 bytes we peeked at. This protocol begins after them.
-        socket.read_exact(&mut [0u8; 4])?;
-        ReceiveWalConn::new(socket, conf)?.run()?; // internal protocol between wal_proposer and wal_acceptor
-    } else {
-        let mut conn_handler = SendWalHandler::new(conf);
-        let mut pgbackend = PostgresBackend::new(socket, AuthType::Trust)?;
-        // libpq replication protocol between wal_acceptor and replicas/pagers
-        pgbackend.run(&mut conn_handler)?;
-    }
+    let mut conn_handler = SendWalHandler::new(conf);
+    let pgbackend = PostgresBackend::new(socket, AuthType::Trust)?;
+    // libpq replication protocol between wal_acceptor and replicas/pagers
+    pgbackend.run(&mut conn_handler)?;
+
    Ok(())
 }
-
-/// Fetch the first 4 bytes from the network (big endian), without consuming them.
-///
-/// This is used to help determine what protocol the peer is using.
-fn peek_u32(stream: &mut TcpStream) -> Result<u32> {
-    let mut buf = [0u8; 4];
-    loop {
-        let num_bytes = stream.peek(&mut buf)?;
-        if num_bytes == 4 {
-            return Ok(u32::from_be_bytes(buf));
-        }
-    }
-}
--- a/zenith/Cargo.toml
+++ b/zenith/Cargo.toml
@@ -12,7 +12,7 @@ anyhow = "1.0"
 serde_json = "1"
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }

-# FIXME: 'pageserver' is needed for ZTimelineId. Refactor
+# FIXME: 'pageserver' is needed for BranchInfo. Refactor
 pageserver = { path = "../pageserver" }
 control_plane = { path = "../control_plane" }
 postgres_ffi = { path = "../postgres_ffi" }
--- a/zenith/src/main.rs
+++ b/zenith/src/main.rs
@@ -4,13 +4,15 @@ use clap::{App, AppSettings, Arg, ArgMatches, SubCommand};
 use control_plane::compute::ComputeControlPlane;
 use control_plane::local_env::{self, LocalEnv};
 use control_plane::storage::PageServerNode;
-use pageserver::ZTenantId;
 use std::collections::btree_map::Entry;
 use std::collections::HashMap;
 use std::process::exit;
 use std::str::FromStr;
+use zenith_utils::auth::{encode_from_key_path, Claims, Scope};
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::{ZTenantId, ZTimelineId};

-use pageserver::{branches::BranchInfo, ZTimelineId};
+use pageserver::branches::BranchInfo;
 use zenith_utils::lsn::Lsn;

 ///
@@ -53,6 +55,12 @@ fn main() -> Result<()> {
                        .long("remote-pageserver")
                        .required(false)
                        .value_name("pageserver-url"),
+                )
+                .arg(
+                    Arg::with_name("enable-auth")
+                        .long("enable-auth")
+                        .takes_value(false)
+                        .help("Enable authentication using ZenithJWT")
                ),
        )
        .subcommand(
@@ -77,8 +85,18 @@ fn main() -> Result<()> {
                .setting(AppSettings::ArgRequiredElseHelp)
                .about("Manage postgres instances")
                .subcommand(SubCommand::with_name("list").arg(tenantid_arg.clone()))
-                .subcommand(SubCommand::with_name("create").arg(timeline_arg.clone()).arg(tenantid_arg.clone()))
-                .subcommand(SubCommand::with_name("start").arg(timeline_arg.clone()).arg(tenantid_arg.clone()))
+                .subcommand(SubCommand::with_name("create")
+                    .about("Create a postgres compute node")
+                    .arg(timeline_arg.clone()).arg(tenantid_arg.clone())
+                    .arg(
+                        Arg::with_name("config-only")
+                            .help("Don't do basebackup, create compute node with only config files")
+                            .long("config-only")
+                            .required(false)
+                    ))
+                .subcommand(SubCommand::with_name("start")
+                    .about("Start a postrges compute node.\n This command actually creates new node from scrath, but preserves existing config files")
+                    .arg(timeline_arg.clone()).arg(tenantid_arg.clone()))
                .subcommand(
                    SubCommand::with_name("stop")
                        .arg(timeline_arg.clone())
@@ -106,19 +124,18 @@ fn main() -> Result<()> {
                        ),
                ),
        )
-        .subcommand(
-            SubCommand::with_name("push")
-                .about("Push timeline to remote pageserver")
-                .arg(Arg::with_name("timeline").required(true))
-                .arg(Arg::with_name("remote").required(true)),
-        )
        .get_matches();

    // Create config file
-    if let ("init", Some(sub_args)) = matches.subcommand() {
+    if let ("init", Some(init_match)) = matches.subcommand() {
        let tenantid = ZTenantId::generate();
-        let pageserver_uri = sub_args.value_of("pageserver-url");
-        local_env::init(pageserver_uri, tenantid)
+        let pageserver_uri = init_match.value_of("pageserver-url");
+        let auth_type = if init_match.is_present("enable-auth") {
+            AuthType::ZenithJWT
+        } else {
+            AuthType::Trust
+        };
+        local_env::init(pageserver_uri, tenantid, auth_type)
            .with_context(|| "Failed to create config file")?;
    }

@@ -132,9 +149,12 @@ fn main() -> Result<()> {
    };

    match matches.subcommand() {
-        ("init", Some(_)) => {
+        ("init", Some(init_match)) => {
            let pageserver = PageServerNode::from_env(&env);
-            if let Err(e) = pageserver.init(Some(&env.tenantid.to_string())) {
+            if let Err(e) = pageserver.init(
+                Some(&env.tenantid.to_string()),
+                init_match.is_present("enable-auth"),
+            ) {
                eprintln!("pageserver init failed: {}", e);
                exit(1);
            }
@@ -189,7 +209,7 @@ fn main() -> Result<()> {

        ("pg", Some(pg_match)) => {
            if let Err(e) = handle_pg(pg_match, &env) {
-                eprintln!("pg operation failed: {}", e);
+                eprintln!("pg operation failed: {:?}", e);
                exit(1);
            }
        }
@@ -201,13 +221,6 @@ fn main() -> Result<()> {
            }
        }

-        ("push", Some(push_match)) => {
-            if let Err(e) = handle_push(push_match, &env) {
-                eprintln!("push operation failed: {}", e);
-                exit(1);
-            }
-        }
-
        _ => {}
    };

@@ -336,7 +349,7 @@ fn get_branch_infos(
    tenantid: &ZTenantId,
 ) -> Result<HashMap<ZTimelineId, BranchInfo>> {
    let page_server = PageServerNode::from_env(env);
-    let branch_infos: Vec<BranchInfo> = page_server.branches_list(tenantid)?;
+    let branch_infos: Vec<BranchInfo> = page_server.branch_list(tenantid)?;
    let branch_infos: HashMap<ZTimelineId, BranchInfo> = branch_infos
        .into_iter()
        .map(|branch_info| (branch_info.timeline_id, branch_info))
@@ -349,7 +362,7 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &local_env::LocalEnv) -> Result
    let pageserver = PageServerNode::from_env(&env);
    match tenant_match.subcommand() {
        ("list", Some(_)) => {
-            for tenant in pageserver.tenants_list()? {
+            for tenant in pageserver.tenant_list()? {
                println!("{}", tenant);
            }
        }
@@ -359,7 +372,7 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &local_env::LocalEnv) -> Result
                None => ZTenantId::generate(),
            };
            println!("using tenant id {}", tenantid);
-            pageserver.tenant_create(&tenantid)?;
+            pageserver.tenant_create(tenantid)?;
            println!("tenant successfully created on the pageserver");
        }
        _ => {}
@@ -389,7 +402,7 @@ fn handle_branch(branch_match: &ArgMatches, env: &local_env::LocalEnv) -> Result
            .value_of("tenantid")
            .map_or(Ok(env.tenantid), |value| value.parse())?;
        // No arguments, list branches for tenant
-        let branches = pageserver.branches_list(&tenantid)?;
+        let branches = pageserver.branch_list(&tenantid)?;
        print_branches_tree(branches)?;
    }

@@ -434,10 +447,9 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
                .value_of("tenantid")
                .map_or(Ok(env.tenantid), |value| value.parse())?;
            let timeline_name = create_match.value_of("timeline").unwrap_or("main");
-            // check is that timeline doesnt already exist
-            // this check here is because it
+            let config_only = create_match.is_present("config-only");

-            cplane.new_node(tenantid, timeline_name)?;
+            cplane.new_node(tenantid, timeline_name, config_only)?;
        }
        ("start", Some(start_match)) => {
            let tenantid: ZTenantId = start_match
@@ -447,12 +459,19 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {

            let node = cplane.nodes.get(&(tenantid, timeline_name.to_owned()));

+            let auth_token = if matches!(env.auth_type, AuthType::ZenithJWT) {
+                let claims = Claims::new(Some(tenantid), Scope::Tenant);
+                Some(encode_from_key_path(&claims, &env.private_key_path)?)
+            } else {
+                None
+            };
+
            println!("Starting postgres on timeline {}...", timeline_name);
            if let Some(node) = node {
-                node.start()?;
+                node.start(&auth_token)?;
            } else {
-                let node = cplane.new_node(tenantid, timeline_name)?;
-                node.start()?;
+                let node = cplane.new_node(tenantid, timeline_name, false)?;
+                node.start(&auth_token)?;
            }
        }
        ("stop", Some(stop_match)) => {
@@ -500,20 +519,3 @@ fn handle_remote(remote_match: &ArgMatches, local_env: &LocalEnv) -> Result<()>

    Ok(())
 }
-
-fn handle_push(push_match: &ArgMatches, local_env: &LocalEnv) -> Result<()> {
-    let timeline_id_str = push_match.value_of("timeline").unwrap();
-    ZTimelineId::from_str(timeline_id_str)?;
-
-    let remote_name = push_match.value_of("remote").unwrap();
-    let remote = local_env
-        .remotes
-        .get(remote_name)
-        .ok_or_else(|| anyhow!("remote {} not found", remote_name))?;
-
-    let page_server = PageServerNode::from_env(local_env);
-    let mut client = page_server.page_server_psql_client()?;
-    client.simple_query(&format!("request_push {} {}", timeline_id_str, remote))?;
-
-    Ok(())
-}
--- a/zenith_metrics/Cargo.toml
+++ b/zenith_metrics/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "zenith_metrics"
+version = "0.1.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+prometheus = "0.12"
+
+[dev-dependencies]
+lazy_static = "1.4.0"
--- a/zenith_metrics/src/lib.rs
+++ b/zenith_metrics/src/lib.rs
@@ -0,0 +1,16 @@
+//! We re-export those from prometheus crate to
+//! make sure that we use the same dep version everywhere.
+//! Otherwise, we might not see all metrics registered via
+//! a default registry.
+pub use prometheus::gather;
+pub use prometheus::{exponential_buckets, linear_buckets};
+pub use prometheus::{register_histogram, Histogram};
+pub use prometheus::{register_histogram_vec, HistogramVec};
+pub use prometheus::{register_int_counter, IntCounter};
+pub use prometheus::{register_int_counter_vec, IntCounterVec};
+pub use prometheus::{register_int_gauge, IntGauge};
+pub use prometheus::{register_int_gauge_vec, IntGaugeVec};
+pub use prometheus::{Encoder, TextEncoder};
+
+mod wrappers;
+pub use wrappers::{CountedReader, CountedWriter};
--- a/zenith_metrics/src/wrappers.rs
+++ b/zenith_metrics/src/wrappers.rs
@@ -0,0 +1,211 @@
+use std::io::{Read, Result, Write};
+
+/// A wrapper for an object implementing [Read](std::io::Read)
+/// which allows a closure to observe the amount of bytes read.
+/// This is useful in conjunction with metrics (e.g. [IntCounter](crate::IntCounter)).
+///
+/// Example:
+///
+/// ```
+/// # use std::io::{Result, Read};
+/// # use zenith_metrics::{register_int_counter, IntCounter};
+/// # use zenith_metrics::CountedReader;
+/// #
+/// # lazy_static::lazy_static! {
+/// #     static ref INT_COUNTER: IntCounter = register_int_counter!(
+/// #         "int_counter",
+/// #         "let's count something!"
+/// #     ).unwrap();
+/// # }
+/// #
+/// fn do_some_reads(stream: impl Read, count: usize) -> Result<Vec<u8>> {
+///     let mut reader = CountedReader::new(stream, |cnt| {
+///         // bump a counter each time we do a read
+///         INT_COUNTER.inc_by(cnt as u64);
+///     });
+///
+///     let mut proto_header = [0; 8];
+///     reader.read_exact(&mut proto_header)?;
+///     assert!(&proto_header == b"deadbeef");
+///
+///     let mut payload = vec![0; count];
+///     reader.read_exact(&mut payload)?;
+///     Ok(payload)
+/// }
+/// ```
+///
+/// NB: rapid concurrent bumping of an atomic counter might incur
+/// a performance penalty. Please make sure to amortize the amount
+/// of atomic operations by either using [BufReader](std::io::BufReader)
+/// or choosing a non-atomic (thread local) counter.
+pub struct CountedReader<'a, T> {
+    reader: T,
+    update_counter: Box<dyn FnMut(usize) + Sync + Send + 'a>,
+}
+
+impl<'a, T> CountedReader<'a, T> {
+    pub fn new(reader: T, update_counter: impl FnMut(usize) + Sync + Send + 'a) -> Self {
+        Self {
+            reader,
+            update_counter: Box::new(update_counter),
+        }
+    }
+
+    /// Get an immutable reference to the underlying [Read](std::io::Read) implementor
+    pub fn inner(&self) -> &T {
+        &self.reader
+    }
+
+    /// Get a mutable reference to the underlying [Read](std::io::Read) implementor
+    pub fn inner_mut(&mut self) -> &mut T {
+        &mut self.reader
+    }
+
+    /// Consume the wrapper and return the underlying [Read](std::io::Read) implementor
+    pub fn into_inner(self) -> T {
+        self.reader
+    }
+}
+
+impl<T: Read> Read for CountedReader<'_, T> {
+    fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
+        let count = self.reader.read(buf)?;
+        (self.update_counter)(count);
+        Ok(count)
+    }
+}
+
+/// A wrapper for an object implementing [Write](std::io::Write)
+/// which allows a closure to observe the amount of bytes written.
+/// This is useful in conjunction with metrics (e.g. [IntCounter](crate::IntCounter)).
+///
+/// Example:
+///
+/// ```
+/// # use std::io::{Result, Write};
+/// # use zenith_metrics::{register_int_counter, IntCounter};
+/// # use zenith_metrics::CountedWriter;
+/// #
+/// # lazy_static::lazy_static! {
+/// #     static ref INT_COUNTER: IntCounter = register_int_counter!(
+/// #         "int_counter",
+/// #         "let's count something!"
+/// #     ).unwrap();
+/// # }
+/// #
+/// fn do_some_writes(stream: impl Write, payload: &[u8]) -> Result<()> {
+///     let mut writer = CountedWriter::new(stream, |cnt| {
+///         // bump a counter each time we do a write
+///         INT_COUNTER.inc_by(cnt as u64);
+///     });
+///
+///     let proto_header = b"deadbeef";
+///     writer.write_all(proto_header)?;
+///     writer.write_all(payload)
+/// }
+/// ```
+///
+/// NB: rapid concurrent bumping of an atomic counter might incur
+/// a performance penalty. Please make sure to amortize the amount
+/// of atomic operations by either using [BufWriter](std::io::BufWriter)
+/// or choosing a non-atomic (thread local) counter.
+pub struct CountedWriter<'a, T> {
+    writer: T,
+    update_counter: Box<dyn FnMut(usize) + Sync + Send + 'a>,
+}
+
+impl<'a, T> CountedWriter<'a, T> {
+    pub fn new(writer: T, update_counter: impl FnMut(usize) + Sync + Send + 'a) -> Self {
+        Self {
+            writer,
+            update_counter: Box::new(update_counter),
+        }
+    }
+
+    /// Get an immutable reference to the underlying [Write](std::io::Write) implementor
+    pub fn inner(&self) -> &T {
+        &self.writer
+    }
+
+    /// Get a mutable reference to the underlying [Write](std::io::Write) implementor
+    pub fn inner_mut(&mut self) -> &mut T {
+        &mut self.writer
+    }
+
+    /// Consume the wrapper and return the underlying [Write](std::io::Write) implementor
+    pub fn into_inner(self) -> T {
+        self.writer
+    }
+}
+
+impl<T: Write> Write for CountedWriter<'_, T> {
+    fn write(&mut self, buf: &[u8]) -> Result<usize> {
+        let count = self.writer.write(buf)?;
+        (self.update_counter)(count);
+        Ok(count)
+    }
+
+    fn flush(&mut self) -> Result<()> {
+        self.writer.flush()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_counted_reader() {
+        let stream = [0; 16];
+        let mut total = 0;
+        let mut reader = CountedReader::new(stream.as_ref(), |cnt| {
+            total += cnt;
+        });
+
+        let mut buffer = [0; 8];
+        reader.read_exact(&mut buffer).unwrap();
+        reader.read_exact(&mut buffer).unwrap();
+
+        drop(reader);
+        assert_eq!(total, stream.len());
+    }
+
+    #[test]
+    fn test_counted_writer() {
+        let mut stream = [0; 16];
+        let mut total = 0;
+        let mut writer = CountedWriter::new(stream.as_mut(), |cnt| {
+            total += cnt;
+        });
+
+        let buffer = [0; 8];
+        writer.write_all(&buffer).unwrap();
+        writer.write_all(&buffer).unwrap();
+
+        drop(writer);
+        assert_eq!(total, stream.len());
+    }
+
+    // This mimicks the constraints of std::thread::spawn
+    fn assert_send_sync(_x: impl Sync + Send + 'static) {}
+
+    #[test]
+    fn test_send_sync_counted_reader() {
+        let stream: &[u8] = &[];
+        let mut reader = CountedReader::new(stream, |_| {});
+
+        assert_send_sync(move || {
+            reader.read_exact(&mut []).unwrap();
+        });
+    }
+
+    #[test]
+    fn test_send_sync_counted_writer() {
+        let stream = Vec::<u8>::new();
+        let mut writer = CountedWriter::new(stream, |_| {});
+
+        assert_send_sync(move || {
+            writer.write_all(&[]).unwrap();
+        });
+    }
+}
--- a/zenith_utils/Cargo.toml
+++ b/zenith_utils/Cargo.toml
@@ -6,15 +6,24 @@ edition = "2018"

 [dependencies]
 anyhow = "1.0"
-bytes = "1.0.1"
-byteorder = "1.4.3"
-log = "0.4.14"
-serde = { version = "1.0", features = ["derive"] }
 bincode = "1.3"
-thiserror = "1.0"
+byteorder = "1.4.3"
+bytes = "1.0.1"
+hyper = { version = "0.14.7", features = ["full"] }
+lazy_static = "1.4.0"
+log = "0.4.14"
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
+routerify = "2"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1"
+thiserror = "1.0"
+tokio = { version = "1.5.0", features = ["full"] }
+
+zenith_metrics = { path = "../zenith_metrics" }
 workspace_hack = { path = "../workspace_hack" }
 rand = "0.8.3"
+jsonwebtoken = "7"
+hex = { version = "0.4.3", features = ["serde"] }

 [dev-dependencies]
 hex-literal = "0.3"
--- a/Show More
+++ b/Show More