Bump postgres version

2026-01-17 18:32:56 +00:00 · 2021-09-27 12:08:55 +03:00 · 2021-09-24 19:21:16 +03:00 · 2021-09-24 19:10:18 +03:00 · 2021-09-24 19:07:30 +03:00 · 2021-09-24 18:19:41 +03:00
128 changed files with 10966 additions and 6714 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -10,6 +10,16 @@ executors:
      - image: cimg/rust:1.52.1

 jobs:
+  check-codestyle:
+    executor: zenith-build-executor
+    steps:
+      - checkout
+
+      - run:
+          name: rustfmt
+          when: always
+          command: |
+            cargo fmt --all -- --check

  # A job to build postgres
  build-postgres:
@@ -118,6 +128,14 @@ jobs:
            - ~/.cargo/git
            - target

+        # Run style checks
+        # has to run separately from cargo fmt section
+        # since needs to run with dependencies
+      - run:
+          name: clippy
+          command: |
+            ./run_clippy.sh
+
        # Run rust unit tests
      - run: cargo test

@@ -183,6 +201,9 @@ jobs:
      needs_postgres_source:
        type: boolean
        default: false
+      run_in_parallel:
+        type: boolean
+        default: true
    steps:
      - attach_workspace:
          at: /tmp/zenith
@@ -211,6 +232,9 @@ jobs:
              echo "test_selection must be set"
              exit 1
            fi
+            if << parameters.run_in_parallel >>; then
+              EXTRA_PARAMS="-n4 $EXTRA_PARAMS"
+            fi;
            # Run the tests.
            #
            # The junit.xml file allows CircleCI to display more fine-grained test information
@@ -220,6 +244,7 @@ jobs:
            # --verbose prints name of each test (helpful when there are
            # multiple tests in one file)
            # -rA prints summary in the end
+            # -n4 uses four processes to run tests via pytest-xdist
            pipenv run pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short -s --verbose -rA $TEST_SELECTION $EXTRA_PARAMS
      - run:
          # CircleCI artifacts are preserved one file at a time, so skipping
@@ -229,7 +254,7 @@ jobs:
          when: always
          command: |
            du -sh /tmp/test_output/*
-            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "wal_acceptor.log" ! -name "regression.diffs" -delete
+            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "wal_acceptor.log" ! -name "regression.diffs" ! -name "junit.xml" ! -name "*.filediff" -delete
            du -sh /tmp/test_output/*
      - store_artifacts:
          path: /tmp/test_output
@@ -254,9 +279,55 @@ jobs:
            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
            docker build -t zenithdb/zenith:latest . && docker push zenithdb/zenith:latest

+  # Trigger a new remote CI job
+  remote-ci-trigger:
+    docker:
+      - image: cimg/base:2021.04
+    parameters:
+      remote_repo:
+        type: string
+    environment:
+      REMOTE_REPO: << parameters.remote_repo >>
+    steps:
+      - run:
+          name: Set PR's status to pending
+          command: |
+            LOCAL_REPO=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
+
+            curl -f -X POST \
+            https://api.github.com/repos/$LOCAL_REPO/statuses/$CIRCLE_SHA1 \
+            -H "Accept: application/vnd.github.v3+json" \
+            --user "$CI_ACCESS_TOKEN" \
+            --data \
+              "{
+                \"state\": \"pending\",
+                \"context\": \"zenith-remote-ci\",
+                \"description\": \"[$REMOTE_REPO] Remote CI job is about to start\"
+              }"
+      - run:
+          name: Request a remote CI test
+          command: |
+            LOCAL_REPO=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
+
+            curl -f -X POST \
+            https://api.github.com/repos/$REMOTE_REPO/actions/workflows/testing.yml/dispatches \
+            -H "Accept: application/vnd.github.v3+json" \
+            --user "$CI_ACCESS_TOKEN" \
+            --data \
+              "{
+                \"ref\": \"main\",
+                \"inputs\": {
+                  \"ci_job_name\": \"zenith-remote-ci\",
+                  \"commit_hash\": \"$CIRCLE_SHA1\",
+                  \"remote_repo\": \"$LOCAL_REPO\",
+                  \"zenith_image_branch\": \"$CIRCLE_BRANCH\"
+                }
+              }"
+
 workflows:
  build_and_test:
    jobs:
+      - check-codestyle
      - build-postgres
      - build-zenith:
          name: build-zenith-<< matrix.build_type >>
@@ -266,7 +337,7 @@ workflows:
          requires:
            - build-postgres
      - run-pytest:
-          name: pg_regress tests << matrix.build_type >>
+          name: pg_regress-tests-<< matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
@@ -275,13 +346,20 @@ workflows:
          requires:
            - build-zenith-<< matrix.build_type >>
      - run-pytest:
-          name: other tests << matrix.build_type >>
+          name: other-tests-<< matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
          test_selection: batch_others
          requires:
            - build-zenith-<< matrix.build_type >>
+      - run-pytest:
+          name: benchmarks
+          build_type: release
+          test_selection: performance
+          run_in_parallel: false
+          requires:
+            - build-zenith-release
      - docker-image:
          # Context gives an ability to login
          context: Docker Hub
@@ -291,5 +369,16 @@ workflows:
              only:
                - main
          requires:
-            - pg_regress tests release
-            - other tests release
+            - pg_regress-tests-release
+            - other-tests-release
+      - remote-ci-trigger:
+          # Context passes credentials for gh api
+          context: CI_ACCESS_TOKEN
+          remote_repo: "zenithdb/console"
+          requires:
+            # XXX: Successful build doesn't mean everything is OK, but
+            # the job to be triggered takes so much time to complete (~22 min)
+            # that it's better not to wait for the commented-out steps
+            - build-zenith-debug
+            # - pg_regress-tests-release
+            # - other-tests-release
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,4 +1,4 @@
 [submodule "vendor/postgres"]
 	path = vendor/postgres
-	url = https://github.com/libzenith/postgres
+	url = https://github.com/zenithdb/postgres
 	branch = main
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -154,9 +154,9 @@ dependencies = [

 [[package]]
 name = "bindgen"
-version = "0.57.0"
+version = "0.59.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd4865004a46a0aafb2a0a5eb19d3c9fc46ee5f063a6cfc605c69ac9ecf5263d"
+checksum = "453c49e5950bb0eb63bb3df640e31618846c89d5b7faa54040d76e98e0134375"
 dependencies = [
 "bitflags",
 "cexpr",
@@ -181,6 +181,18 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"

+[[package]]
+name = "bitvec"
+version = "0.19.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8942c8d352ae1838c9dda0b0ca2ab657696ef2232a20147cf1b30ae1a9cb4321"
+dependencies = [
+ "funty",
+ "radium",
+ "tap",
+ "wyz",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.9.0"
@@ -234,15 +246,12 @@ name = "cc"
 version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2"
-dependencies = [
- "jobserver",
-]

 [[package]]
 name = "cexpr"
-version = "0.4.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27"
+checksum = "db507a7679252d2276ed0dd8113c6875ec56d3089f9225b2b42c30cc1f8e5c89"
 dependencies = [
 "nom",
 ]
@@ -312,9 +321,11 @@ dependencies = [
 "postgres_ffi",
 "rand",
 "regex",
+ "reqwest",
 "serde",
 "serde_json",
 "tar",
+ "thiserror",
 "toml",
 "url",
 "walkeeper",
@@ -537,10 +548,10 @@ dependencies = [
 ]

 [[package]]
-name = "fs_extra"
-version = "1.2.0"
+name = "funty"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394"
+checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7"

 [[package]]
 name = "futures"
@@ -856,15 +867,6 @@ version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"

-[[package]]
-name = "jobserver"
-version = "0.1.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "972f5ae5d1cb9c6ae417789196c803205313edde988685da5e3aae0827b9e7fd"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "js-sys"
 version = "0.3.51"
@@ -902,9 +904,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"

 [[package]]
 name = "libc"
-version = "0.2.98"
+version = "0.2.101"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790"
+checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21"

 [[package]]
 name = "libloading"
@@ -916,18 +918,6 @@ dependencies = [
 "winapi",
 ]

-[[package]]
-name = "librocksdb-sys"
-version = "6.17.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5da125e1c0f22c7cae785982115523a0738728498547f415c9054cb17c7e89f9"
-dependencies = [
- "bindgen",
- "cc",
- "glob",
- "libc",
-]
-
 [[package]]
 name = "lock_api"
 version = "0.4.4"
@@ -1064,10 +1054,12 @@ dependencies = [

 [[package]]
 name = "nom"
-version = "5.1.2"
+version = "6.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af"
+checksum = "e7413f999671bd4745a7b624bd370a569fb6bc574b23c83a3c5ed2e453f3d5e2"
 dependencies = [
+ "bitvec",
+ "funty",
 "memchr",
 "version_check",
 ]
@@ -1181,6 +1173,7 @@ name = "pageserver"
 version = "0.1.0"
 dependencies = [
 "anyhow",
+ "async-trait",
 "bookfile",
 "byteorder",
 "bytes",
@@ -1188,34 +1181,27 @@ dependencies = [
 "clap",
 "crc32c",
 "daemonize",
- "fs_extra",
 "futures",
 "hex",
 "humantime",
+ "hyper",
 "lazy_static",
 "log",
 "postgres",
- "postgres-protocol 0.6.1 (git+https://github.com/zenithdb/rust-postgres.git?rev=9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858)",
- "postgres-types 0.2.1 (git+https://github.com/zenithdb/rust-postgres.git?rev=9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858)",
+ "postgres-protocol",
+ "postgres-types",
 "postgres_ffi",
 "rand",
 "regex",
- "rocksdb",
+ "routerify",
 "rust-s3",
 "scopeguard",
 "serde",
 "serde_json",
- "slog",
- "slog-async",
- "slog-scope",
- "slog-stdlog",
- "slog-term",
 "tar",
 "thiserror",
 "tokio",
- "tokio-stream",
 "toml",
- "walkdir",
 "workspace_hack",
 "zenith_metrics",
 "zenith_utils",
@@ -1314,27 +1300,9 @@ dependencies = [
 "fallible-iterator",
 "futures",
 "log",
- "postgres-protocol 0.6.1 (git+https://github.com/zenithdb/rust-postgres.git?rev=9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858)",
+ "postgres-protocol",
 "tokio",
- "tokio-postgres 0.7.1",
-]
-
-[[package]]
-name = "postgres-protocol"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff3e0f70d32e20923cabf2df02913be7c1842d4c772db8065c00fcfdd1d1bff3"
-dependencies = [
- "base64 0.13.0",
- "byteorder",
- "bytes",
- "fallible-iterator",
- "hmac",
- "md-5",
- "memchr",
- "rand",
- "sha2",
- "stringprep",
+ "tokio-postgres",
 ]

 [[package]]
@@ -1355,17 +1323,6 @@ dependencies = [
 "stringprep",
 ]

-[[package]]
-name = "postgres-types"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "430f4131e1b7657b0cd9a2b0c3408d77c9a43a042d300b8c77f981dffcc43a2f"
-dependencies = [
- "bytes",
- "fallible-iterator",
- "postgres-protocol 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "postgres-types"
 version = "0.2.1"
@@ -1373,7 +1330,7 @@ source = "git+https://github.com/zenithdb/rust-postgres.git?rev=9eb0dbfbeb6a6c1b
 dependencies = [
 "bytes",
 "fallible-iterator",
- "postgres-protocol 0.6.1 (git+https://github.com/zenithdb/rust-postgres.git?rev=9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858)",
+ "postgres-protocol",
 ]

 [[package]]
@@ -1392,6 +1349,7 @@ dependencies = [
 "memoffset",
 "rand",
 "regex",
+ "serde",
 "thiserror",
 "workspace_hack",
 "zenith_utils",
@@ -1435,16 +1393,9 @@ dependencies = [
 "lazy_static",
 "memchr",
 "parking_lot",
- "protobuf",
 "thiserror",
 ]

-[[package]]
-name = "protobuf"
-version = "2.24.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db50e77ae196458ccd3dc58a31ea1a90b0698ab1b7928d89f644c25d72070267"
-
 [[package]]
 name = "proxy"
 version = "0.1.0"
@@ -1455,10 +1406,11 @@ dependencies = [
 "hex",
 "md5",
 "rand",
+ "rustls",
 "serde",
 "serde_json",
 "tokio",
- "tokio-postgres 0.7.2",
+ "tokio-postgres",
 "zenith_utils",
 ]

@@ -1480,6 +1432,12 @@ dependencies = [
 "proc-macro2",
 ]

+[[package]]
+name = "radium"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8"
+
 [[package]]
 name = "rand"
 version = "0.8.4"
@@ -1541,9 +1499,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.5.4"
+version = "1.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
+checksum = "2a26af418b574bd56588335b3a3659a65725d4e636eb1016c2f9e3b38c7cc759"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -1616,13 +1574,16 @@ dependencies = [
 ]

 [[package]]
-name = "rocksdb"
-version = "0.16.0"
+name = "routerify"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c749134fda8bfc90d0de643d59bfc841dcb3ac8a1062e12b6754bd60235c48b3"
+checksum = "0c6bb49594c791cadb5ccfa5f36d41b498d40482595c199d10cd318800280bd9"
 dependencies = [
- "libc",
- "librocksdb-sys",
+ "http",
+ "hyper",
+ "lazy_static",
+ "percent-encoding",
+ "regex",
 ]

 [[package]]
@@ -1682,6 +1643,29 @@ dependencies = [
 "semver",
 ]

+[[package]]
+name = "rustls"
+version = "0.19.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7"
+dependencies = [
+ "base64 0.13.0",
+ "log",
+ "ring",
+ "sct",
+ "webpki",
+]
+
+[[package]]
+name = "rustls-split"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9d63f11490b4d8d45a362e171d7fe4a9ef154770a339e696a05eb354bc36837"
+dependencies = [
+ "rustls",
+ "webpki",
+]
+
 [[package]]
 name = "rustversion"
 version = "1.0.5"
@@ -1719,6 +1703,16 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"

+[[package]]
+name = "sct"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
 [[package]]
 name = "security-framework"
 version = "2.3.1"
@@ -1837,9 +1831,9 @@ dependencies = [

 [[package]]
 name = "shlex"
-version = "0.1.1"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
+checksum = "42a568c8f2cd051a4d283bd6eb0343ac214c1b0f1ac19f93e1175b2dee38c73d"

 [[package]]
 name = "signal-hook-registry"
@@ -1987,6 +1981,12 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60"

+[[package]]
+name = "tap"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
+
 [[package]]
 name = "tar"
 version = "0.4.35"
@@ -2098,9 +2098,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"

 [[package]]
 name = "tokio"
-version = "1.8.1"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98c8b05dc14c75ea83d63dd391100353789f5f24b8b3866542a5e85c8be8e985"
+checksum = "b4efe6fc2395938c8155973d7be49fe8d03a843726e285e100a8a383cc0154ce"
 dependencies = [
 "autocfg",
 "bytes",
@@ -2109,7 +2109,6 @@ dependencies = [
 "mio",
 "num_cpus",
 "once_cell",
- "parking_lot",
 "pin-project-lite",
 "signal-hook-registry",
 "tokio-macros",
@@ -2152,31 +2151,8 @@ dependencies = [
 "percent-encoding",
 "phf",
 "pin-project-lite",
- "postgres-protocol 0.6.1 (git+https://github.com/zenithdb/rust-postgres.git?rev=9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858)",
- "postgres-types 0.2.1 (git+https://github.com/zenithdb/rust-postgres.git?rev=9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858)",
- "socket2",
- "tokio",
- "tokio-util",
-]
-
-[[package]]
-name = "tokio-postgres"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d2b1383c7e4fb9a09e292c7c6afb7da54418d53b045f1c1fac7a911411a2b8b"
-dependencies = [
- "async-trait",
- "byteorder",
- "bytes",
- "fallible-iterator",
- "futures",
- "log",
- "parking_lot",
- "percent-encoding",
- "phf",
- "pin-project-lite",
- "postgres-protocol 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "postgres-types 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "postgres-protocol",
+ "postgres-types",
 "socket2",
 "tokio",
 "tokio-util",
@@ -2349,16 +2325,11 @@ dependencies = [
 "log",
 "pageserver",
 "postgres",
- "postgres-protocol 0.6.1 (git+https://github.com/zenithdb/rust-postgres.git?rev=9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858)",
+ "postgres-protocol",
 "postgres_ffi",
 "regex",
 "rust-s3",
 "serde",
- "slog",
- "slog-async",
- "slog-scope",
- "slog-stdlog",
- "slog-term",
 "tokio",
 "tokio-stream",
 "walkdir",
@@ -2460,6 +2431,16 @@ dependencies = [
 "wasm-bindgen",
 ]

+[[package]]
+name = "webpki"
+version = "0.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
 [[package]]
 name = "which"
 version = "3.1.1"
@@ -2531,6 +2512,12 @@ dependencies = [
 "syn",
 ]

+[[package]]
+name = "wyz"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214"
+
 [[package]]
 name = "xattr"
 version = "0.2.2"
@@ -2566,6 +2553,7 @@ name = "zenith_metrics"
 version = "0.1.0"
 dependencies = [
 "lazy_static",
+ "libc",
 "prometheus",
 ]

@@ -2585,9 +2573,19 @@ dependencies = [
 "log",
 "postgres",
 "rand",
+ "routerify",
+ "rustls",
+ "rustls-split",
 "serde",
+ "serde_json",
+ "slog",
+ "slog-async",
+ "slog-scope",
+ "slog-stdlog",
+ "slog-term",
 "thiserror",
 "tokio",
+ "webpki",
 "workspace_hack",
 "zenith_metrics",
 ]
--- a/2
+++ b/2
@@ -69,7 +69,7 @@ RUN rm -rf postgres_install/build && \
    cd postgres_install && tar -czf /data/postgres_install.tar.gz . && cd .. && \
    rm -rf postgres_install

-RUN useradd -m -d /data zenith
+RUN useradd -d /data zenith && chown -R zenith:zenith /data

 VOLUME ["/data"]
 USER zenith
--- a/12
+++ b/12
@@ -21,6 +21,7 @@ all: zenith postgres
 ### Zenith Rust bits
 #
 # The 'postgres_ffi' depends on the Postgres headers.
+.PHONY: zenith
 zenith: postgres-headers
 	cargo build

@@ -37,15 +38,18 @@ tmp_install/build/config.status:
 		--prefix=$(abspath tmp_install) > configure.log)

 # nicer alias for running 'configure'
+.PHONY: postgres-configure
 postgres-configure: tmp_install/build/config.status

 # Install the PostgreSQL header files into tmp_install/include
+.PHONY: postgres-headers
 postgres-headers: postgres-configure
 	+@echo "Installing PostgreSQL headers"
 	$(MAKE) -C tmp_install/build/src/include MAKELEVEL=0 install


 # Compile and install PostgreSQL and contrib/zenith
+.PHONY: postgres
 postgres: postgres-configure
 	+@echo "Compiling PostgreSQL"
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 install
@@ -67,4 +71,10 @@ distclean:
 	rm -rf tmp_install
 	cargo clean

-.PHONY: postgres-configure postgres postgres-headers zenith
+.PHONY: fmt
+fmt:
+	./pre-commit.py --fix-inplace
+
+.PHONY: setup-pre-commit-hook
+setup-pre-commit-hook:
+	ln -s -f ../../pre-commit.py .git/hooks/pre-commit
--- a/README.md
+++ b/README.md
@@ -2,6 +2,22 @@

 Zenith substitutes PostgreSQL storage layer and redistributes data across a cluster of nodes

+## Architecture overview
+
+A Zenith installation consists of Compute nodes and Storage engine.
+
+Compute nodes are stateless PostgreSQL nodes, backed by zenith storage.
+
+Zenith storage engine consists of two major components:
+- Pageserver. Scalable storage backend for compute nodes.
+- WAL service. The service that receives WAL from compute node and ensures that it is stored durably.
+
+Pageserver consists of:
+- Repository - Zenith storage implementation.
+- WAL receiver - service that receives WAL from WAL service and stores it in the repository.
+- Page service - service that communicates with compute nodes and responds with pages from the repository.
+- WAL redo - service that builds pages from base images and WAL records on Page service request.
+
 ## Running local installation

 1. Install build dependencies and other useful packages
@@ -51,7 +67,7 @@ main	127.0.0.1:55432	0/1609610	running

 4. Now it is possible to connect to postgres and run some queries:
 ```text
-> psql -p55432 -h 127.0.0.1 postgres
+> psql -p55432 -h 127.0.0.1 -U zenith_admin postgres
 postgres=# CREATE TABLE t(key int primary key, value text);
 CREATE TABLE
 postgres=# insert into t values(1,1);
@@ -81,7 +97,7 @@ waiting for server to start.... done

 # this new postgres instance will have all the data from 'main' postgres,
 # but all modifications would not affect data in original postgres
-> psql -p55433 -h 127.0.0.1 postgres
+> psql -p55433 -h 127.0.0.1 -U zenith_admin postgres
 postgres=# select * from t;
 key | value
 -----+-------
@@ -95,7 +111,7 @@ INSERT 0 1
 ## Running tests

 ```sh
-git clone --recursive https://github.com/libzenith/zenith.git
+git clone --recursive https://github.com/zenithdb/zenith.git
 make # builds also postgres and installs it to ./tmp_install
 cd test_runner
 pytest
@@ -105,52 +121,23 @@ pytest

 Now we use README files to cover design ideas and overall architecture for each module and `rustdoc` style documentation comments. See also [/docs/](/docs/) a top-level overview of all available markdown documentation.

+- [/docs/sourcetree.md](/docs/sourcetree.md) contains overview of source tree layout.
+
 To view your `rustdoc` documentation in a browser, try running `cargo doc --no-deps --open`

-## Source tree layout
+### Postgres-specific terms

-`/control_plane`:
+Due to Zenith's very close relation with PostgreSQL internals, there are numerous specific terms used.
+Same applies to certain spelling: i.e. we use MB to denote 1024 * 1024 bytes, while MiB would be technically more correct, it's inconsistent with what PostgreSQL code and its documentation use.

-Local control plane.
-Functions to start, configure and stop pageserver and postgres instances running as a local processes.
-Intended to be used in integration tests and in CLI tools for local installations.
+To get more familiar with this aspect, refer to:

-`/zenith`
+- [Zenith glossary](/docs/glossary.md)
+- [PostgreSQL glossary](https://www.postgresql.org/docs/13/glossary.html)
+- Other PostgreSQL documentation and sources (Zenith fork sources can be found [here](https://github.com/zenithdb/postgres))

-Main entry point for the 'zenith' CLI utility.
-TODO: Doesn't it belong to control_plane?
+## Join the development

-`/postgres_ffi`:
-
-Utility functions for interacting with PostgreSQL file formats.
-Misc constants, copied from PostgreSQL headers.
-
-`/zenith_utils`:
-
-Helpers that are shared between other crates in this repository.
-
-`/walkeeper`:
-
-WAL safekeeper (also known as WAL acceptor). Written in Rust.
-
-`/pageserver`:
-
-Page Server. Written in Rust.
-
-Depends on the modified 'postgres' binary for WAL redo.
-
-`/vendor/postgres`:
-
-PostgreSQL source tree, with the modifications needed for Zenith.
-
-`/vendor/postgres/contrib/zenith`:
-
-PostgreSQL extension that implements storage manager API and network communications with remote page server.
-
-`/test_runner`:
-
-Integration tests, written in Python using the `pytest` framework.
-
-`test_runner/zenith_regress`:
-
-Quick way to add new SQL regression test to integration tests set.
+- Read `CONTRIBUTING.md` to learn about project code style and practices.
+- To get familiar with a source tree layout, use [/docs/sourcetree.md](/docs/sourcetree.md).
+- To learn more about PostgreSQL internals, check http://www.interdb.jp/pg/index.html
--- a/control_plane/Cargo.toml
+++ b/control_plane/Cargo.toml
@@ -16,10 +16,12 @@ toml = "0.5"
 lazy_static = "1.4"
 regex = "1"
 anyhow = "1.0"
+thiserror = "1"
 bytes = "1.0.1"
 nix = "0.20"
 url = "2.2.2"
 hex = { version = "0.4.3", features = ["serde"] }
+reqwest = { version = "0.11", features = ["blocking", "json"] }

 pageserver = { path = "../pageserver" }
 walkeeper = { path = "../walkeeper" }
--- a/control_plane/src/compute.rs
+++ b/control_plane/src/compute.rs
@@ -1,20 +1,19 @@
+use std::fs::{self, File, OpenOptions};
 use std::io::Write;
 use std::net::SocketAddr;
 use std::net::TcpStream;
 use std::os::unix::fs::PermissionsExt;
-use std::process::Command;
+use std::process::{Command, Stdio};
+use std::str::FromStr;
 use std::sync::Arc;
 use std::time::Duration;
 use std::{collections::BTreeMap, path::PathBuf};
-use std::{
-    fs::{self, File, OpenOptions},
-    io::Read,
-};

 use anyhow::{Context, Result};
 use lazy_static::lazy_static;
 use regex::Regex;
 use zenith_utils::connstring::connection_host_port;
+use zenith_utils::lsn::Lsn;
 use zenith_utils::postgres_backend::AuthType;
 use zenith_utils::zid::ZTenantId;
 use zenith_utils::zid::ZTimelineId;
@@ -89,40 +88,31 @@ impl ComputeControlPlane {
        &mut self,
        tenantid: ZTenantId,
        branch_name: &str,
-        config_only: bool,
+        port: Option<u16>,
    ) -> Result<Arc<PostgresNode>> {
        let timeline_id = self
            .pageserver
            .branch_get_by_name(&tenantid, branch_name)?
            .timeline_id;

+        let port = port.unwrap_or_else(|| self.get_port());
        let node = Arc::new(PostgresNode {
            name: branch_name.to_owned(),
-            address: SocketAddr::new("127.0.0.1".parse().unwrap(), self.get_port()),
+            address: SocketAddr::new("127.0.0.1".parse().unwrap(), port),
            env: self.env.clone(),
            pageserver: Arc::clone(&self.pageserver),
            is_test: false,
            timelineid: timeline_id,
            tenantid,
+            uses_wal_proposer: false,
        });

-        node.init_from_page_server(self.env.auth_type, config_only)?;
+        node.create_pgdata()?;
+        node.setup_pg_conf(self.env.auth_type)?;
+
        self.nodes
            .insert((tenantid, node.name.clone()), Arc::clone(&node));

-        // Configure the node to stream WAL directly to the pageserver
-        node.append_conf(
-            "postgresql.conf",
-            format!(
-                concat!(
-                    "synchronous_standby_names = 'pageserver'\n", // TODO: add a new function arg?
-                    "zenith.callmemaybe_connstring = '{}'\n",     // FIXME escaping
-                ),
-                node.connstr(),
-            )
-            .as_str(),
-        )?;
-
        Ok(node)
    }
 }
@@ -138,6 +128,7 @@ pub struct PostgresNode {
    is_test: bool,
    pub timelineid: ZTimelineId,
    pub tenantid: ZTenantId,
+    uses_wal_proposer: bool,
 }

 impl PostgresNode {
@@ -222,6 +213,8 @@ impl PostgresNode {
            .parse()
            .with_context(|| err_msg)?;

+        let uses_wal_proposer = config.contains("wal_acceptors");
+
        // ok now
        Ok(PostgresNode {
            address: SocketAddr::new("127.0.0.1".parse().unwrap(), port),
@@ -231,71 +224,91 @@ impl PostgresNode {
            is_test: false,
            timelineid,
            tenantid,
+            uses_wal_proposer,
        })
    }

-    pub fn do_basebackup(&self) -> Result<()> {
-        let pgdata = self.pgdata();
+    fn sync_walkeepers(&self) -> Result<Lsn> {
+        let pg_path = self.env.pg_bin_dir().join("postgres");
+        let sync_handle = Command::new(pg_path)
+            .arg("--sync-safekeepers")
+            .env_clear()
+            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
+            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
+            .env("PGDATA", self.pgdata().to_str().unwrap())
+            .stdout(Stdio::piped())
+            // Comment this to avoid capturing stderr (useful if command hangs)
+            .stderr(Stdio::piped())
+            .spawn()
+            .expect("postgres --sync-safekeepers failed to start");
+
+        let sync_output = sync_handle
+            .wait_with_output()
+            .expect("postgres --sync-safekeepers failed");
+        if !sync_output.status.success() {
+            anyhow::bail!(
+                "sync-safekeepers failed: '{}'",
+                String::from_utf8_lossy(&sync_output.stderr)
+            );
+        }
+
+        let lsn = Lsn::from_str(std::str::from_utf8(&sync_output.stdout)?.trim())?;
+        println!("Walkeepers synced on {}", lsn);
+        Ok(lsn)
+    }
+
+    /// Get basebackup from the pageserver as a tar archive and extract it
+    /// to the `self.pgdata()` directory.
+    fn do_basebackup(&self, lsn: Option<Lsn>) -> Result<()> {
+        println!(
+            "Extracting base backup to create postgres instance: path={} port={}",
+            self.pgdata().display(),
+            self.address.port()
+        );
+
+        let sql = if let Some(lsn) = lsn {
+            format!("basebackup {} {} {}", self.tenantid, self.timelineid, lsn)
+        } else {
+            format!("basebackup {} {}", self.tenantid, self.timelineid)
+        };

-        let sql = format!("basebackup {} {}", self.tenantid, self.timelineid);
        let mut client = self
            .pageserver
            .page_server_psql_client()
            .with_context(|| "connecting to page server failed")?;

-        let mut copyreader = client
+        let copyreader = client
            .copy_out(sql.as_str())
            .with_context(|| "page server 'basebackup' command failed")?;

-        // FIXME: Currently, we slurp the whole tarball into memory, and then extract it,
-        // but we really should do this:
-        //let mut ar = tar::Archive::new(copyreader);
-        let mut buf = vec![];
-        copyreader
-            .read_to_end(&mut buf)
-            .with_context(|| "reading base backup from page server failed")?;
-        let mut ar = tar::Archive::new(buf.as_slice());
-        ar.unpack(&pgdata)
+        // Read the archive directly from the `CopyOutReader`
+        tar::Archive::new(copyreader)
+            .unpack(&self.pgdata())
            .with_context(|| "extracting page backup failed")?;

        Ok(())
    }

-    // Connect to a page server, get base backup, and untar it to initialize a
-    // new data directory
-    pub fn init_from_page_server(&self, auth_type: AuthType, config_only: bool) -> Result<()> {
-        let pgdata = self.pgdata();
-
-        println!(
-            "Extracting base backup to create postgres instance: path={} port={}",
-            pgdata.display(),
-            self.address.port()
-        );
-
-        // initialize data directory
-        if self.is_test {
-            fs::remove_dir_all(&pgdata).ok();
-        }
-
-        fs::create_dir_all(&pgdata)
-            .with_context(|| format!("could not create data directory {}", pgdata.display()))?;
-        fs::set_permissions(pgdata.as_path(), fs::Permissions::from_mode(0o700)).with_context(
-            || {
+    fn create_pgdata(&self) -> Result<()> {
+        fs::create_dir_all(&self.pgdata()).with_context(|| {
+            format!(
+                "could not create data directory {}",
+                self.pgdata().display()
+            )
+        })?;
+        fs::set_permissions(self.pgdata().as_path(), fs::Permissions::from_mode(0o700))
+            .with_context(|| {
                format!(
                    "could not set permissions in data directory {}",
-                    pgdata.display()
+                    self.pgdata().display()
                )
-            },
-        )?;
+            })
+    }

-        if config_only {
-            //Just create an empty config file
-            File::create(self.pgdata().join("postgresql.conf").to_str().unwrap())?;
-        } else {
-            self.do_basebackup()?;
-            fs::create_dir_all(self.pgdata().join("pg_wal"))?;
-            fs::create_dir_all(self.pgdata().join("pg_wal").join("archive_status"))?;
-        }
+    // Connect to a page server, get base backup, and untar it to initialize a
+    // new data directory
+    fn setup_pg_conf(&self, auth_type: AuthType) -> Result<()> {
+        File::create(self.pgdata().join("postgresql.conf").to_str().unwrap())?;

        // wal_log_hints is mandatory when running against pageserver (see gh issue#192)
        // TODO: is it possible to check wal_log_hints at pageserver side via XLOG_PARAMETER_CHANGE?
@@ -311,6 +324,8 @@ impl PostgresNode {
                 max_connections = 100\n\
                 wal_sender_timeout = 0\n\
                 wal_level = replica\n\
+                 zenith.file_cache_size = 4096\n\
+                 zenith.file_cache_path = '/tmp/file.cache'\n\
                 listen_addresses = '{address}'\n\
                 port = {port}\n",
                address = self.address.ip(),
@@ -320,7 +335,7 @@ impl PostgresNode {

        // Never clean up old WAL. TODO: We should use a replication
        // slot or something proper, to prevent the compute node
-        // from removing WAL that hasn't been streamed to the safekeepr or
+        // from removing WAL that hasn't been streamed to the safekeeper or
        // page server yet. (gh issue #349)
        self.append_conf("postgresql.conf", "wal_keep_size='10TB'\n")?;

@@ -332,7 +347,7 @@ impl PostgresNode {
        };

        // Configure that node to take pages from pageserver
-        let (host, port) = connection_host_port(&self.pageserver.connection_config);
+        let (host, port) = connection_host_port(&self.pageserver.pg_connection_config);
        self.append_conf(
            "postgresql.conf",
            format!(
@@ -349,6 +364,40 @@ impl PostgresNode {
            .as_str(),
        )?;

+        // Configure the node to stream WAL directly to the pageserver
+        self.append_conf(
+            "postgresql.conf",
+            format!(
+                concat!(
+                    "synchronous_standby_names = 'pageserver'\n", // TODO: add a new function arg?
+                    "zenith.callmemaybe_connstring = '{}'\n",     // FIXME escaping
+                ),
+                self.connstr(),
+            )
+            .as_str(),
+        )?;
+
+        Ok(())
+    }
+
+    fn load_basebackup(&self) -> Result<()> {
+        let lsn = if self.uses_wal_proposer {
+            // LSN 0 means that it is bootstrap and we need to download just
+            // latest data from the pageserver. That is a bit clumsy but whole bootstrap
+            // procedure evolves quite actively right now, so let's think about it again
+            // when things would be more stable (TODO).
+            let lsn = self.sync_walkeepers()?;
+            if lsn == Lsn(0) {
+                None
+            } else {
+                Some(lsn)
+            }
+        } else {
+            None
+        };
+
+        self.do_basebackup(lsn)?;
+
        Ok(())
    }

@@ -415,38 +464,22 @@ impl PostgresNode {
        }

        // 1. We always start compute node from scratch, so
-        // if old dir exists, preserve config files and drop the directory
-
-        // XXX Now we only use 'postgresql.conf'.
-        // If we will need 'pg_hba.conf', support it here too
-
+        // if old dir exists, preserve 'postgresql.conf' and drop the directory
        let postgresql_conf_path = self.pgdata().join("postgresql.conf");
-        let postgresql_conf = fs::read(postgresql_conf_path.clone()).with_context(|| {
+        let postgresql_conf = fs::read(&postgresql_conf_path).with_context(|| {
            format!(
                "failed to read config file in {}",
                postgresql_conf_path.to_str().unwrap()
            )
        })?;
-
-        println!(
-            "Destroying postgres data directory '{}'",
-            self.pgdata().to_str().unwrap()
-        );
        fs::remove_dir_all(&self.pgdata())?;
+        self.create_pgdata()?;

-        // 2. Create new node
-        self.init_from_page_server(self.env.auth_type, false)?;
+        // 2. Bring back config files
+        fs::write(&postgresql_conf_path, postgresql_conf)?;

-        // 3. Bring back config files
-
-        if let Ok(mut file) = OpenOptions::new()
-            .append(false)
-            .write(true)
-            .open(&postgresql_conf_path)
-        {
-            file.write_all(&postgresql_conf)?;
-            file.sync_all()?;
-        }
+        // 3. Load basebackup
+        self.load_basebackup()?;

        // 4. Finally start the compute node postgres
        println!("Starting postgres node at '{}'", self.connstr());
@@ -458,13 +491,22 @@ impl PostgresNode {
    }

    pub fn stop(&self, destroy: bool) -> Result<()> {
-        self.pg_ctl(&["-m", "immediate", "stop"], &None)?;
+        // If we are going to destroy data directory,
+        // use immediate shutdown mode, otherwise,
+        // shutdown gracefully to leave the data directory sane.
+        //
+        // Compute node always starts from scratch, so stop
+        // without destroy only used for testing and debugging.
+        //
        if destroy {
+            self.pg_ctl(&["-m", "immediate", "stop"], &None)?;
            println!(
                "Destroying postgres data directory '{}'",
                self.pgdata().to_str().unwrap()
            );
            fs::remove_dir_all(&self.pgdata())?;
+        } else {
+            self.pg_ctl(&["stop"], &None)?;
        }
        Ok(())
    }
--- a/control_plane/src/local_env.rs
+++ b/control_plane/src/local_env.rs
@@ -4,27 +4,24 @@
 // Now it also provides init method which acts like a stub for proper installation
 // script which will use local paths.
 //
-use anyhow::{anyhow, Context, Result};
-use hex;
+use anyhow::{Context, Result};
 use serde::{Deserialize, Serialize};
+use std::env;
 use std::fs;
 use std::path::PathBuf;
 use std::process::{Command, Stdio};
-use std::{collections::BTreeMap, env};
-use url::Url;
 use zenith_utils::auth::{encode_from_key_path, Claims, Scope};
 use zenith_utils::postgres_backend::AuthType;
 use zenith_utils::zid::ZTenantId;

-pub type Remotes = BTreeMap<String, String>;
-
 //
 // This data structures represent deserialized zenith CLI config
 //
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct LocalEnv {
-    // Pageserver connection strings
-    pub pageserver_connstring: String,
+    // Pageserver connection settings
+    pub pageserver_pg_port: u16,
+    pub pageserver_http_port: u16,

    // Base directory for both pageserver and compute nodes
    pub base_data_dir: PathBuf,
@@ -35,16 +32,13 @@ pub struct LocalEnv {
    // to four separate paths and match OS-specific installation layout.
    pub pg_distrib_dir: PathBuf,

-    // Path to pageserver binary. Empty for remote pageserver.
-    pub zenith_distrib_dir: Option<PathBuf>,
+    // Path to pageserver binary.
+    pub zenith_distrib_dir: PathBuf,

    // keeping tenant id in config to reduce copy paste when running zenith locally with single tenant
    #[serde(with = "hex")]
    pub tenantid: ZTenantId,

-    // Repository format, 'rocksdb' or 'layered' or None for default
-    pub repository_format: Option<String>,
-
    // jwt auth token used for communication with pageserver
    pub auth_token: String,

@@ -53,8 +47,6 @@ pub struct LocalEnv {

    // used to issue tokens during e.g pg start
    pub private_key_path: PathBuf,
-
-    pub remotes: Remotes,
 }

 impl LocalEnv {
@@ -67,11 +59,7 @@ impl LocalEnv {
    }

    pub fn pageserver_bin(&self) -> Result<PathBuf> {
-        Ok(self
-            .zenith_distrib_dir
-            .as_ref()
-            .ok_or_else(|| anyhow!("Can not manage remote pageserver"))?
-            .join("pageserver"))
+        Ok(self.zenith_distrib_dir.join("pageserver"))
    }

    pub fn pg_data_dirs_path(&self) -> PathBuf {
@@ -101,10 +89,10 @@ fn base_path() -> PathBuf {
 // Initialize a new Zenith repository
 //
 pub fn init(
-    remote_pageserver: Option<&str>,
+    pageserver_pg_port: u16,
+    pageserver_http_port: u16,
    tenantid: ZTenantId,
    auth_type: AuthType,
-    repository_format: Option<&str>,
 ) -> Result<()> {
    // check if config already exists
    let base_path = base_path();
@@ -169,41 +157,22 @@ pub fn init(
    let auth_token =
        encode_from_key_path(&Claims::new(None, Scope::PageServerApi), &private_key_path)?;

-    let conf = if let Some(addr) = remote_pageserver {
-        // check that addr is parsable
-        let _uri = Url::parse(addr).map_err(|e| anyhow!("{}: {}", addr, e))?;
+    // Find zenith binaries.
+    let zenith_distrib_dir = env::current_exe()?.parent().unwrap().to_owned();
+    if !zenith_distrib_dir.join("pageserver").exists() {
+        anyhow::bail!("Can't find pageserver binary.",);
+    }

-        LocalEnv {
-            pageserver_connstring: format!("postgresql://{}/", addr),
-            pg_distrib_dir,
-            zenith_distrib_dir: None,
-            base_data_dir: base_path,
-            remotes: BTreeMap::default(),
-            tenantid,
-            repository_format: repository_format.map(|x| x.into()),
-            auth_token,
-            auth_type,
-            private_key_path,
-        }
-    } else {
-        // Find zenith binaries.
-        let zenith_distrib_dir = env::current_exe()?.parent().unwrap().to_owned();
-        if !zenith_distrib_dir.join("pageserver").exists() {
-            anyhow::bail!("Can't find pageserver binary.",);
-        }
-
-        LocalEnv {
-            pageserver_connstring: "postgresql://127.0.0.1:6400".to_string(),
-            pg_distrib_dir,
-            zenith_distrib_dir: Some(zenith_distrib_dir),
-            base_data_dir: base_path,
-            remotes: BTreeMap::default(),
-            tenantid,
-            repository_format: repository_format.map(|x| x.into()),
-            auth_token,
-            auth_type,
-            private_key_path,
-        }
+    let conf = LocalEnv {
+        pageserver_pg_port,
+        pageserver_http_port,
+        pg_distrib_dir,
+        zenith_distrib_dir,
+        base_data_dir: base_path,
+        tenantid,
+        auth_token,
+        auth_type,
+        private_key_path,
    };

    fs::create_dir_all(conf.pg_data_dirs_path())?;
@@ -231,12 +200,3 @@ pub fn load_config() -> Result<LocalEnv> {
    let config = fs::read_to_string(repopath.join("config"))?;
    toml::from_str(config.as_str()).map_err(|e| e.into())
 }
-
-// Save config. We use that to change set of remotes from CLI itself.
-pub fn save_config(conf: &LocalEnv) -> Result<()> {
-    let config_path = base_path().join("config");
-    let conf_str = toml::to_string_pretty(conf)?;
-
-    fs::write(config_path, conf_str)?;
-    Ok(())
-}
--- a/control_plane/src/storage.rs
+++ b/control_plane/src/storage.rs
@@ -1,14 +1,19 @@
-use std::collections::HashMap;
+use std::io::Write;
 use std::net::TcpStream;
 use std::path::PathBuf;
 use std::process::Command;
-use std::thread;
 use std::time::Duration;
+use std::{io, result, thread};

-use anyhow::{anyhow, bail, Result};
+use anyhow::{anyhow, bail};
 use nix::sys::signal::{kill, Signal};
 use nix::unistd::Pid;
+use pageserver::http::models::{BranchCreateRequest, TenantCreateRequest};
 use postgres::{Config, NoTls};
+use reqwest::blocking::{Client, RequestBuilder, Response};
+use reqwest::{IntoUrl, Method};
+use thiserror::Error;
+use zenith_utils::http::error::HttpErrorBody;
 use zenith_utils::postgres_backend::AuthType;
 use zenith_utils::zid::ZTenantId;

@@ -17,6 +22,39 @@ use crate::read_pidfile;
 use pageserver::branches::BranchInfo;
 use zenith_utils::connstring::connection_address;

+#[derive(Error, Debug)]
+pub enum PageserverHttpError {
+    #[error("Reqwest error: {0}")]
+    Transport(#[from] reqwest::Error),
+
+    #[error("Error: {0}")]
+    Response(String),
+}
+
+type Result<T> = result::Result<T, PageserverHttpError>;
+
+pub trait ResponseErrorMessageExt: Sized {
+    fn error_from_body(self) -> Result<Self>;
+}
+
+impl ResponseErrorMessageExt for Response {
+    fn error_from_body(self) -> Result<Self> {
+        let status = self.status();
+        if !(status.is_client_error() || status.is_server_error()) {
+            return Ok(self);
+        }
+
+        // reqwest do not export it's error construction utility functions, so lets craft the message ourselves
+        let url = self.url().to_owned();
+        Err(PageserverHttpError::Response(
+            match self.json::<HttpErrorBody>() {
+                Ok(err_body) => format!("Error: {}", err_body.msg),
+                Err(_) => format!("Http error ({}) at {}.", status.as_u16(), url),
+            },
+        ))
+    }
+}
+
 //
 // Control routines for pageserver.
 //
@@ -25,13 +63,15 @@ use zenith_utils::connstring::connection_address;
 #[derive(Debug)]
 pub struct PageServerNode {
    pub kill_on_exit: bool,
-    pub connection_config: Config,
+    pub pg_connection_config: Config,
    pub env: LocalEnv,
+    pub http_client: Client,
+    pub http_base_url: String,
 }

 impl PageServerNode {
    pub fn from_env(env: &LocalEnv) -> PageServerNode {
-        let password = if matches!(env.auth_type, AuthType::ZenithJWT) {
+        let password = if env.auth_type == AuthType::ZenithJWT {
            &env.auth_token
        } else {
            ""
@@ -39,30 +79,36 @@ impl PageServerNode {

        PageServerNode {
            kill_on_exit: false,
-            connection_config: Self::default_config(password), // default
+            pg_connection_config: Self::pageserver_connection_config(
+                password,
+                env.pageserver_pg_port,
+            ),
            env: env.clone(),
+            http_client: Client::new(),
+            http_base_url: format!("http://localhost:{}/v1", env.pageserver_http_port),
        }
    }

-    fn default_config(password: &str) -> Config {
-        format!("postgresql://no_user:{}@localhost:64000/no_db", password)
+    fn pageserver_connection_config(password: &str, port: u16) -> Config {
+        format!("postgresql://no_user:{}@localhost:{}/no_db", password, port)
            .parse()
            .unwrap()
    }

-    pub fn init(
-        &self,
-        create_tenant: Option<&str>,
-        enable_auth: bool,
-        repository_format: Option<&str>,
-    ) -> Result<()> {
+    pub fn init(&self, create_tenant: Option<&str>, enable_auth: bool) -> anyhow::Result<()> {
        let mut cmd = Command::new(self.env.pageserver_bin()?);
+        let listen_pg = format!("localhost:{}", self.env.pageserver_pg_port);
+        let listen_http = format!("localhost:{}", self.env.pageserver_http_port);
        let mut args = vec![
            "--init",
            "-D",
            self.env.base_data_dir.to_str().unwrap(),
            "--postgres-distrib",
            self.env.pg_distrib_dir.to_str().unwrap(),
+            "--listen-pg",
+            &listen_pg,
+            "--listen-http",
+            &listen_http,
        ];

        if enable_auth {
@@ -70,11 +116,10 @@ impl PageServerNode {
            args.extend(&["--auth-type", "ZenithJWT"]);
        }

-        if let Some(repo_format) = repository_format {
-            args.extend(&["--repository-format", repo_format]);
+        if let Some(tenantid) = create_tenant {
+            args.extend(&["--create-tenant", tenantid])
        }

-        create_tenant.map(|tenantid| args.extend(&["--create-tenant", tenantid]));
        let status = cmd
            .args(args)
            .env_clear()
@@ -97,12 +142,13 @@ impl PageServerNode {
        self.repo_path().join("pageserver.pid")
    }

-    pub fn start(&self) -> Result<()> {
-        println!(
-            "Starting pageserver at '{}' in {}",
-            connection_address(&self.connection_config),
+    pub fn start(&self) -> anyhow::Result<()> {
+        print!(
+            "Starting pageserver at '{}' in '{}'",
+            connection_address(&self.pg_connection_config),
            self.repo_path().display()
        );
+        io::stdout().flush().unwrap();

        let mut cmd = Command::new(self.env.pageserver_bin()?);
        cmd.args(&["-D", self.repo_path().to_str().unwrap()])
@@ -119,25 +165,41 @@ impl PageServerNode {

        // It takes a while for the page server to start up. Wait until it is
        // open for business.
-        for retries in 1..15 {
-            match self.page_server_psql_client() {
+        const RETRIES: i8 = 15;
+        for retries in 1..RETRIES {
+            match self.check_status() {
                Ok(_) => {
-                    println!("Pageserver started");
+                    println!("\nPageserver started");
                    return Ok(());
                }
                Err(err) => {
-                    println!(
-                        "Pageserver not responding yet, err {} retrying ({})...",
-                        err, retries
-                    );
+                    match err {
+                        PageserverHttpError::Transport(err) => {
+                            if err.is_connect() && retries < 5 {
+                                print!(".");
+                                io::stdout().flush().unwrap();
+                            } else {
+                                if retries == 5 {
+                                    println!() // put a line break after dots for second message
+                                }
+                                println!(
+                                    "Pageserver not responding yet, err {} retrying ({})...",
+                                    err, retries
+                                );
+                            }
+                        }
+                        PageserverHttpError::Response(msg) => {
+                            bail!("pageserver failed to start: {} ", msg)
+                        }
+                    }
                    thread::sleep(Duration::from_secs(1));
                }
            }
        }
-        bail!("pageserver failed to start");
+        bail!("pageserver failed to start in {} seconds", RETRIES);
    }

-    pub fn stop(&self) -> Result<()> {
+    pub fn stop(&self) -> anyhow::Result<()> {
        let pid = read_pidfile(&self.pid_file())?;
        let pid = Pid::from_raw(pid);
        if kill(pid, Signal::SIGTERM).is_err() {
@@ -145,7 +207,7 @@ impl PageServerNode {
        }

        // wait for pageserver stop
-        let address = connection_address(&self.connection_config);
+        let address = connection_address(&self.pg_connection_config);
        for _ in 0..5 {
            let stream = TcpStream::connect(&address);
            thread::sleep(Duration::from_secs(1));
@@ -160,50 +222,59 @@ impl PageServerNode {
    }

    pub fn page_server_psql(&self, sql: &str) -> Vec<postgres::SimpleQueryMessage> {
-        let mut client = self.connection_config.connect(NoTls).unwrap();
+        let mut client = self.pg_connection_config.connect(NoTls).unwrap();

        println!("Pageserver query: '{}'", sql);
        client.simple_query(sql).unwrap()
    }

-    pub fn page_server_psql_client(&self) -> Result<postgres::Client, postgres::Error> {
-        self.connection_config.connect(NoTls)
+    pub fn page_server_psql_client(&self) -> result::Result<postgres::Client, postgres::Error> {
+        self.pg_connection_config.connect(NoTls)
    }

-    pub fn tenants_list(&self) -> Result<Vec<String>> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query("tenant_list")?;
-        let tenants_json = query_result
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
-            })
-            .flatten()
-            .ok_or_else(|| anyhow!("missing tenants"))?;
-
-        Ok(serde_json::from_str(tenants_json)?)
+    fn http_request<U: IntoUrl>(&self, method: Method, url: U) -> RequestBuilder {
+        let mut builder = self.http_client.request(method, url);
+        if self.env.auth_type == AuthType::ZenithJWT {
+            builder = builder.bearer_auth(&self.env.auth_token)
+        }
+        builder
    }

-    pub fn tenant_create(&self, tenantid: &ZTenantId) -> Result<()> {
-        let mut client = self.page_server_psql_client()?;
-        client.simple_query(format!("tenant_create {}", tenantid).as_str())?;
+    pub fn check_status(&self) -> Result<()> {
+        self.http_request(Method::GET, format!("{}/{}", self.http_base_url, "status"))
+            .send()?
+            .error_from_body()?;
        Ok(())
    }

-    pub fn branches_list(&self, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query(&format!("branch_list {}", tenantid))?;
-        let branches_json = query_result
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
-            })
-            .flatten()
-            .ok_or_else(|| anyhow!("missing branches"))?;
+    pub fn tenant_list(&self) -> Result<Vec<String>> {
+        Ok(self
+            .http_request(Method::GET, format!("{}/{}", self.http_base_url, "tenant"))
+            .send()?
+            .error_from_body()?
+            .json()?)
+    }

-        Ok(serde_json::from_str(branches_json)?)
+    pub fn tenant_create(&self, tenantid: ZTenantId) -> Result<()> {
+        Ok(self
+            .http_request(Method::POST, format!("{}/{}", self.http_base_url, "tenant"))
+            .json(&TenantCreateRequest {
+                tenant_id: tenantid,
+            })
+            .send()?
+            .error_from_body()?
+            .json()?)
+    }
+
+    pub fn branch_list(&self, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
+        Ok(self
+            .http_request(
+                Method::GET,
+                format!("{}/branch/{}", self.http_base_url, tenantid),
+            )
+            .send()?
+            .error_from_body()?
+            .json()?)
    }

    pub fn branch_create(
@@ -212,49 +283,31 @@ impl PageServerNode {
        startpoint: &str,
        tenantid: &ZTenantId,
    ) -> Result<BranchInfo> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query(
-            format!("branch_create {} {} {}", tenantid, branch_name, startpoint).as_str(),
-        )?;
-
-        let branch_json = query_result
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
+        Ok(self
+            .http_request(Method::POST, format!("{}/branch", self.http_base_url))
+            .json(&BranchCreateRequest {
+                tenant_id: tenantid.to_owned(),
+                name: branch_name.to_owned(),
+                start_point: startpoint.to_owned(),
            })
-            .flatten()
-            .ok_or_else(|| anyhow!("missing branch"))?;
-
-        let res: BranchInfo = serde_json::from_str(branch_json).map_err(|e| {
-            anyhow!(
-                "failed to parse branch_create response: {}: {}",
-                branch_json,
-                e
-            )
-        })?;
-
-        Ok(res)
+            .send()?
+            .error_from_body()?
+            .json()?)
    }

-    // TODO: make this a separate request type and avoid loading all the branches
    pub fn branch_get_by_name(
        &self,
        tenantid: &ZTenantId,
        branch_name: &str,
    ) -> Result<BranchInfo> {
-        let branch_infos = self.branches_list(tenantid)?;
-        let branche_by_name: Result<HashMap<String, BranchInfo>> = branch_infos
-            .into_iter()
-            .map(|branch_info| Ok((branch_info.name.clone(), branch_info)))
-            .collect();
-        let branche_by_name = branche_by_name?;
-
-        let branch = branche_by_name
-            .get(branch_name)
-            .ok_or_else(|| anyhow!("Branch {} not found", branch_name))?;
-
-        Ok(branch.clone())
+        Ok(self
+            .http_request(
+                Method::GET,
+                format!("{}/branch/{}/{}", self.http_base_url, tenantid, branch_name),
+            )
+            .send()?
+            .error_for_status()?
+            .json()?)
    }
 }

--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -1,4 +1,6 @@
 #!/bin/sh
+set -eux
+
 if [ "$1" = 'pageserver' ]; then
    if [ ! -d "/data/tenants" ]; then
        echo "Initializing pageserver data directory"
--- a/docs/README.md
+++ b/docs/README.md
@@ -4,8 +4,11 @@

 - [authentication.md](authentication.md) — pageserver JWT authentication.
 - [docker.md](docker.md) — Docker images and building pipeline.
+- [glossary.md](glossary.md) — Glossary of all the terms used in codebase.
 - [multitenancy.md](multitenancy.md) — how multitenancy is organized in the pageserver and Zenith CLI.
+- [sourcetree.md](sourcetree.md) — Overview of the source tree layeout.
 - [pageserver/README](/pageserver/README) — pageserver overview.
 - [postgres_ffi/README](/postgres_ffi/README) — Postgres FFI overview.
 - [test_runner/README.md](/test_runner/README.md) — tests infrastructure overview.
 - [walkeeper/README](/walkeeper/README.md) — WAL service overview.
+- [core_changes.md](core_changes.md) - Description of Zenith changes in Postgres core
--- a/docs/core_changes.md
+++ b/docs/core_changes.md
@@ -0,0 +1,202 @@
+1. Add t_cid to XLOG record
+- Why?
+  The cmin/cmax on a heap page is a real bummer. I don't see any other way to fix that than bite the bullet and modify the WAL-logging routine to include the cmin/cmax.
+
+  To recap, the problem is that the XLOG_HEAP_INSERT record does not include the command id of the inserted row. And same with deletion/update. So in the primary, a row is inserted with current xmin + cmin. But in the replica, the cmin is always set to 1. That works, because the command id is only relevant to the inserting transaction itself. After commit/abort, no one cares abut it anymore.
+
+- Alternatives?
+  I don't know
+
+2. Add PD_WAL_LOGGED.
+- Why?
+  Postgres sometimes writes data to the page before it is wal-logged. If such page ais swapped out, we  will loose this change. The problem is currently solved by setting PD_WAL_LOGGED bit in page header. When page without this bit set is written to the SMGR, then it is forced to be written to the WAL as FPI using log_newpage_copy() function.
+
+  There was wrong assumption that it can happen only during construction of some exotic indexes (like gist). It is not true. The same situation can happen with COPY,VACUUM and when record hint bits are set.
+
+- Discussion:
+  https://discord.com/channels/869525774699462656/882681420986851359
+
+- Alternatives:
+  Do not store this flag in page header, but associate this bit with shared buffer. Logically it is more correct but in practice we will get not advantages: neither in space, neither in CPU overhead.
+
+
+3. XLogReadBufferForRedo not always loads and pins requested buffer. So we need to add extra checks that buffer is really pinned. Also do not use BufferGetBlockNumber for buffer returned by XLogReadBufferForRedo.
+- Why?
+  XLogReadBufferForRedo is not pinning pages which are not requested by wal-redo. It is specific only for wal-redo Postgres.
+
+- Alternatives?
+  No
+
+
+4. Eliminate reporting of some warnings related with hint bits, for example
+"page is not marked all-visible but visibility map bit is set in relation".
+- Why?
+  Hint bit may be not WAL logged.
+
+- Alternative?
+  Always wal log any page changes.
+
+
+5. Maintain last written LSN.
+- Why?
+  When compute node requests page from page server, we need to specify LSN. Ideally it should be LSN
+  of WAL record performing last update of this pages. But we do not know it, because we do not have page.
+  We can use current WAL flush position, but in this case there is high probability that page server
+  will be blocked until this peace of WAL is delivered.
+  As better approximation we can keep max LSN of written page. It will be better to take in account LSNs only of evicted pages,
+  but SMGR API doesn't provide such knowledge.
+
+- Alternatives?
+  Maintain map of LSNs of evicted pages.
+
+
+6. Launching Postgres without WAL.
+- Why?
+  According to Zenith architecture compute node is stateless. So when we are launching
+  compute node, we need to provide some dummy PG_DATADIR. Relation pages
+  can be requested on demand from page server. But Postgres still need some non-relational data:
+  control and configuration files, SLRUs,...
+  It is currently implemented  using basebackup (do not mix with pg_basebackup) which is created
+  by pageserver. It includes in this tarball config/control files, SLRUs and required directories.
+  As far as pageserver do not have original (non-scattered) WAL segments, it includes in
+  this tarball dummy WAL segment which contains only SHUTDOWN_CHECKPOINT record at the beginning of segment,
+  which redo field points to the end of wal. It allows to load checkpoint record in more or less
+  standard way with minimal changes of Postgres, but then some special handling is needed,
+  including restoring previous record position from zenith.signal file.
+  Also we have to correctly initialize header of last WAL page (pointed by checkpoint.redo)
+  to pass checks performed by XLogReader.
+
+- Alternatives?
+  We may not include fake WAL segment in tarball at all and modify xlog.c to load checkpoint record
+  in special way. But it may only increase number of changes in xlog.c
+
+7. Add redo_read_buffer_filter callback to XLogReadBufferForRedoExtended
+- Why?
+  We need a way in wal-redo Postgres to ignore pages which are not requested by pageserver.
+  So wal-redo Postgres reconstructs only requested page and for all other returns BLK_DONE
+  which means that recovery for them is not needed.
+
+- Alternatives?
+  No
+
+8. Enforce WAL logging of sequence updates.
+- Why?
+  Due to performance reasons Postgres don't want to log each fetching of a value from a sequence,
+  so we pre-log a few fetches in advance. In the event of crash we can lose
+  (skip over) as many values as we pre-logged.
+  But it doesn't work with Zenith because page with sequence value can be evicted from buffer cache
+  and we will get a gap in sequence values even without crash.
+
+- Alternatives:
+  Do not try to preserve sequential order but avoid performance penalty.
+
+
+9. Treat unlogged tables as normal (permanent) tables.
+- Why?
+  Unlogged tables are not transient, so them have to survive node restart (unlike temporary tables).
+  But as far as compute node is stateless, we need to persist their data to storage node.
+  And it can only be done through the WAL.
+
+- Alternatives?
+  * Store unlogged tables locally (violates requirement of stateless compute nodes).
+  * Prohibit unlogged tables at all.
+
+
+10. Support start Postgres in wal-redo mode
+- Why?
+  To be able to apply WAL record and reconstruct pages at page server.
+
+- Alternatives?
+  * Rewrite redo handlers in Rust
+  * Do not reconstruct pages at page server at all and do it at compute node.
+
+
+11. WAL proposer
+- Why?
+  WAL proposer is communicating with safekeeper and ensures WAL durability by quorum writes.
+  It is currently implemented as patch to standard WAL sender.
+
+- Alternatives?
+  Can be moved to extension if some extra callbacks will be added to wal sender code.
+
+
+12. Secure Computing BPF API wrapper.
+- Why?
+  Pageserver delegates complex WAL decoding duties to Postgres,
+  which means that the latter might fall victim to carefully designed
+  malicious WAL records and start doing harmful things to the system.
+  To prevent this, it has been decided to limit possible interactions
+  with the outside world using the Secure Computing BPF mode.
+
+- Alternatives:
+  * Rewrite redo handlers in Rust.
+  * Add more checks to guarantee correctness of WAL records.
+  * Move seccomp.c to extension
+  * Many other discussed approaches to neutralize incorrect WAL records vulnerabilities.
+
+
+13. Callbacks for replica feedbacks
+- Why?
+  Allowing waproposer to interact with walsender code.
+
+- Alternatives
+  Copy walsender code to walproposer.
+
+
+14. Support multiple SMGR implementations.
+- Why?
+  Postgres provides abstract API for storage manager but it has only one implementation
+  and provides no way to replace it with custom storage manager.
+
+- Alternatives?
+  None.
+
+
+15. Calculate database size as sum of all database relations.
+- Why?
+  Postgres is calculating database size by traversing data directory
+  but as far as Zenith compute node is stateless we can not do it.
+
+- Alternatives?
+  Send this request directly to pageserver and calculate real (physical) size
+  of Zenith representation of database/timeline, rather than sum logical size of all relations.
+
+
+-----------------------------------------------
+Not currently committed but proposed:
+
+1. Disable ring buffer buffer manager strategies
+- Why?
+  Postgres tries to avoid cache flushing by bulk operations (copy, seqscan, vacuum,...).
+  Even if there are free space in buffer cache, pages may be evicted.
+  Negative effect of it can be somehow compensated by file system cache, but in case of Zenith
+  cost of requesting page from page server is much higher.
+
+- Alternatives?
+  Instead of just prohibiting ring buffer we may try to implement more flexible eviction policy,
+  for example copy evicted page from ring buffer to some other buffer if there is free space
+  in buffer cache.
+
+2. Disable marking page as dirty when hint bits are set.
+- Why?
+  Postgres has to modify page twice: first time when some tuple is updated and second time when
+  hint bits are set. Wal logging hint bits updates requires FPI which significantly increase size of WAL.
+
+- Alternatives?
+  Add special WAL record for setting page hints.
+
+3. Prefetching
+- Why?
+  As far as pages in Zenith are loaded on demand, to reduce node startup time
+  and also sppedup some massive queries we need some mechanism for bulk loading to
+  reduce page request round-trip overhead.
+
+  Currently Postgres is supporting prefetching only for bitmap scan.
+  In Zenith we also use prefetch for sequential and index scan. For sequential scan we prefetch
+  some number of following pages. For index scan we prefetch pages of heap relation addressed by TIDs.
+
+4. Prewarming.
+- Why?
+  Short downtime (or, in other words, fast compute node restart time) is one of the key feature of Zenith.
+  But overhead of request-response round-trip for loading pages on demand can make started node warm-up quite slow.
+  We can capture state of compute node buffer cache and send bulk request for this pages at startup.
--- a/docs/glossary.md
+++ b/docs/glossary.md
@@ -0,0 +1,218 @@
+# Glossary
+
+### Authentication
+
+### Base image (page image)
+
+### Basebackup
+
+A tarball with files needed to bootstrap a compute node[] and a corresponding command to create it.
+NOTE:It has nothing to do with PostgreSQL pg_basebackup.
+
+### Branch
+
+We can create branch at certain LSN using `zenith branch` command.
+Each Branch lives in a corresponding timeline[] and has an ancestor[].
+
+
+### Checkpoint (PostgreSQL)
+
+NOTE: This is an overloaded term.
+
+A checkpoint record in the WAL marks a point in the WAL sequence at which it is guaranteed that all data files have been updated with all information from shared memory modified before that checkpoint; 
+
+### Checkpoint (Layered repository)
+
+NOTE: This is an overloaded term.
+
+Whenever enough WAL has been accumulated in memory, the page server []
+writes out the changes from in-memory layers into new layer files[]. This process
+is called "checkpointing". The page server only creates layer files for
+relations that have been modified since the last checkpoint. 
+
+Configuration parameter `checkpoint_distance` defines the distance
+from current LSN to perform checkpoint of in-memory layers.
+Default is `DEFAULT_CHECKPOINT_DISTANCE`.
+Set this parameter to `0` to force checkpoint of every layer.
+
+Configuration parameter `checkpoint_period` defines the interval between checkpoint iterations.
+Default is `DEFAULT_CHECKPOINT_PERIOD`.
+### Compute node
+
+Stateless Postgres node that stores data in pageserver.
+
+### Garbage collection
+
+The process of removing old on-disk layers that are not needed by any timeline anymore.
+### Fork
+
+Each of the separate segmented file sets in which a relation is stored. The main fork is where the actual data resides. There also exist two secondary forks for metadata: the free space map and the visibility map.
+Each PostgreSQL fork is considered a separate relish.
+
+### Layer
+
+Each layer corresponds to the specific version of a relish Segment in a range of LSNs.
+There are two kinds of layers, in-memory and on-disk layers. In-memory
+layers are used to ingest incoming WAL, and provide fast access
+to the recent page versions. On-disk layers are stored as files on disk, and
+are immutable.
+### Layer file (on-disk layer)
+
+Layered repository on-disk format is based on immutable files.  The
+files are called "layer files". Each file corresponds to one RELISH_SEG_SIZE
+segment of a PostgreSQL relation fork. There are two kinds of layer
+files: image files and delta files. An image file contains a
+"snapshot" of the segment at a particular LSN, and a delta file
+contains WAL records applicable to the segment, in a range of LSNs.
+
+### Layer map
+
+The layer map tracks what layers exist for all the relishes in a timeline.
+### Layered repository
+
+Zenith repository implementation that keeps data in layers.
+### LSN
+
+
+### Page (block)
+
+The basic structure used to store relation data. All pages are of the same size.
+This is the unit of data exchange between compute node and pageserver.
+
+### Pageserver
+
+Zenith storage engine: repositories + wal receiver + page service + wal redo.
+
+### Page service
+
+The Page Service listens for GetPage@LSN requests from the Compute Nodes,
+and responds with pages from the repository.
+
+
+### PITR (Point-in-time-recovery)
+
+PostgreSQL's ability to restore up to a specified LSN.
+
+### Primary node
+
+
+### Proxy
+
+Postgres protocol proxy/router.
+This service listens psql port, can check auth via external service
+and create new databases and accounts (control plane API in our case).
+
+### Relation
+
+The generic term in PostgreSQL for all objects in a database that have a name and a list of attributes defined in a specific order.
+
+### Relish
+
+We call each relation and other file that is stored in the
+repository a "relish". It comes from "rel"-ish, as in "kind of a
+rel", because it covers relations as well as other things that are
+not relations, but are treated similarly for the purposes of the
+storage layer.
+
+### Replication slot
+
+
+### Replica node
+
+
+### Repository
+
+Repository stores multiple timelines, forked off from the same initial call to 'initdb'
+and has associated WAL redo service.
+One repository corresponds to one Tenant.
+
+### Retention policy
+
+How much history do we need to keep around for PITR and read-only nodes?
+
+### Segment (PostgreSQL)
+
+NOTE: This is an overloaded term.
+
+A physical file that stores data for a given relation. File segments are
+limited in size by a compile-time setting (1 gigabyte by default), so if a
+relation exceeds that size, it is split into multiple segments.
+
+### Segment (Layered Repository)
+
+NOTE: This is an overloaded term.
+
+Segment is a RELISH_SEG_SIZE slice of relish (identified by a SegmentTag).
+
+### SLRU
+
+SLRUs include pg_clog, pg_multixact/members, and
+pg_multixact/offsets. There are other SLRUs in PostgreSQL, but
+they don't need to be stored permanently (e.g. pg_subtrans),
+or we do not support them in zenith yet (pg_commit_ts).
+Each SLRU segment is considered a separate relish[].
+
+### Tenant (Multitenancy)
+Tenant represents a single customer, interacting with Zenith.
+Wal redo[] activity, timelines[], layers[] are managed for each tenant independently.
+One pageserver[] can serve multiple tenants at once.
+One safekeeper 
+
+See `docs/multitenancy.md` for more.
+
+### Timeline
+
+Timeline accepts page changes and serves get_page_at_lsn() and
+get_rel_size() requests. The term "timeline" is used internally
+in the system, but to users they are exposed as "branches", with
+human-friendly names.
+
+NOTE: this has nothing to do with PostgreSQL WAL timelines.
+
+### XLOG
+
+PostgreSQL alias for WAL[].
+
+### WAL (Write-ahead log)
+
+The journal that keeps track of the changes in the database cluster as user- and system-invoked operations take place. It comprises many individual WAL records[] written sequentially to WAL files[].
+
+### WAL acceptor, WAL proposer
+
+In the context of the consensus algorithm, the Postgres
+compute node is also known as the WAL proposer, and the safekeeper is also known
+as the acceptor. Those are the standard terms in the Paxos algorithm.
+
+### WAL receiver (WAL decoder)
+
+The WAL receiver connects to the external WAL safekeeping service (or
+directly to the primary) using PostgreSQL physical streaming
+replication, and continuously receives WAL. It decodes the WAL records,
+and stores them to the repository.
+
+We keep one WAL receiver active per timeline.
+
+### WAL record
+
+A low-level description of an individual data change.
+
+### WAL redo
+
+A service that runs PostgreSQL in a special wal_redo mode
+to apply given WAL records over an old page image and return new page image.
+
+### WAL safekeeper
+
+One node that participates in the quorum. All the safekeepers
+together form the WAL service.
+
+### WAL segment (WAL file)
+
+Also known as WAL segment or WAL segment file. Each of the sequentially-numbered files that provide storage space for WAL. The files are all of the same predefined size and are written in sequential order, interspersing changes as they occur in multiple simultaneous sessions.
+
+### WAL service
+
+The service as whole that ensures that WAL is stored durably.
+
+### Web console
+
--- a/docs/multitenancy.md
+++ b/docs/multitenancy.md
@@ -37,7 +37,7 @@ On the page server tenants introduce one level of indirection, so data directory
   ├── de182bc61fb11a5a6b390a8aed3a804a
   └── ee6016ec31116c1b7c33dfdfca38891f
 ```
-Wal redo activity, timelines, snapshots are managed for each tenant independently.
+Wal redo activity and timelines are managed for each tenant independently.

 For local environment used for example in tests there also new level of indirection for tenants. It touches `pgdatadirs` directory. Now it contains `tenants` subdirectory so the structure looks the following way:

--- a/docs/sourcetree.md
+++ b/docs/sourcetree.md
@@ -0,0 +1,81 @@
+## Source tree layout
+
+Below you will find a brief overview of each subdir in the source tree in alphabetical order.
+
+`/control_plane`:
+
+Local control plane.
+Functions to start, configure and stop pageserver and postgres instances running as a local processes.
+Intended to be used in integration tests and in CLI tools for local installations.
+
+`/docs`:
+
+Documentaion of the Zenith features and concepts.
+Now it is mostly dev documentation.
+
+`/monitoring`:
+
+TODO
+
+`/pageserver`:
+
+Zenith storage service.
+The pageserver has a few different duties:
+
+- Store and manage the data.
+- Generate a tarball with files needed to bootstrap ComputeNode.
+- Respond to GetPage@LSN requests from the Compute Nodes.
+- Receive WAL from the WAL service and decode it.
+- Replay WAL that's applicable to the chunks that the Page Server maintains
+
+For more detailed info, see `/pageserver/README`
+
+`/postgres_ffi`:
+
+Utility functions for interacting with PostgreSQL file formats.
+Misc constants, copied from PostgreSQL headers.
+
+`/proxy`:
+
+Postgres protocol proxy/router.
+This service listens psql port, can check auth via external service
+and create new databases and accounts (control plane API in our case).
+
+`/test_runner`:
+
+Integration tests, written in Python using the `pytest` framework.
+
+`/vendor/postgres`:
+
+PostgreSQL source tree, with the modifications needed for Zenith.
+
+`/vendor/postgres/contrib/zenith`:
+
+PostgreSQL extension that implements storage manager API and network communications with remote page server.
+
+`/vendor/postgres/contrib/zenith_test_utils`:
+
+PostgreSQL extension that contains functions needed for testing and debugging.
+
+`/walkeeper`:
+
+The zenith WAL service that receives WAL from a primary compute nodes and streams it to the pageserver.
+It acts as a holding area and redistribution center for recently generated WAL.
+
+For more detailed info, see `/walkeeper/README`
+
+`/workspace_hack`:
+The workspace_hack crate exists only to pin down some dependencies.
+
+`/zenith`
+
+Main entry point for the 'zenith' CLI utility.
+TODO: Doesn't it belong to control_plane?
+
+`/zenith_metrics`:
+
+Helpers for exposing Prometheus metrics from the server.
+
+`/zenith_utils`:
+
+Helpers that are shared between other crates in this repository.
--- a/pageserver/Cargo.toml
+++ b/pageserver/Cargo.toml
@@ -4,8 +4,6 @@ version = "0.1.0"
 authors = ["Stas Kelvich <stas@zenith.tech>"]
 edition = "2018"

-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
 bookfile = "^0.3"
 chrono = "0.4.19"
@@ -14,35 +12,28 @@ regex = "1.4.5"
 bytes = { version = "1.0.1", features = ['serde'] }
 byteorder = "1.4.3"
 futures = "0.3.13"
+hyper = "0.14"
 lazy_static = "1.4.0"
-slog-stdlog = "4.1.0"
-slog-async = "2.6.0"
-slog-scope = "4.4.0"
-slog-term = "2.8.0"
-slog = "2.7.0"
 log = "0.4.14"
 clap = "2.33.0"
 daemonize = "0.4.1"
-rust-s3 = { version = "0.27.0-rc4", features = ["no-verify-ssl"] }
-tokio = { version = "1.5.0", features = ["full"] }
-tokio-stream = { version = "0.1.5" }
+tokio = { version = "1.11", features = ["process", "macros", "fs"] }
 postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-# by default rust-rocksdb tries to build a lot of compression algos. Use lz4 only for now as it is simplest dependency.
-rocksdb = { version = "0.16.0", features = ["lz4"], default-features = false }
+routerify = "2"
 anyhow = "1.0"
 crc32c = "0.6.0"
-walkdir = "2"
 thiserror = "1.0"
-hex = "0.4.3"
+hex = { version = "0.4.3", features = ["serde"] }
 tar = "0.4.33"
 humantime = "2.1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
-fs_extra = "1.2.0"
 toml = "0.5"
 scopeguard = "1.1.0"
+rust-s3 = { version = "0.27.0-rc4", features = ["no-verify-ssl"] }
+async-trait = "0.1"

 postgres_ffi = { path = "../postgres_ffi" }
 zenith_metrics = { path = "../zenith_metrics" }
--- a/pageserver/README
+++ b/pageserver/README
@@ -1,16 +1,5 @@
 ## Page server architecture

-The Page Server is responsible for all operations on a number of
-"chunks" of relation data. A chunk corresponds to a PostgreSQL
-relation segment (i.e. one max. 1 GB file in the data directory), but
-it holds all the different versions of every page in the segment that
-are still needed by the system.
-
-Currently we do not specifically organize data in chunks.
-All page images and corresponding WAL records are stored as entries in a key-value storage,
-where StorageKey is a zenith_timeline_id + BufferTag + LSN.
-
-
 The Page Server has a few different duties:

 - Respond to GetPage@LSN requests from the Compute Nodes
@@ -19,10 +8,11 @@ The Page Server has a few different duties:
 - Backup to S3


-The Page Server consists of multiple threads that operate on a shared
-cache of page versions:


+The Page Server consists of multiple threads that operate on a shared
+repository of page versions:
+
                                           | WAL
                                           V
                                   +--------------+
@@ -34,16 +24,14 @@ cache of page versions:
                  +---------+                              ..........            |    |
                  |         |                              .        .            |    |
 GetPage@LSN      |         |                              . backup .  ------->  | S3 |
------------->    |  Page   |         page cache           .        .            |    |
+------------->    |  Page   |         repository           .        .            |    |
                  | Service |                              ..........            |    |
   page           |         |                                                    +----+
 <-------------    |         |
-                  +---------+
-
-                             ...................................
-                             .                                 .
-                             . Garbage Collection / Compaction .
-                             ...................................
+                  +---------+      +--------------------+
+		                   |   Checkpointing /  |
+				   | Garbage collection |
+                                   +--------------------+

 Legend:

@@ -63,7 +51,7 @@ Page Service
 ------------

 The Page Service listens for GetPage@LSN requests from the Compute Nodes,
-and responds with pages from the page cache.
+and responds with pages from the repository.


 WAL Receiver
@@ -72,47 +60,59 @@ WAL Receiver
 The WAL receiver connects to the external WAL safekeeping service (or
 directly to the primary) using PostgreSQL physical streaming
 replication, and continuously receives WAL. It decodes the WAL records,
-and stores them to the page cache.
+and stores them to the repository.


-Page Cache
+Repository
 ----------

-The Page Cache is a switchboard to access different Repositories.
+The repository stores all the page versions, or WAL records needed to
+reconstruct them. Each tenant has a separate Repository, which is
+stored in the .zenith/tenants/<tenantid> directory.

-#### Repository
-Repository corresponds to one .zenith directory.
-Repository is needed to manage Timelines.
+Repository is an abstract trait, defined in `repository.rs`. It is
+implemented by the LayeredRepository object in
+`layered_repository.rs`. There is only that one implementation of the
+Repository trait, but it's still a useful abstraction that keeps the
+interface for the low-level storage functionality clean. The layered
+storage format is described in layered_repository/README.md.

-#### Timeline
-Timeline is a page cache workhorse that accepts page changes
-and serves get_page_at_lsn() and get_rel_size() requests.
-Note: this has nothing to do with PostgreSQL WAL timeline.
+Each repository consists of multiple Timelines. Timeline is a
+workhorse that accepts page changes from the WAL, and serves
+get_page_at_lsn() and get_rel_size() requests. Note: this has nothing
+to do with PostgreSQL WAL timeline. The term "timeline" is mostly
+interchangeable with "branch", there is a one-to-one mapping from
+branch to timeline. A timeline has a unique ID within the tenant,
+represented as 16-byte hex string that never changes, whereas a
+branch is a user-given name for a timeline.

-#### Branch
-We can create branch at certain LSN.
-Each Branch lives in a corresponding timeline and has an ancestor.
-
-To get full snapshot of data at certain moment we need to traverse timeline and its ancestors.
-
-#### ObjectRepository
-ObjectRepository implements Repository and has associated ObjectStore and WAL redo service.
-
-#### ObjectStore
-ObjectStore is an interface for key-value store for page images and wal records.
-Currently it has one implementation - RocksDB.
-
-#### WAL redo service
-WAL redo service - service that runs PostgreSQL in a special wal_redo mode
-to apply given WAL records over an old page image and return new page image.
+Each repository also has a WAL redo manager associated with it, see
+`walredo.rs`. The WAL redo manager is used to replay PostgreSQL WAL
+records, whenever we need to reconstruct a page version from WAL to
+satisfy a GetPage@LSN request, or to avoid accumulating too much WAL
+for a page. The WAL redo manager uses a Postgres process running in
+special zenith wal-redo mode to do the actual WAL redo, and
+communicates with the process using a pipe.


-TODO: Garbage Collection / Compaction
-------------------------------------
+Checkpointing / Garbage Collection
+----------------------------------

-Periodically, the Garbage Collection / Compaction thread runs
-and applies pending WAL records, and removes old page versions that
-are no longer needed.
+Periodically, the checkpointer thread wakes up and performs housekeeping
+duties on the repository. It has two duties:
+
+### Checkpointing
+
+Flush WAL that has accumulated in memory to disk, so that the old WAL
+can be truncated away in the WAL safekeepers. Also, to free up memory
+for receiving new WAL. This process is called "checkpointing". It's
+similar to checkpointing in PostgreSQL or other DBMSs, but in the page
+server, checkpointing happens on a per-segment basis.
+
+### Garbage collection
+
+Remove old on-disk layer files that are no longer needed according to the
+PITR retention policy


 TODO: Backup service
--- a/pageserver/src/basebackup.rs
+++ b/pageserver/src/basebackup.rs
@@ -10,6 +10,7 @@
 //! This module is responsible for creation of such tarball
 //! from data stored in object storage.
 //!
+use anyhow::Result;
 use bytes::{BufMut, BytesMut};
 use log::*;
 use std::io;
@@ -34,19 +35,63 @@ pub struct Basebackup<'a> {
    prev_record_lsn: Lsn,
 }

+// Create basebackup with non-rel data in it. Omit relational data.
+//
+// Currently we use empty lsn in two cases:
+//  * During the basebackup right after timeline creation
+//  * When working without safekeepers. In this situation it is important to match the lsn
+//    we are taking basebackup on with the lsn that is used in pageserver's walreceiver
+//    to start the replication.
 impl<'a> Basebackup<'a> {
    pub fn new(
        write: &'a mut dyn Write,
        timeline: &'a Arc<dyn Timeline>,
-        lsn: Lsn,
-        prev_record_lsn: Lsn,
-    ) -> Basebackup<'a> {
-        Basebackup {
+        req_lsn: Option<Lsn>,
+    ) -> Result<Basebackup<'a>> {
+        // Compute postgres doesn't have any previous WAL files, but the first
+        // record that it's going to write needs to include the LSN of the
+        // previous record (xl_prev). We include prev_record_lsn in the
+        // "zenith.signal" file, so that postgres can read it during startup.
+        //
+        // We don't keep full history of record boundaries in the page server,
+        // however, only the predecessor of the latest record on each
+        // timeline. So we can only provide prev_record_lsn when you take a
+        // base backup at the end of the timeline, i.e. at last_record_lsn.
+        // Even at the end of the timeline, we sometimes don't have a valid
+        // prev_lsn value; that happens if the timeline was just branched from
+        // an old LSN and it doesn't have any WAL of its own yet. We will set
+        // prev_lsn to Lsn(0) if we cannot provide the correct value.
+        let (backup_prev, backup_lsn) = if let Some(req_lsn) = req_lsn {
+            // Backup was requested at a particular LSN. Wait for it to arrive.
+            timeline.wait_lsn(req_lsn)?;
+
+            // If the requested point is the end of the timeline, we can
+            // provide prev_lsn. (get_last_record_rlsn() might return it as
+            // zero, though, if no WAL has been generated on this timeline
+            // yet.)
+            let end_of_timeline = timeline.get_last_record_rlsn();
+            if req_lsn == end_of_timeline.last {
+                (end_of_timeline.prev, req_lsn)
+            } else {
+                (Lsn(0), req_lsn)
+            }
+        } else {
+            // Backup was requested at end of the timeline.
+            let end_of_timeline = timeline.get_last_record_rlsn();
+            (end_of_timeline.prev, end_of_timeline.last)
+        };
+
+        info!(
+            "taking basebackup lsn={}, prev_lsn={}",
+            backup_prev, backup_lsn
+        );
+
+        Ok(Basebackup {
            ar: Builder::new(write),
            timeline,
-            lsn,
-            prev_record_lsn,
-        }
+            lsn: backup_lsn,
+            prev_record_lsn: backup_prev,
+        })
    }

    pub fn send_tarball(&mut self) -> anyhow::Result<()> {
@@ -61,10 +106,10 @@ impl<'a> Basebackup<'a> {
        for filepath in pg_constants::PGDATA_SPECIAL_FILES.iter() {
            if *filepath == "pg_hba.conf" {
                let data = pg_constants::PG_HBA.as_bytes();
-                let header = new_tar_header(&filepath, data.len() as u64)?;
-                self.ar.append(&header, &data[..])?;
+                let header = new_tar_header(filepath, data.len() as u64)?;
+                self.ar.append(&header, data)?;
            } else {
-                let header = new_tar_header(&filepath, 0)?;
+                let header = new_tar_header(filepath, 0)?;
                self.ar.append(&header, &mut io::empty())?;
            }
        }
@@ -114,11 +159,9 @@ impl<'a> Basebackup<'a> {
        let mut slru_buf: Vec<u8> =
            Vec::with_capacity(nblocks as usize * pg_constants::BLCKSZ as usize);
        for blknum in 0..nblocks {
-            let img = self.timeline.get_page_at_lsn_nowait(
-                RelishTag::Slru { slru, segno },
-                blknum,
-                self.lsn,
-            )?;
+            let img =
+                self.timeline
+                    .get_page_at_lsn(RelishTag::Slru { slru, segno }, blknum, self.lsn)?;
            assert!(img.len() == pg_constants::BLCKSZ as usize);

            slru_buf.extend_from_slice(&img);
@@ -137,20 +180,18 @@ impl<'a> Basebackup<'a> {
    // Along with them also send PG_VERSION for each database.
    //
    fn add_relmap_file(&mut self, spcnode: u32, dbnode: u32) -> anyhow::Result<()> {
-        let img = self.timeline.get_page_at_lsn_nowait(
+        let img = self.timeline.get_page_at_lsn(
            RelishTag::FileNodeMap { spcnode, dbnode },
            0,
            self.lsn,
        )?;
        let path = if spcnode == pg_constants::GLOBALTABLESPACE_OID {
-            let dst_path = "PG_VERSION";
            let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
-            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
-            self.ar.append(&header, &version_bytes[..])?;
+            let header = new_tar_header("PG_VERSION", version_bytes.len() as u64)?;
+            self.ar.append(&header, version_bytes)?;

-            let dst_path = format!("global/PG_VERSION");
-            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
-            self.ar.append(&header, &version_bytes[..])?;
+            let header = new_tar_header("global/PG_VERSION", version_bytes.len() as u64)?;
+            self.ar.append(&header, version_bytes)?;

            String::from("global/pg_filenode.map") // filenode map for global tablespace
        } else {
@@ -165,7 +206,7 @@ impl<'a> Basebackup<'a> {
            let dst_path = format!("base/{}/PG_VERSION", dbnode);
            let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
-            self.ar.append(&header, &version_bytes[..])?;
+            self.ar.append(&header, version_bytes)?;

            format!("base/{}/pg_filenode.map", dbnode)
        };
@@ -179,18 +220,18 @@ impl<'a> Basebackup<'a> {
    // Extract twophase state files
    //
    fn add_twophase_file(&mut self, xid: TransactionId) -> anyhow::Result<()> {
-        if let Ok(img) =
-            self.timeline
-                .get_page_at_lsn_nowait(RelishTag::TwoPhase { xid }, 0, self.lsn)
-        {
-            let mut buf = BytesMut::new();
-            buf.extend_from_slice(&img[..]);
-            let crc = crc32c::crc32c(&img[..]);
-            buf.put_u32_le(crc);
-            let path = format!("pg_twophase/{:>08X}", xid);
-            let header = new_tar_header(&path, buf.len() as u64)?;
-            self.ar.append(&header, &buf[..])?;
-        }
+        let img = self
+            .timeline
+            .get_page_at_lsn(RelishTag::TwoPhase { xid }, 0, self.lsn)?;
+
+        let mut buf = BytesMut::new();
+        buf.extend_from_slice(&img[..]);
+        let crc = crc32c::crc32c(&img[..]);
+        buf.put_u32_le(crc);
+        let path = format!("pg_twophase/{:>08X}", xid);
+        let header = new_tar_header(&path, buf.len() as u64)?;
+        self.ar.append(&header, &buf[..])?;
+
        Ok(())
    }

@@ -199,12 +240,12 @@ impl<'a> Basebackup<'a> {
    // Also send zenith.signal file with extra bootstrap data.
    //
    fn add_pgcontrol_file(&mut self) -> anyhow::Result<()> {
-        let checkpoint_bytes =
-            self.timeline
-                .get_page_at_lsn_nowait(RelishTag::Checkpoint, 0, self.lsn)?;
+        let checkpoint_bytes = self
+            .timeline
+            .get_page_at_lsn(RelishTag::Checkpoint, 0, self.lsn)?;
        let pg_control_bytes =
            self.timeline
-                .get_page_at_lsn_nowait(RelishTag::ControlFile, 0, self.lsn)?;
+                .get_page_at_lsn(RelishTag::ControlFile, 0, self.lsn)?;
        let mut pg_control = ControlFileData::decode(&pg_control_bytes)?;
        let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;

@@ -215,7 +256,7 @@ impl<'a> Basebackup<'a> {
            XLOG_SIZE_OF_XLOG_LONG_PHD as u32,
            pg_constants::WAL_SEGMENT_SIZE,
        );
-        checkpoint.redo = self.lsn.0 + self.lsn.calc_padding(8u32);
+        checkpoint.redo = normalize_lsn(self.lsn, pg_constants::WAL_SEGMENT_SIZE).0;

        //reset some fields we don't want to preserve
        //TODO Check this.
@@ -228,9 +269,14 @@ impl<'a> Basebackup<'a> {
        pg_control.state = pg_constants::DB_SHUTDOWNED;

        // add zenith.signal file
+        let xl_prev = if self.prev_record_lsn == Lsn(0) {
+            0xBAD0 // magic value to indicate that we don't know prev_lsn
+        } else {
+            self.prev_record_lsn.0
+        };
        self.ar.append(
            &new_tar_header("zenith.signal", 8)?,
-            &self.prev_record_lsn.0.to_le_bytes()[..],
+            &xl_prev.to_le_bytes()[..],
        )?;

        //send pg_control
--- a/pageserver/src/bin/dump_layerfile.rs
+++ b/pageserver/src/bin/dump_layerfile.rs
@@ -0,0 +1,25 @@
+//! Main entry point for the dump_layerfile executable
+//!
+//! A handy tool for debugging, that's all.
+use anyhow::Result;
+use clap::{App, Arg};
+use pageserver::layered_repository::dump_layerfile_from_path;
+use std::path::PathBuf;
+
+fn main() -> Result<()> {
+    let arg_matches = App::new("Zenith dump_layerfile utility")
+        .about("Dump contents of one layer file, for debugging")
+        .arg(
+            Arg::with_name("path")
+                .help("Path to file to dump")
+                .required(true)
+                .index(1),
+        )
+        .get_matches();
+
+    let path = PathBuf::from(arg_matches.value_of("path").unwrap());
+
+    dump_layerfile_from_path(&path)?;
+
+    Ok(())
+}
--- a/pageserver/src/bin/pageserver.rs
+++ b/pageserver/src/bin/pageserver.rs
@@ -3,6 +3,7 @@
 //

 use log::*;
+use pageserver::defaults::*;
 use serde::{Deserialize, Serialize};
 use std::{
    env,
@@ -10,38 +11,49 @@ use std::{
    path::{Path, PathBuf},
    process::exit,
    str::FromStr,
-    sync::Arc,
    thread,
-    time::Duration,
 };
-use zenith_utils::{auth::JwtAuth, postgres_backend::AuthType};
+use zenith_utils::{auth::JwtAuth, logging, postgres_backend::AuthType};

-use anyhow::{ensure, Result};
+use anyhow::{bail, ensure, Result};
 use clap::{App, Arg, ArgMatches};
 use daemonize::Daemonize;

-use pageserver::{branches, logger, page_cache, page_service, PageServerConf, RepositoryFormat};
-use zenith_utils::http_endpoint;
-
-const DEFAULT_LISTEN_ADDR: &str = "127.0.0.1:64000";
-const DEFAULT_HTTP_ENDPOINT_ADDR: &str = "127.0.0.1:9898";
-
-const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
-const DEFAULT_GC_PERIOD: Duration = Duration::from_secs(10);
-
-const DEFAULT_SUPERUSER: &str = "zenith_admin";
+use pageserver::{
+    branches, http, page_service, tenant_mgr, PageServerConf, RelishStorageConfig, S3Config,
+    LOG_FILE_NAME,
+};
+use zenith_utils::http::endpoint;

 /// String arguments that can be declared via CLI or config file
 #[derive(Serialize, Deserialize)]
 struct CfgFileParams {
-    listen_addr: Option<String>,
-    http_endpoint_addr: Option<String>,
+    listen_pg_addr: Option<String>,
+    listen_http_addr: Option<String>,
+    checkpoint_distance: Option<String>,
+    checkpoint_period: Option<String>,
    gc_horizon: Option<String>,
    gc_period: Option<String>,
    pg_distrib_dir: Option<String>,
    auth_validation_public_key_path: Option<String>,
    auth_type: Option<String>,
-    repository_format: Option<String>,
+    // see https://github.com/alexcrichton/toml-rs/blob/6c162e6562c3e432bf04c82a3d1d789d80761a86/examples/enum_external.rs for enum deserialisation examples
+    relish_storage: Option<RelishStorage>,
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+enum RelishStorage {
+    Local {
+        local_path: String,
+    },
+    AwsS3 {
+        bucket_name: String,
+        bucket_region: String,
+        #[serde(skip_serializing)]
+        access_key_id: Option<String>,
+        #[serde(skip_serializing)]
+        secret_access_key: Option<String>,
+    },
 }

 impl CfgFileParams {
@@ -51,15 +63,32 @@ impl CfgFileParams {
            arg_matches.value_of(arg_name).map(str::to_owned)
        };

+        let relish_storage = if let Some(local_path) = get_arg("relish-storage-local-path") {
+            Some(RelishStorage::Local { local_path })
+        } else if let Some((bucket_name, bucket_region)) =
+            get_arg("relish-storage-s3-bucket").zip(get_arg("relish-storage-region"))
+        {
+            Some(RelishStorage::AwsS3 {
+                bucket_name,
+                bucket_region,
+                access_key_id: get_arg("relish-storage-access-key"),
+                secret_access_key: get_arg("relish-storage-secret-access-key"),
+            })
+        } else {
+            None
+        };
+
        Self {
-            listen_addr: get_arg("listen"),
-            http_endpoint_addr: get_arg("http_endpoint"),
+            listen_pg_addr: get_arg("listen-pg"),
+            listen_http_addr: get_arg("listen-http"),
+            checkpoint_distance: get_arg("checkpoint_distance"),
+            checkpoint_period: get_arg("checkpoint_period"),
            gc_horizon: get_arg("gc_horizon"),
            gc_period: get_arg("gc_period"),
            pg_distrib_dir: get_arg("postgres-distrib"),
            auth_validation_public_key_path: get_arg("auth-validation-public-key-path"),
            auth_type: get_arg("auth-type"),
-            repository_format: get_arg("repository-format"),
+            relish_storage,
        }
    }

@@ -67,8 +96,10 @@ impl CfgFileParams {
    fn or(self, other: CfgFileParams) -> Self {
        // TODO cleaner way to do this
        Self {
-            listen_addr: self.listen_addr.or(other.listen_addr),
-            http_endpoint_addr: self.http_endpoint_addr.or(other.http_endpoint_addr),
+            listen_pg_addr: self.listen_pg_addr.or(other.listen_pg_addr),
+            listen_http_addr: self.listen_http_addr.or(other.listen_http_addr),
+            checkpoint_distance: self.checkpoint_distance.or(other.checkpoint_distance),
+            checkpoint_period: self.checkpoint_period.or(other.checkpoint_period),
            gc_horizon: self.gc_horizon.or(other.gc_horizon),
            gc_period: self.gc_period.or(other.gc_period),
            pg_distrib_dir: self.pg_distrib_dir.or(other.pg_distrib_dir),
@@ -76,7 +107,7 @@ impl CfgFileParams {
                .auth_validation_public_key_path
                .or(other.auth_validation_public_key_path),
            auth_type: self.auth_type.or(other.auth_type),
-            repository_format: self.repository_format.or(other.repository_format),
+            relish_storage: self.relish_storage.or(other.relish_storage),
        }
    }

@@ -84,14 +115,23 @@ impl CfgFileParams {
    fn try_into_config(&self) -> Result<PageServerConf> {
        let workdir = PathBuf::from(".");

-        let listen_addr = match self.listen_addr.as_ref() {
+        let listen_pg_addr = match self.listen_pg_addr.as_ref() {
            Some(addr) => addr.clone(),
-            None => DEFAULT_LISTEN_ADDR.to_owned(),
+            None => DEFAULT_PG_LISTEN_ADDR.to_owned(),
        };

-        let http_endpoint_addr = match self.http_endpoint_addr.as_ref() {
+        let listen_http_addr = match self.listen_http_addr.as_ref() {
            Some(addr) => addr.clone(),
-            None => DEFAULT_HTTP_ENDPOINT_ADDR.to_owned(),
+            None => DEFAULT_HTTP_LISTEN_ADDR.to_owned(),
+        };
+
+        let checkpoint_distance: u64 = match self.checkpoint_distance.as_ref() {
+            Some(checkpoint_distance_str) => checkpoint_distance_str.parse()?,
+            None => DEFAULT_CHECKPOINT_DISTANCE,
+        };
+        let checkpoint_period = match self.checkpoint_period.as_ref() {
+            Some(checkpoint_period_str) => humantime::parse_duration(checkpoint_period_str)?,
+            None => DEFAULT_CHECKPOINT_PERIOD,
        };

        let gc_horizon: u64 = match self.gc_horizon.as_ref() {
@@ -117,11 +157,11 @@ impl CfgFileParams {
            .auth_type
            .as_ref()
            .map_or(Ok(AuthType::Trust), |auth_type| {
-                AuthType::from_str(&auth_type)
+                AuthType::from_str(auth_type)
            })?;

        if !pg_distrib_dir.join("bin/postgres").exists() {
-            anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
+            bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
        }

        if auth_type == AuthType::ZenithJWT {
@@ -136,21 +176,33 @@ impl CfgFileParams {
            );
        }

-        let repository_format = match self.repository_format.as_ref() {
-            Some(repo_format_str) if repo_format_str == "rocksdb" => RepositoryFormat::RocksDb,
-            Some(repo_format_str) if repo_format_str == "layered" => RepositoryFormat::Layered,
-            Some(repo_format_str) => anyhow::bail!(
-                "invalid --repository-format '{}', must be 'rocksdb' or 'layered'",
-                repo_format_str
-            ),
-            None => RepositoryFormat::Layered, // default
-        };
+        let relish_storage_config =
+            self.relish_storage
+                .as_ref()
+                .map(|storage_params| match storage_params.clone() {
+                    RelishStorage::Local { local_path } => {
+                        RelishStorageConfig::LocalFs(PathBuf::from(local_path))
+                    }
+                    RelishStorage::AwsS3 {
+                        bucket_name,
+                        bucket_region,
+                        access_key_id,
+                        secret_access_key,
+                    } => RelishStorageConfig::AwsS3(S3Config {
+                        bucket_name,
+                        bucket_region,
+                        access_key_id,
+                        secret_access_key,
+                    }),
+                });

        Ok(PageServerConf {
            daemonize: false,

-            listen_addr,
-            http_endpoint_addr,
+            listen_pg_addr,
+            listen_http_addr,
+            checkpoint_distance,
+            checkpoint_period,
            gc_horizon,
            gc_period,

@@ -162,8 +214,7 @@ impl CfgFileParams {

            auth_validation_public_key_path,
            auth_type,
-
-            repository_format,
+            relish_storage_config,
        })
    }
 }
@@ -172,12 +223,20 @@ fn main() -> Result<()> {
    let arg_matches = App::new("Zenith page server")
        .about("Materializes WAL stream to pages and serves them to the postgres")
        .arg(
-            Arg::with_name("listen")
+            Arg::with_name("listen-pg")
                .short("l")
-                .long("listen")
+                .long("listen-pg")
+                .alias("listen") // keep some compatibility
                .takes_value(true)
                .help("listen for incoming page requests on ip:port (default: 127.0.0.1:5430)"),
        )
+        .arg(
+            Arg::with_name("listen-http")
+                .long("listen-http")
+                .alias("http_endpoint") // keep some compatibility
+                .takes_value(true)
+                .help("http endpoint address for for metrics and management API calls ip:port (default: 127.0.0.1:5430)"),
+        )
        .arg(
            Arg::with_name("daemonize")
                .short("d")
@@ -191,6 +250,18 @@ fn main() -> Result<()> {
                .takes_value(false)
                .help("Initialize pageserver repo"),
        )
+        .arg(
+            Arg::with_name("checkpoint_distance")
+                .long("checkpoint_distance")
+                .takes_value(true)
+                .help("Distance from current LSN to perform checkpoint of in-memory layers"),
+        )
+        .arg(
+            Arg::with_name("checkpoint_period")
+                .long("checkpoint_period")
+                .takes_value(true)
+                .help("Interval between checkpoint iterations"),
+        )
        .arg(
            Arg::with_name("gc_horizon")
                .long("gc_horizon")
@@ -236,10 +307,41 @@ fn main() -> Result<()> {
                .help("Authentication scheme type. One of: Trust, MD5, ZenithJWT"),
        )
        .arg(
-            Arg::with_name("repository-format")
-                .long("repository-format")
+            Arg::with_name("relish-storage-local-path")
+                .long("relish-storage-local-path")
                .takes_value(true)
-                .help("Which repository implementation to use, 'rocksdb' or 'layered'"),
+                .help("Path to the local directory, to be used as an external relish storage")
+                .conflicts_with_all(&[
+                    "relish-storage-s3-bucket",
+                    "relish-storage-region",
+                    "relish-storage-access-key",
+                    "relish-storage-secret-access-key",
+                ]),
+        )
+        .arg(
+            Arg::with_name("relish-storage-s3-bucket")
+                .long("relish-storage-s3-bucket")
+                .takes_value(true)
+                .help("Name of the AWS S3 bucket to use an external relish storage")
+                .requires("relish-storage-region"),
+        )
+        .arg(
+            Arg::with_name("relish-storage-region")
+                .long("relish-storage-region")
+                .takes_value(true)
+                .help("Region of the AWS S3 bucket"),
+        )
+        .arg(
+            Arg::with_name("relish-storage-access-key")
+                .long("relish-storage-access-key")
+                .takes_value(true)
+                .help("Credentials to access the AWS S3 bucket"),
+        )
+        .arg(
+            Arg::with_name("relish-storage-secret-access-key")
+                .long("relish-storage-secret-access-key")
+                .takes_value(true)
+                .help("Credentials to access the AWS S3 bucket"),
        )
        .get_matches();

@@ -295,18 +397,27 @@ fn main() -> Result<()> {

 fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
    // Initialize logger
-    let (_scope_guard, log_file) = logger::init_logging(&conf, "pageserver.log")?;
-    let _log_guard = slog_stdlog::init()?;
-
-    // Note: this `info!(...)` macro comes from `log` crate
-    info!("standard logging redirected to slog");
+    let (_scope_guard, log_file) = logging::init(LOG_FILE_NAME, conf.daemonize)?;

    // TODO: Check that it looks like a valid repository before going further

+    // bind sockets before daemonizing so we report errors early and do not return until we are listening
+    info!(
+        "Starting pageserver http handler on {}",
+        conf.listen_http_addr
+    );
+    let http_listener = TcpListener::bind(conf.listen_http_addr.clone())?;
+
+    info!(
+        "Starting pageserver pg protocol handler on {}",
+        conf.listen_pg_addr
+    );
+    let pageserver_listener = TcpListener::bind(conf.listen_pg_addr.clone())?;
+
    if conf.daemonize {
        info!("daemonizing...");

-        // There should'n be any logging to stdin/stdout. Redirect it to the main log so
+        // There shouldn't be any logging to stdin/stdout. Redirect it to the main log so
        // that we will see any accidental manual fprintf's or backtraces.
        let stdout = log_file.try_clone().unwrap();
        let stderr = log_file;
@@ -323,29 +434,35 @@ fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
        }
    }

-    // Spawn a new thread for the http endpoint
-    thread::Builder::new()
-        .name("Metrics thread".into())
-        .spawn(move || http_endpoint::thread_main(conf.http_endpoint_addr.clone()))?;
+    // Initialize tenant manager.
+    tenant_mgr::init(conf);

-    // Check that we can bind to address before starting threads to simplify shutdown
-    // sequence if port is occupied.
-    info!("Starting pageserver on {}", conf.listen_addr);
-    let pageserver_listener = TcpListener::bind(conf.listen_addr.clone())?;
-
-    // Initialize page cache, this will spawn walredo_thread
-    page_cache::init(conf);
+    // keep join handles for spawned threads
+    let mut join_handles = vec![];

    // initialize authentication for incoming connections
    let auth = match &conf.auth_type {
-        AuthType::Trust | AuthType::MD5 => Arc::new(None),
+        AuthType::Trust | AuthType::MD5 => None,
        AuthType::ZenithJWT => {
            // unwrap is ok because check is performed when creating config, so path is set and file exists
            let key_path = conf.auth_validation_public_key_path.as_ref().unwrap();
-            Arc::new(Some(JwtAuth::from_key_path(key_path)?))
+            Some(JwtAuth::from_key_path(key_path)?.into())
        }
    };
    info!("Using auth: {:#?}", conf.auth_type);
+
+    // Spawn a new thread for the http endpoint
+    // bind before launching separate thread so the error reported before startup exits
+    let cloned = auth.clone();
+    let http_endpoint_thread = thread::Builder::new()
+        .name("http_endpoint_thread".into())
+        .spawn(move || {
+            let router = http::make_router(conf, cloned);
+            endpoint::serve_thread_main(router, http_listener)
+        })?;
+
+    join_handles.push(http_endpoint_thread);
+
    // Spawn a thread to listen for connections. It will spawn further threads
    // for each connection.
    let page_service_thread = thread::Builder::new()
@@ -354,9 +471,13 @@ fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
            page_service::thread_main(conf, auth, pageserver_listener, conf.auth_type)
        })?;

-    page_service_thread
-        .join()
-        .expect("Page service thread has panicked")?;
+    join_handles.push(page_service_thread);

+    for handle in join_handles.into_iter() {
+        handle
+            .join()
+            .expect("thread panicked")
+            .expect("thread exited with an error")
+    }
    Ok(())
 }
--- a/pageserver/src/branches.rs
+++ b/pageserver/src/branches.rs
@@ -17,22 +17,77 @@ use std::{
 use zenith_utils::zid::{ZTenantId, ZTimelineId};

 use log::*;
+use zenith_utils::logging;
 use zenith_utils::lsn::Lsn;

-use crate::logger;
-use crate::object_repository::ObjectRepository;
-use crate::page_cache;
-use crate::restore_local_repo;
+use crate::tenant_mgr;
 use crate::walredo::WalRedoManager;
-use crate::{repository::Repository, PageServerConf, RepositoryFormat};
+use crate::{repository::Repository, PageServerConf};
+use crate::{restore_local_repo, LOG_FILE_NAME};

 #[derive(Serialize, Deserialize, Clone)]
 pub struct BranchInfo {
    pub name: String,
+    #[serde(with = "hex")]
    pub timeline_id: ZTimelineId,
-    pub latest_valid_lsn: Option<Lsn>,
+    pub latest_valid_lsn: Lsn,
    pub ancestor_id: Option<String>,
    pub ancestor_lsn: Option<String>,
+    pub current_logical_size: usize,
+    pub current_logical_size_non_incremental: usize,
+}
+
+impl BranchInfo {
+    pub fn from_path<T: AsRef<Path>>(
+        path: T,
+        conf: &PageServerConf,
+        tenantid: &ZTenantId,
+        repo: &Arc<dyn Repository>,
+    ) -> Result<Self> {
+        let name = path
+            .as_ref()
+            .file_name()
+            .unwrap()
+            .to_str()
+            .unwrap()
+            .to_string();
+        let timeline_id = std::fs::read_to_string(path)?.parse::<ZTimelineId>()?;
+
+        let timeline = repo.get_timeline(timeline_id)?;
+
+        let ancestor_path = conf.ancestor_path(&timeline_id, tenantid);
+        let mut ancestor_id: Option<String> = None;
+        let mut ancestor_lsn: Option<String> = None;
+
+        if ancestor_path.exists() {
+            let ancestor = std::fs::read_to_string(ancestor_path)?;
+            let mut strings = ancestor.split('@');
+
+            ancestor_id = Some(
+                strings
+                    .next()
+                    .with_context(|| "wrong branch ancestor point in time format")?
+                    .to_owned(),
+            );
+            ancestor_lsn = Some(
+                strings
+                    .next()
+                    .with_context(|| "wrong branch ancestor point in time format")?
+                    .to_owned(),
+            );
+        }
+
+        Ok(BranchInfo {
+            name,
+            timeline_id,
+            latest_valid_lsn: timeline.get_last_record_lsn(),
+            ancestor_id,
+            ancestor_lsn,
+            current_logical_size: timeline.get_current_logical_size(),
+            current_logical_size_non_incremental: timeline
+                .get_current_logical_size_non_incremental(timeline.get_last_record_lsn())?,
+        })
+    }
 }

 #[derive(Debug, Clone, Copy)]
@@ -43,18 +98,25 @@ pub struct PointInTime {

 pub fn init_pageserver(conf: &'static PageServerConf, create_tenant: Option<&str>) -> Result<()> {
    // Initialize logger
-    let (_scope_guard, _log_file) = logger::init_logging(&conf, "pageserver.log")?;
-    let _log_guard = slog_stdlog::init()?;
+    // use true as daemonize parameter because otherwise we pollute zenith cli output with a few pages long output of info messages
+    let (_scope_guard, _log_file) = logging::init(LOG_FILE_NAME, true)?;
+
+    // We don't use the real WAL redo manager, because we don't want to spawn the WAL redo
+    // process during repository initialization.
+    //
+    // FIXME: That caused trouble, because the WAL redo manager spawned a thread that launched
+    // initdb in the background, and it kept running even after the "zenith init" had exited.
+    // In tests, we started the  page server immediately after that, so that initdb was still
+    // running in the background, and we failed to run initdb again in the same directory. This
+    // has been solved for the rapid init+start case now, but the general race condition remains
+    // if you restart the server quickly. The WAL redo manager doesn't use a separate thread
+    // anymore, but I think that could still happen.
+    let dummy_redo_mgr = Arc::new(crate::walredo::DummyRedoManager {});

    if let Some(tenantid) = create_tenant {
        let tenantid = ZTenantId::from_str(tenantid)?;
        println!("initializing tenantid {}", tenantid);
-        create_repo(
-            conf,
-            tenantid,
-            Arc::new(crate::walredo::DummyRedoManager {}),
-        )
-        .with_context(|| "failed to create repo")?;
+        create_repo(conf, tenantid, dummy_redo_mgr).with_context(|| "failed to create repo")?;
    }
    fs::create_dir_all(conf.tenants_path())?;

@@ -76,9 +138,6 @@ pub fn create_repo(
    fs::create_dir_all(&repo_dir)
        .with_context(|| format!("could not create directory {}", repo_dir.display()))?;

-    // Note: this `info!(...)` macro comes from `log` crate
-    info!("standard logging redirected to slog");
-
    fs::create_dir(conf.timelines_path(&tenantid))?;
    fs::create_dir_all(conf.branches_path(&tenantid))?;
    fs::create_dir_all(conf.tags_path(&tenantid))?;
@@ -87,31 +146,11 @@ pub fn create_repo(

    let tli = create_timeline(conf, None, &tenantid)?;

-    // We don't use page_cache here, because we don't want to spawn the WAL redo thread during
-    // repository initialization.
-    //
-    // FIXME: That caused trouble, because the WAL redo thread launched initdb in the background,
-    // and it kept running even after the "zenith init" had exited. In tests, we started the
-    // page server immediately after that, so that initdb was still running in the background,
-    // and we failed to run initdb again in the same directory. This has been solved for the
-    // rapid init+start case now, but the general race condition remains if you restart the
-    // server quickly.
-    let repo: Arc<dyn Repository + Sync + Send> =
-        match conf.repository_format {
-            RepositoryFormat::Layered => Arc::new(
-                crate::layered_repository::LayeredRepository::new(conf, wal_redo_manager, tenantid),
-            ),
-            RepositoryFormat::RocksDb => {
-                let obj_store = crate::rocksdb_storage::RocksObjectStore::create(conf, &tenantid)?;
-
-                Arc::new(ObjectRepository::new(
-                    conf,
-                    Arc::new(obj_store),
-                    wal_redo_manager,
-                    tenantid,
-                ))
-            }
-        };
+    let repo = Arc::new(crate::layered_repository::LayeredRepository::new(
+        conf,
+        wal_redo_manager,
+        tenantid,
+    ));

    // Load data into pageserver
    // TODO To implement zenith import we need to
@@ -131,7 +170,7 @@ fn get_lsn_from_controlfile(path: &Path) -> Result<Lsn> {
    Ok(Lsn(lsn))
 }

-// Create the cluster temporarily in a initdbpath directory inside the repository
+// Create the cluster temporarily in 'initdbpath' directory inside the repository
 // to get bootstrap data for timeline initialization.
 //
 fn run_initdb(conf: &'static PageServerConf, initdbpath: &Path) -> Result<()> {
@@ -175,15 +214,15 @@ fn bootstrap_timeline(
    run_initdb(conf, &initdb_path)?;
    let pgdata_path = initdb_path;

-    let lsn = get_lsn_from_controlfile(&pgdata_path)?;
+    let lsn = get_lsn_from_controlfile(&pgdata_path)?.align();

    info!("bootstrap_timeline {:?} at lsn {}", pgdata_path, lsn);

-    let timeline = repo.create_empty_timeline(tli, lsn)?;
+    // Import the contents of the data directory at the initial checkpoint
+    // LSN, and any WAL after that.
+    let timeline = repo.create_empty_timeline(tli)?;
    restore_local_repo::import_timeline_from_postgres_datadir(&pgdata_path, &*timeline, lsn)?;
-
-    let wal_dir = pgdata_path.join("pg_wal");
-    restore_local_repo::import_timeline_wal(&wal_dir, &*timeline, timeline.get_last_record_lsn())?;
+    timeline.checkpoint()?;

    println!(
        "created initial timeline {} timeline.lsn {}",
@@ -214,7 +253,7 @@ pub(crate) fn get_tenants(conf: &PageServerConf) -> Result<Vec<String>> {
 }

 pub(crate) fn get_branches(conf: &PageServerConf, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
-    let repo = page_cache::get_repository_for_tenant(tenantid)?;
+    let repo = tenant_mgr::get_repository_for_tenant(*tenantid)?;

    // Each branch has a corresponding record (text file) in the refs/branches
    // with timeline_id.
@@ -223,43 +262,7 @@ pub(crate) fn get_branches(conf: &PageServerConf, tenantid: &ZTenantId) -> Resul
    std::fs::read_dir(&branches_dir)?
        .map(|dir_entry_res| {
            let dir_entry = dir_entry_res?;
-            let name = dir_entry.file_name().to_str().unwrap().to_string();
-            let timeline_id = std::fs::read_to_string(dir_entry.path())?.parse::<ZTimelineId>()?;
-
-            let latest_valid_lsn = repo
-                .get_timeline(timeline_id)
-                .map(|timeline| timeline.get_last_valid_lsn())
-                .ok();
-
-            let ancestor_path = conf.ancestor_path(&timeline_id, tenantid);
-            let mut ancestor_id: Option<String> = None;
-            let mut ancestor_lsn: Option<String> = None;
-
-            if ancestor_path.exists() {
-                let ancestor = std::fs::read_to_string(ancestor_path)?;
-                let mut strings = ancestor.split('@');
-
-                ancestor_id = Some(
-                    strings
-                        .next()
-                        .with_context(|| "wrong branch ancestor point in time format")?
-                        .to_owned(),
-                );
-                ancestor_lsn = Some(
-                    strings
-                        .next()
-                        .with_context(|| "wrong branch ancestor point in time format")?
-                        .to_owned(),
-                );
-            }
-
-            Ok(BranchInfo {
-                name,
-                timeline_id,
-                latest_valid_lsn,
-                ancestor_id,
-                ancestor_lsn,
-            })
+            BranchInfo::from_path(dir_entry.path(), conf, tenantid, &repo)
        })
        .collect()
 }
@@ -270,21 +273,36 @@ pub(crate) fn create_branch(
    startpoint_str: &str,
    tenantid: &ZTenantId,
 ) -> Result<BranchInfo> {
-    let repo = page_cache::get_repository_for_tenant(tenantid)?;
+    let repo = tenant_mgr::get_repository_for_tenant(*tenantid)?;

    if conf.branch_path(branchname, tenantid).exists() {
        anyhow::bail!("branch {} already exists", branchname);
    }

    let mut startpoint = parse_point_in_time(conf, startpoint_str, tenantid)?;
-
+    let timeline = repo.get_timeline(startpoint.timelineid)?;
    if startpoint.lsn == Lsn(0) {
        // Find end of WAL on the old timeline
-        let end_of_wal = repo
-            .get_timeline(startpoint.timelineid)?
-            .get_last_record_lsn();
-        println!("branching at end of WAL: {}", end_of_wal);
+        let end_of_wal = timeline.get_last_record_lsn();
+        info!("branching at end of WAL: {}", end_of_wal);
        startpoint.lsn = end_of_wal;
+    } else {
+        // Wait for the WAL to arrive and be processed on the parent branch up
+        // to the requested branch point. The repository code itself doesn't
+        // require it, but if we start to receive WAL on the new timeline,
+        // decoding the new WAL might need to look up previous pages, relation
+        // sizes etc. and that would get confused if the previous page versions
+        // are not in the repository yet.
+        timeline.wait_lsn(startpoint.lsn)?;
+    }
+    startpoint.lsn = startpoint.lsn.align();
+    if timeline.get_start_lsn() > startpoint.lsn {
+        anyhow::bail!(
+            "invalid startpoint {} for the branch {}: less than timeline start {}",
+            startpoint.lsn,
+            branchname,
+            timeline.get_start_lsn()
+        );
    }

    // create a new timeline directory for it
@@ -297,14 +315,16 @@ pub(crate) fn create_branch(
    // FIXME: there's a race condition, if you create a branch with the same
    // name concurrently.
    let data = newtli.to_string();
-    fs::write(conf.branch_path(&branchname, tenantid), data)?;
+    fs::write(conf.branch_path(branchname, tenantid), data)?;

    Ok(BranchInfo {
        name: branchname.to_string(),
        timeline_id: newtli,
-        latest_valid_lsn: Some(startpoint.lsn),
+        latest_valid_lsn: startpoint.lsn,
        ancestor_id: None,
        ancestor_lsn: None,
+        current_logical_size: 0,
+        current_logical_size_non_incremental: 0,
    })
 }

@@ -346,21 +366,21 @@ fn parse_point_in_time(

    // Check if it's a tag
    if lsn.is_none() {
-        let tagpath = conf.tag_path(name, &tenantid);
+        let tagpath = conf.tag_path(name, tenantid);
        if tagpath.exists() {
            let pointstr = fs::read_to_string(tagpath)?;

-            return parse_point_in_time(conf, &pointstr, &tenantid);
+            return parse_point_in_time(conf, &pointstr, tenantid);
        }
    }

    // Check if it's a branch
    // Check if it's branch @ LSN
-    let branchpath = conf.branch_path(name, &tenantid);
+    let branchpath = conf.branch_path(name, tenantid);
    if branchpath.exists() {
        let pointstr = fs::read_to_string(branchpath)?;

-        let mut result = parse_point_in_time(conf, &pointstr, &tenantid)?;
+        let mut result = parse_point_in_time(conf, &pointstr, tenantid)?;

        result.lsn = lsn.unwrap_or(Lsn(0));
        return Ok(result);
@@ -369,7 +389,7 @@ fn parse_point_in_time(
    // Check if it's a timelineid
    // Check if it's timelineid @ LSN
    if let Ok(timelineid) = ZTimelineId::from_str(name) {
-        let tlipath = conf.timeline_path(&timelineid, &tenantid);
+        let tlipath = conf.timeline_path(&timelineid, tenantid);
        if tlipath.exists() {
            return Ok(PointInTime {
                timelineid,
--- a/pageserver/src/http/mod.rs
+++ b/pageserver/src/http/mod.rs
@@ -0,0 +1,3 @@
+pub mod models;
+pub mod routes;
+pub use routes::make_router;
--- a/pageserver/src/http/models.rs
+++ b/pageserver/src/http/models.rs
@@ -0,0 +1,17 @@
+use serde::{Deserialize, Serialize};
+
+use crate::ZTenantId;
+
+#[derive(Serialize, Deserialize)]
+pub struct BranchCreateRequest {
+    #[serde(with = "hex")]
+    pub tenant_id: ZTenantId,
+    pub name: String,
+    pub start_point: String,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct TenantCreateRequest {
+    #[serde(with = "hex")]
+    pub tenant_id: ZTenantId,
+}
--- a/pageserver/src/http/openapi_spec.yml
+++ b/pageserver/src/http/openapi_spec.yml
@@ -0,0 +1,291 @@
+openapi: "3.0.2"
+info:
+  title: Page Server API
+  version: "1.0"
+servers:
+  - url: ""
+paths:
+  /v1/status:
+    description: Healthcheck endpoint
+    get:
+      description: Healthcheck
+      security: []
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+  /v1/branch/{tenant_id}:
+    parameters:
+      - name: tenant_id
+        in: path
+        required: true
+        schema:
+          type: string
+          format: hex
+    get:
+      description: Get branches for tenant
+      responses:
+        "200":
+          description: BranchInfo
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/BranchInfo"
+        "400":
+          description: Error when no tenant id found in path
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+  /v1/branch/{tenant_id}/{branch_name}:
+    parameters:
+      - name: tenant_id
+        in: path
+        required: true
+        schema:
+          type: string
+          format: hex
+      - name: branch_name
+        in: path
+        required: true
+        schema:
+          type: string
+    get:
+      description: Get branches for tenant
+      responses:
+        "200":
+          description: BranchInfo
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BranchInfo"
+        "400":
+          description: Error when no tenant id found in path or no branch name
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+  /v1/branch/:
+    post:
+      description: Create branch
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - "tenant_id"
+                - "name"
+                - "start_point"
+              properties:
+                tenant_id:
+                  type: string
+                  format: hex
+                name:
+                  type: string
+                start_point:
+                  type: string
+      responses:
+        "201":
+          description: BranchInfo
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/BranchInfo"
+        "400":
+          description: Malformed branch create request
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+  /v1/tenant/:
+    get:
+      description: Get tenants list
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+    post:
+      description: Create tenant
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - "tenant_id"
+              properties:
+                tenant_id:
+                  type: string
+                  format: hex
+      responses:
+        "201":
+          description: CREATED
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+        "400":
+          description: Malformed tenant create request
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+
+components:
+  securitySchemes:
+    JWT:
+      type: http
+      scheme: bearer
+      bearerFormat: JWT
+  schemas:
+    BranchInfo:
+      type: object
+      required:
+        - name
+        - timeline_id
+        - latest_valid_lsn
+        - current_logical_size
+        - current_logical_size_non_incremental
+      properties:
+        name:
+          type: string
+        timeline_id:
+          type: string
+          format: hex
+        ancestor_id:
+          type: string
+        ancestor_lsn:
+          type: string
+        current_logical_size:
+          type: integer
+        current_logical_size_non_incremental:
+          type: integer
+    Error:
+      type: object
+      required:
+        - msg
+      properties:
+        msg:
+          type: string
+    UnauthorizedError:
+      type: object
+      required:
+        - msg
+      properties:
+        msg:
+          type: string
+    ForbiddenError:
+      type: object
+      required:
+        - msg
+      properties:
+        msg:
+          type: string
+
+security:
+  - JWT: []
--- a/pageserver/src/http/routes.rs
+++ b/pageserver/src/http/routes.rs
@@ -0,0 +1,201 @@
+use std::str::FromStr;
+use std::sync::Arc;
+
+use anyhow::Result;
+use hyper::header;
+use hyper::StatusCode;
+use hyper::{Body, Request, Response, Uri};
+use routerify::{ext::RequestExt, RouterBuilder};
+use zenith_utils::auth::JwtAuth;
+use zenith_utils::http::endpoint::attach_openapi_ui;
+use zenith_utils::http::endpoint::auth_middleware;
+use zenith_utils::http::endpoint::check_permission;
+use zenith_utils::http::error::ApiError;
+use zenith_utils::http::{
+    endpoint,
+    error::HttpErrorBody,
+    json::{json_request, json_response},
+};
+
+use super::models::BranchCreateRequest;
+use super::models::TenantCreateRequest;
+use crate::branches::BranchInfo;
+use crate::{branches, tenant_mgr, PageServerConf, ZTenantId};
+
+#[derive(Debug)]
+struct State {
+    conf: &'static PageServerConf,
+    auth: Option<Arc<JwtAuth>>,
+    allowlist_routes: Vec<Uri>,
+}
+
+impl State {
+    fn new(conf: &'static PageServerConf, auth: Option<Arc<JwtAuth>>) -> Self {
+        let allowlist_routes = ["/v1/status", "/v1/doc", "/swagger.yml"]
+            .iter()
+            .map(|v| v.parse().unwrap())
+            .collect::<Vec<_>>();
+        Self {
+            conf,
+            auth,
+            allowlist_routes,
+        }
+    }
+}
+
+#[inline(always)]
+fn get_state(request: &Request<Body>) -> &State {
+    request
+        .data::<Arc<State>>()
+        .expect("unknown state type")
+        .as_ref()
+}
+
+#[inline(always)]
+fn get_config(request: &Request<Body>) -> &'static PageServerConf {
+    get_state(request).conf
+}
+
+fn get_request_param<'a>(
+    request: &'a Request<Body>,
+    param_name: &str,
+) -> Result<&'a str, ApiError> {
+    match request.param(param_name) {
+        Some(arg) => Ok(arg),
+        None => {
+            return Err(ApiError::BadRequest(format!(
+                "no {} specified in path param",
+                param_name
+            )))
+        }
+    }
+}
+
+fn parse_request_param<T: FromStr>(
+    request: &Request<Body>,
+    param_name: &str,
+) -> Result<T, ApiError> {
+    match get_request_param(request, param_name)?.parse() {
+        Ok(v) => Ok(v),
+        Err(_) => Err(ApiError::BadRequest(
+            "failed to parse tenant id".to_string(),
+        )),
+    }
+}
+
+// healthcheck handler
+async fn status_handler(_: Request<Body>) -> Result<Response<Body>, ApiError> {
+    Ok(Response::builder()
+        .status(StatusCode::OK)
+        .header(header::CONTENT_TYPE, "application/json")
+        .body(Body::from("{}"))
+        .map_err(ApiError::from_err)?)
+}
+
+async fn branch_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let request_data: BranchCreateRequest = json_request(&mut request).await?;
+
+    check_permission(&request, Some(request_data.tenant_id))?;
+
+    let response_data = tokio::task::spawn_blocking(move || {
+        branches::create_branch(
+            get_config(&request),
+            &request_data.name,
+            &request_data.start_point,
+            &request_data.tenant_id,
+        )
+    })
+    .await
+    .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::CREATED, response_data)?)
+}
+
+async fn branch_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let tenantid: ZTenantId = parse_request_param(&request, "tenant_id")?;
+
+    check_permission(&request, Some(tenantid))?;
+
+    let response_data = tokio::task::spawn_blocking(move || {
+        crate::branches::get_branches(get_config(&request), &tenantid)
+    })
+    .await
+    .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::OK, response_data)?)
+}
+
+// TODO add to swagger
+async fn branch_detail_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let tenantid: ZTenantId = parse_request_param(&request, "tenant_id")?;
+    let branch_name: &str = get_request_param(&request, "branch_name")?;
+    let conf = get_state(&request).conf;
+    let path = conf.branch_path(branch_name, &tenantid);
+
+    let response_data = tokio::task::spawn_blocking(move || {
+        let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
+        BranchInfo::from_path(path, conf, &tenantid, &repo)
+    })
+    .await
+    .map_err(ApiError::from_err)??;
+
+    Ok(json_response(StatusCode::OK, response_data)?)
+}
+
+async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    // check for management permission
+    check_permission(&request, None)?;
+
+    let response_data =
+        tokio::task::spawn_blocking(move || crate::branches::get_tenants(get_config(&request)))
+            .await
+            .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::OK, response_data)?)
+}
+
+async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    // check for management permission
+    check_permission(&request, None)?;
+
+    let request_data: TenantCreateRequest = json_request(&mut request).await?;
+
+    let response_data = tokio::task::spawn_blocking(move || {
+        tenant_mgr::create_repository_for_tenant(get_config(&request), request_data.tenant_id)
+    })
+    .await
+    .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::CREATED, response_data)?)
+}
+
+async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
+    json_response(
+        StatusCode::NOT_FOUND,
+        HttpErrorBody::from_msg("page not found".to_owned()),
+    )
+}
+
+pub fn make_router(
+    conf: &'static PageServerConf,
+    auth: Option<Arc<JwtAuth>>,
+) -> RouterBuilder<hyper::Body, ApiError> {
+    let spec = include_bytes!("openapi_spec.yml");
+    let mut router = attach_openapi_ui(endpoint::make_router(), spec, "/swagger.yml", "/v1/doc");
+    if auth.is_some() {
+        router = router.middleware(auth_middleware(|request| {
+            let state = get_state(request);
+            if state.allowlist_routes.contains(request.uri()) {
+                None
+            } else {
+                state.auth.as_deref()
+            }
+        }))
+    }
+
+    router
+        .data(Arc::new(State::new(conf, auth)))
+        .get("/v1/status", status_handler)
+        .get("/v1/branch/:tenant_id", branch_list_handler)
+        .get("/v1/branch/:tenant_id/:branch_name", branch_detail_handler)
+        .post("/v1/branch", branch_create_handler)
+        .get("/v1/tenant", tenant_list_handler)
+        .post("/v1/tenant", tenant_create_handler)
+        .any(handler_404)
+}
--- a/pageserver/src/layered_repository.rs
+++ b/pageserver/src/layered_repository.rs
--- a/pageserver/src/layered_repository/README.md
+++ b/pageserver/src/layered_repository/README.md
@@ -4,52 +4,107 @@ The on-disk format is based on immutable files. The page server
 receives a stream of incoming WAL, parses the WAL records to determine
 which pages they apply to, and accumulates the incoming changes in
 memory. Every now and then, the accumulated changes are written out to
-new files.
+new immutable files. This process is called checkpointing. Old versions
+of on-disk files that are not needed by any timeline are removed by GC
+process.

-The files are called "snapshot files". Each snapshot file corresponds
-to one 10 MB slice of a PostgreSQL relation fork. The snapshot files
+# Terms used in layered repository
+
+- Relish - one PostgreSQL relation or similarly treated file.
+- Segment - one slice of a Relish that is stored in a LayeredTimeline.
+- Layer -  specific version of a relish Segment in a range of LSNs.
+
+Layers can be InMemory or OnDisk:
+- InMemory layer is not durably stored and needs to rebuild from WAL on pageserver start.
+- OnDisk layer is durably stored.
+
+OnDisk layers can be Image or Delta:
+- ImageLayer represents an image or a snapshot of a segment at one particular LSN.
+- DeltaLayer represents a collection of WAL records or page images in a range of LSNs.
+
+Dropped segments are always represented on disk by DeltaLayer.
+
+LSN range defined by start_lsn and end_lsn:
+- start_lsn is always inclusive.
+- end_lsn depends on layer kind:
+	- InMemoryLayer is either unbounded (end_lsn = MAX_LSN) or dropped (end_lsn = drop_lsn)
+    - ImageLayer represents snapshot at one LSN, so end_lsn = lsn.
+    - DeltaLayer has explicit end_lsn, which represents end of incremental layer.
+
+Layers can be open or historical:
+- Open layer is a writeable one. Only InMemory layer can be open.
+FIXME: If open layer is dropped, it is not writeable, so it should be turned into historical, 
+but now it is not implemented - see bug #569.
+- Historical layer is the one that cannot be modified anymore. Now only OnDisk layers can be historical.
+
+- LayerMap - a map that tracks what layers exist for all the relishes in a timeline.
+
+LayerMap consists of two data structures:
+- segs - All the layers keyed by segment tag
+- open_layers - data structure that hold all open layers ordered by oldest_pending_lsn for quick access during checkpointing. oldest_pending_lsn is the LSN of the oldest page version stored in this layer.
+
+All operations that update InMemory Layers should update both structures to keep them up-to-date.
+
+- LayeredTimeline - implements Timeline interface.
+
+All methods of LayeredTimeline are aware of its ancestors and return data taking them into account.
+TODO: Are there any exceptions to this?
+For example, timeline.list_rels(lsn) will return all segments that are visible in this timeline at the LSN,
+including ones that were not modified in this timeline and thus don't have a layer in the timeline's LayerMap.
+
+TODO:
+Describe GC and checkpoint interval settings.
+
+# Layer files (On-disk layers)
+
+The files are called "layer files". Each layer file corresponds
+to one RELISH_SEG_SIZE slice of a PostgreSQL relation fork or
+non-rel file in a range of LSNs. The layer files
 for each timeline are stored in the timeline's subdirectory under
 .zenith/tenants/<tenantid>/timelines.

-The files are named like this:
+There are two kind of layer file: base images, and deltas. A base
+image file contains a layer of a segment as it was at one LSN,
+whereas a delta file contains modifications to a segment - mostly in
+the form of WAL records - in a range of LSN
+
+base image file:
+
+    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>
+
+delta file:

    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>_<end LSN>

 For example:

+    rel_1663_13990_2609_0_10_000000000169C348
    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000

-Some non-relation files are also stored in repository. For example,
-a CLOG segment would be named like this:
+In addition to the relations, with "rel_*" prefix, we use the same
+format for storing various smaller files from the PostgreSQL data
+directory. They will use different suffixes and the naming scheme up
+to the LSNs vary. The Zenith source code uses the term "relish" to
+mean "a relation, or other file that's treated like a relation in the
+storage" For example, a base image of a CLOG segment would be named
+like this:

-    pg_xact_0000_0_00000000198B06B0_00000000198C2550
+    pg_xact_0000_0_00000000198B06B0

 There is no difference in how the relation and non-relation files are
 managed, except that the first part of file names is different.
 Internally, the relations and non-relation files that are managed in
 the versioned store are together called "relishes".

-Each snapshot file contains a full snapshot, that is, full copy of all
-pages in the relation, as of the "start LSN". It also contains all WAL
-records applicable to the relation between the start and end
-LSNs. With this information, the page server can reconstruct any page
-version of the relation in the LSN range.
-
-If a file has been dropped, the last snapshot file for it is created
+If a file has been dropped, the last layer file for it is created
 with the _DROPPED suffix, e.g.

    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000_DROPPED

-In addition to the relations, with "rel_*" prefix, we use the same
-format for storing various smaller files from the PostgreSQL data
-directory. They will use different suffixes and the naming scheme
-up to the LSN range varies. The Zenith source code uses the term
-"relish" to mean "a relation, or other file that's treated like a
-relation in the storage"

 ## Notation used in this document

-The full path of a snapshot file looks like this:
+The full path of a delta file looks like this:

    .zenith/tenants/941ddc8604413b88b3d208bddf90396c/timelines/4af489b06af8eed9e27a841775616962/rel_1663_13990_2609_0_10_000000000169C348_0000000001702000

@@ -57,50 +112,68 @@ For simplicity, the examples below use a simplified notation for the
 paths.  The tenant ID is left out, the timeline ID is replaced with
 the human-readable branch name, and spcnode+dbnode+relnode+forkum+segno
 with a human-readable table name. The LSNs are also shorter. For
-example, a snapshot file for 'orders' table on 'main' branch, with LSN
-range 100-200 would be:
+example, a base image file at LSN 100 and a delta file between 100-200
+for 'orders' table on 'main' branch is represented like this:

+    main/orders_100
    main/orders_100_200


-# Creating snapshot files
+# Creating layer files

 Let's start with a simple example with a system that contains one
 branch called 'main' and two tables, 'orders' and 'customers'. The end
 of WAL is currently at LSN 250. In this starting situation, you would
-have two files on disk:
+have these files on disk:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
+	main/customers_100
 	main/customers_100_200
+	main/customers_200

 In addition to those files, the recent changes between LSN 200 and the
 end of WAL at 250 are kept in memory. If the page server crashes, the
 latest records between 200-250 need to be re-read from the WAL.

 Whenever enough WAL has been accumulated in memory, the page server
-writes out the changes in memory into new snapshot files. This process
+writes out the changes in memory into new layer files. This process
 is called "checkpointing" (not to be confused with the PostgreSQL
 checkpoints, that's a different thing). The page server only creates
-snapshot files for relations that have been modified since the last
+layer files for relations that have been modified since the last
 checkpoint. For example, if the current end of WAL is at LSN 450, and
 the last checkpoint happened at LSN 400 but there hasn't been any
 recent changes to 'customers' table, you would have these files on
 disk:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
+	main/orders_300
 	main/orders_300_400
+	main/orders_400
+	main/customers_100
 	main/customers_100_200
+	main/customers_200

 If the customers table is modified later, a new file is created for it
 at the next checkpoint. The new file will cover the "gap" from the
-last snapshot file, so the LSN ranges are always contiguous:
+last layer file, so the LSN ranges are always contiguous:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
+	main/orders_300
 	main/orders_300_400
+	main/orders_400
+	main/customers_100
 	main/customers_100_200
+	main/customers_200
 	main/customers_200_500
+	main/customers_500

 ## Reading page versions

@@ -109,29 +182,17 @@ page server needs to reconstruct the requested page, as it was at the
 requested LSN. To do that, the page server first checks the recent
 in-memory layer; if the requested page version is found there, it can
 be returned immediatedly without looking at the files on
-disk. Otherwise the page server needs to locate the snapshot file that
+disk. Otherwise the page server needs to locate the layer file that
 contains the requested page version.

 For example, if a request comes in for table 'orders' at LSN 250, the
 page server would load the 'main/orders_200_300' file into memory, and
 reconstruct and return the requested page from it, as it was at
-LSN 250. Because the snapshot file consists of a full image of the
+LSN 250. Because the layer file consists of a full image of the
 relation at the start LSN and the WAL, reconstructing the page
 involves replaying any WAL records applicable to the page between LSNs
 200-250, starting from the base image at LSN 200.

-A request at a file boundary can be satisfied using either file. For
-example, if there are two files on disk:
-
-	main/orders_100_200
-	main/orders_200_300
-
-And a request comes with LSN 200, either file can be used for it. It
-is better to use the later file, however, because it contains an
-already materialized version of all the pages at LSN 200. Using the
-first file, you would need to apply any WAL records between 100 and
-200 to reconstruct the requested page.
-
 # Multiple branches

 Imagine that a child branch is created at LSN 250:
@@ -145,16 +206,24 @@ Imagine that a child branch is created at LSN 250:
 Then, the 'orders' table is updated differently on the 'main' and
 'child' branches. You now have this situation on disk:

+    main/orders_100
    main/orders_100_200
+    main/orders_200
    main/orders_200_300
+    main/orders_300
    main/orders_300_400
+    main/orders_400
+    main/customers_100
    main/customers_100_200
+    main/customers_200
    child/orders_250_300
+    child/orders_300
    child/orders_300_400
+    child/orders_400

 Because the 'customers' table hasn't been modified on the child
 branch, there is no file for it there. If you request a page for it on
-the 'child' branch, the page server will not find any snapshot file
+the 'child' branch, the page server will not find any layer file
 for it in the 'child' directory, so it will recurse to look into the
 parent 'main' branch instead.

@@ -163,24 +232,34 @@ is linear, and the request's LSN identifies unambiguously which file
 you need to look at. For example, the history for the 'orders' table
 on the 'main' branch consists of these files:

+    main/orders_100
    main/orders_100_200
+    main/orders_200
    main/orders_200_300
+    main/orders_300
    main/orders_300_400
+    main/orders_400

 And from the 'child' branch's point of view, it consists of these
 files:

+    main/orders_100
    main/orders_100_200
+    main/orders_200
    main/orders_200_300
    child/orders_250_300
+    child/orders_300
    child/orders_300_400
+    child/orders_400

 The branch metadata includes the point where the child branch was
 created, LSN 250. If a page request comes with LSN 275, we read the
-page version from the 'child/orders_250_300' file. If the request LSN
-is 225, we read it from the 'main/orders_200_300' file instead.  The
-page versions between 250-300 in the 'main/orders_200_300' file are
-ignored when operating on the child branch.
+page version from the 'child/orders_250_300' file. We might also
+need to reconstruct the page version as it was at LSN 250, in order
+to replay the WAL up to LSN 275, using 'main/orders_200_300' and
+'main/orders_200'. The page versions between 250-300 in the
+'main/orders_200_300' file are ignored when operating on the child
+branch.

 Note: It doesn't make any difference if the child branch is created
 when the end of the main branch was at LSN 250, or later when the tip of
@@ -190,7 +269,7 @@ branch at a historic LSN, is how we support PITR in Zenith.

 # Garbage collection

-In this scheme, we keep creating new snapshot files over time. We also
+In this scheme, we keep creating new layer files over time. We also
 need a mechanism to remove old files that are no longer needed,
 because disk space isn't infinite.

@@ -204,18 +283,38 @@ Let's look at the single branch scenario again. Imagine that the end
 of the branch is LSN 525, so that the GC horizon is currently at
 525-150 = 375

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
+	main/orders_300
 	main/orders_300_400
+	main/orders_400
 	main/orders_400_500
+	main/orders_500
+	main/customers_100
 	main/customers_100_200
+	main/customers_200

-We can remove files 'main/orders_100_200' and 'main/orders_200_300',
-because the end LSNs of those files are older than GC horizon 375, and
-there are more recent snapshot files for the table. 'main/orders_300_400'
-and 'main/orders_400_500' are still within the horizon, so they must be
-retained. 'main/customers_100_200' is old enough, but it cannot be
-removed because there is no newer snapshot file for the table.
+We can remove the following files because the end LSNs of those files are
+older than GC horizon 375, and there are more recent layer files for the
+table:
+
+	main/orders_100       DELETE
+	main/orders_100_200   DELETE
+	main/orders_200       DELETE
+	main/orders_200_300   DELETE
+	main/orders_300       STILL NEEDED BY orders_300_400
+	main/orders_300_400   KEEP, NEWER THAN GC HORIZON
+	main/orders_400       .. 
+	main/orders_400_500   .. 
+	main/orders_500       .. 
+	main/customers_100      DELETE
+	main/customers_100_200  DELETE
+	main/customers_200      KEEP, NO NEWER VERSION
+
+'main/customers_100_200' is old enough, but it cannot be
+removed because there is no newer layer file for the table.

 Things get slightly more complicated with multiple branches. All of
 the above still holds, but in addition to recent files we must also
@@ -223,41 +322,47 @@ retain older shapshot files that are still needed by child branches.
 For example, if child branch is created at LSN 150, and the 'customers'
 table is updated on the branch, you would have these files:

-	main/orders_100_200
-	main/orders_200_300
-	main/orders_300_400
-	main/orders_400_500
-	main/customers_100_200
-	child/customers_150_300
+	main/orders_100        KEEP, NEEDED BY child BRANCH
+	main/orders_100_200    KEEP, NEEDED BY child BRANCH
+	main/orders_200        DELETE
+	main/orders_200_300    DELETE
+	main/orders_300        KEEP, NEWER THAN GC HORIZON
+	main/orders_300_400    KEEP, NEWER THAN GC HORIZON
+	main/orders_400        KEEP, NEWER THAN GC HORIZON
+	main/orders_400_500    KEEP, NEWER THAN GC HORIZON
+	main/orders_500        KEEP, NEWER THAN GC HORIZON
+	main/customers_100       DELETE
+	main/customers_100_200   DELETE
+	main/customers_200       KEEP, NO NEWER VERSION
+	child/customers_150_300  DELETE
+	child/customers_300      KEEP, NO NEWER VERSION

-In this situation, the 'main/orders_100_200' file cannot be removed,
-even though it is older than the GC horizon, because it is still
-needed by the child branch.  'main/orders_200_300' can still be
-removed. So after garbage collection, these files would remain:
-
-	main/orders_100_200
-
-	main/orders_300_400
-	main/orders_400_500
-	main/customers_100_200
-	child/customers_150_300
+In this situation, 'main/orders_100' and 'main/orders_100_200' cannot
+be removed, even though they are older than the GC horizon, because
+they are still needed by the child branch. 'main/orders_200'
+and 'main/orders_200_300' can still be removed.

 If 'orders' is modified later on the 'child' branch, we will create a
-snapshot file for it on the child:
+new base image and delta file for it on the child:

+	main/orders_100
 	main/orders_100_200

+	main/orders_300
 	main/orders_300_400
+	main/orders_400
 	main/orders_400_500
-	main/customers_100_200
-	child/customers_150_300
+	main/orders_500
+	main/customers_200
+	child/customers_300
 	child/orders_150_400
+	child/orders_400

-After this, the 'main/orders_100_200' file can be removed. It is no
-longer needed by the child branch, because there is a newer snapshot
-file there. TODO: This optimization hasn't been implemented! The GC
-algorithm will currently keep the file on the 'main' branch anyway, for
-as long as the child branch exists.
+After this, the 'main/orders_100' and 'main/orders_100_200' file could
+be removed. It is no longer needed by the child branch, because there
+is a newer layer file there. TODO: This optimization hasn't been
+implemented! The GC algorithm will currently keep the file on the
+'main' branch anyway, for as long as the child branch exists.


 # TODO: On LSN ranges
@@ -265,34 +370,48 @@ as long as the child branch exists.
 In principle, each relation can be checkpointed separately, i.e. the
 LSN ranges of the files don't need to line up. So this would be legal:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
+	main/orders_300
 	main/orders_300_400
+	main/orders_400
+	main/customers_150
 	main/customers_150_250
+	main/customers_250
 	main/customers_250_500
+	main/customers_500

 However, the code currently always checkpoints all relations together.
 So that situation doesn't arise in practice.

 It would also be OK to have overlapping LSN ranges for the same relation:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
+	main/orders_300
 	main/orders_250_350
+	main/orders_350
 	main/orders_300_400
+	main/orders_400

-The code that reads the snapshot files should cope with this, but this
+The code that reads the layer files should cope with this, but this
 situation doesn't arise either, because the checkpointing code never
 does that.  It could be useful, however, as a transient state when
 garbage collecting around branch points, or explicit recovery
 points. For example, if we start with this:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
-	main/orders_300_400
+	main/orders_300

 And there is a branch or explicit recovery point at LSN 150, we could
-replace 'main/orders_100_200' with 'main/orders_150_150' to keep a
-snapshot only at that exact point that's still needed, removing the
+replace 'main/orders_100_200' with 'main/orders_150' to keep a
+layer only at that exact point that's still needed, removing the
 other page versions around it. But such compaction has not been
 implemented yet.
--- a/pageserver/src/layered_repository/blob.rs
+++ b/pageserver/src/layered_repository/blob.rs
@@ -0,0 +1,45 @@
+use std::{fs::File, io::Write};
+
+use anyhow::Result;
+use bookfile::{BookWriter, BoundedReader, ChapterId, ChapterWriter};
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+pub struct BlobRange {
+    offset: u64,
+    size: usize,
+}
+
+pub fn read_blob(reader: &BoundedReader<&'_ File>, range: &BlobRange) -> Result<Vec<u8>> {
+    let mut buf = vec![0u8; range.size];
+    reader.read_exact_at(&mut buf, range.offset)?;
+    Ok(buf)
+}
+
+pub struct BlobWriter<W> {
+    writer: ChapterWriter<W>,
+    offset: u64,
+}
+
+impl<W: Write> BlobWriter<W> {
+    // This function takes a BookWriter and creates a new chapter to ensure offset is 0.
+    pub fn new(book_writer: BookWriter<W>, chapter_id: impl Into<ChapterId>) -> Self {
+        let writer = book_writer.new_chapter(chapter_id);
+        Self { writer, offset: 0 }
+    }
+
+    pub fn write_blob(&mut self, blob: &[u8]) -> Result<BlobRange> {
+        self.writer.write_all(blob)?;
+
+        let range = BlobRange {
+            offset: self.offset,
+            size: blob.len(),
+        };
+        self.offset += blob.len() as u64;
+        Ok(range)
+    }
+
+    pub fn close(self) -> bookfile::Result<BookWriter<W>> {
+        self.writer.close()
+    }
+}
--- a/pageserver/src/layered_repository/delta_layer.rs
+++ b/pageserver/src/layered_repository/delta_layer.rs
@@ -0,0 +1,618 @@
+//!
+//! A DeltaLayer represents a collection of WAL records or page images in a range of
+//! LSNs, for one segment. It is stored on a file on disk.
+//!
+//! Usually a delta layer only contains differences - in the form of WAL records against
+//! a base LSN. However, if a segment is newly created, by creating a new relation or
+//! extending an old one, there might be no base image. In that case, all the entries in
+//! the delta layer must be page images or WAL records with the 'will_init' flag set, so
+//! that they can be replayed without referring to an older page version. Also in some
+//! circumstances, the predecessor layer might actually be another delta layer. That
+//! can happen when you create a new branch in the middle of a delta layer, and the WAL
+//! records on the new branch are put in a new delta layer.
+//!
+//! When a delta file needs to be accessed, we slurp the metadata and relsize chapters
+//! into memory, into the DeltaLayerInner struct. See load() and unload() functions.
+//! To access a page/WAL record, we search `page_version_metas` for the block # and LSN.
+//! The byte ranges in the metadata can be used to find the page/WAL record in
+//! PAGE_VERSIONS_CHAPTER.
+//!
+//! On disk, the delta files are stored in timelines/<timelineid> directory.
+//! Currently, there are no subdirectories, and each delta file is named like this:
+//!
+//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>_<end LSN>
+//!
+//! For example:
+//!
+//!    1663_13990_2609_0_5_000000000169C348_000000000169C349
+//!
+//! If a relation is dropped, we add a '_DROPPED' to the end of the filename to indicate that.
+//! So the above example would become:
+//!
+//!    1663_13990_2609_0_5_000000000169C348_000000000169C349_DROPPED
+//!
+//! The end LSN indicates when it was dropped in that case, we don't store it in the
+//! file contents in any way.
+//!
+//! A detlta file is constructed using the 'bookfile' crate. Each file consists of two
+//! parts: the page versions and the relation sizes. They are stored as separate chapters.
+//!
+use crate::layered_repository::blob::BlobWriter;
+use crate::layered_repository::filename::{DeltaFileName, PathOrConf};
+use crate::layered_repository::storage_layer::{
+    Layer, PageReconstructData, PageReconstructResult, PageVersion, SegmentTag,
+};
+use crate::repository::WALRecord;
+use crate::waldecoder;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
+use anyhow::{bail, Result};
+use bytes::Bytes;
+use log::*;
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
+// avoid binding to Write (conflicts with std::io::Write)
+// while being able to use std::fmt::Write's methods
+use std::fmt::Write as _;
+use std::fs;
+use std::fs::File;
+use std::io::{BufWriter, Write};
+use std::ops::Bound::Included;
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, Mutex, MutexGuard};
+
+use bookfile::{Book, BookWriter};
+
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
+
+use super::blob::{read_blob, BlobRange};
+
+// Magic constant to identify a Zenith delta file
+pub const DELTA_FILE_MAGIC: u32 = 0x5A616E01;
+
+/// Mapping from (block #, lsn) -> page/WAL record
+/// byte ranges in PAGE_VERSIONS_CHAPTER
+static PAGE_VERSION_METAS_CHAPTER: u64 = 1;
+/// Page/WAL bytes - cannot be interpreted
+/// without PAGE_VERSION_METAS_CHAPTER
+static PAGE_VERSIONS_CHAPTER: u64 = 2;
+static REL_SIZES_CHAPTER: u64 = 3;
+
+/// Contains the [`Summary`] struct
+static SUMMARY_CHAPTER: u64 = 4;
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+struct Summary {
+    tenantid: ZTenantId,
+    timelineid: ZTimelineId,
+    seg: SegmentTag,
+
+    start_lsn: Lsn,
+    end_lsn: Lsn,
+
+    dropped: bool,
+}
+
+impl From<&DeltaLayer> for Summary {
+    fn from(layer: &DeltaLayer) -> Self {
+        Self {
+            tenantid: layer.tenantid,
+            timelineid: layer.timelineid,
+            seg: layer.seg,
+
+            start_lsn: layer.start_lsn,
+            end_lsn: layer.end_lsn,
+
+            dropped: layer.dropped,
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+struct PageVersionMeta {
+    page_image_range: Option<BlobRange>,
+    record_range: Option<BlobRange>,
+}
+
+///
+/// DeltaLayer is the in-memory data structure associated with an
+/// on-disk delta file.  We keep a DeltaLayer in memory for each
+/// file, in the LayerMap. If a layer is in "loaded" state, we have a
+/// copy of the file in memory, in 'inner'. Otherwise the struct is
+/// just a placeholder for a file that exists on disk, and it needs to
+/// be loaded before using it in queries.
+///
+pub struct DeltaLayer {
+    path_or_conf: PathOrConf,
+
+    pub tenantid: ZTenantId,
+    pub timelineid: ZTimelineId,
+    pub seg: SegmentTag,
+
+    //
+    // This entry contains all the changes from 'start_lsn' to 'end_lsn'. The
+    // start is inclusive, and end is exclusive.
+    //
+    pub start_lsn: Lsn,
+    pub end_lsn: Lsn,
+
+    dropped: bool,
+
+    /// Predecessor layer
+    predecessor: Option<Arc<dyn Layer>>,
+
+    inner: Mutex<DeltaLayerInner>,
+}
+
+pub struct DeltaLayerInner {
+    /// If false, the 'page_version_metas' and 'relsizes' have not been
+    /// loaded into memory yet.
+    loaded: bool,
+
+    /// All versions of all pages in the file are are kept here.
+    /// Indexed by block number and LSN.
+    page_version_metas: BTreeMap<(u32, Lsn), PageVersionMeta>,
+
+    /// `relsizes` tracks the size of the relation at different points in time.
+    relsizes: BTreeMap<Lsn, u32>,
+}
+
+impl Layer for DeltaLayer {
+    fn get_timeline_id(&self) -> ZTimelineId {
+        self.timelineid
+    }
+
+    fn get_seg_tag(&self) -> SegmentTag {
+        self.seg
+    }
+
+    fn is_dropped(&self) -> bool {
+        self.dropped
+    }
+
+    fn get_start_lsn(&self) -> Lsn {
+        self.start_lsn
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
+        self.end_lsn
+    }
+
+    fn filename(&self) -> PathBuf {
+        PathBuf::from(
+            DeltaFileName {
+                seg: self.seg,
+                start_lsn: self.start_lsn,
+                end_lsn: self.end_lsn,
+                dropped: self.dropped,
+            }
+            .to_string(),
+        )
+    }
+
+    fn path(&self) -> Option<PathBuf> {
+        Some(Self::path_for(
+            &self.path_or_conf,
+            self.timelineid,
+            self.tenantid,
+            &DeltaFileName {
+                seg: self.seg,
+                start_lsn: self.start_lsn,
+                end_lsn: self.end_lsn,
+                dropped: self.dropped,
+            },
+        ))
+    }
+
+    /// Look up given page in the cache.
+    fn get_page_reconstruct_data(
+        &self,
+        blknum: u32,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<PageReconstructResult> {
+        let mut need_image = true;
+
+        assert!(self.seg.blknum_in_seg(blknum));
+
+        {
+            // Open the file and lock the metadata in memory
+            // TODO: avoid opening the file for each read
+            let (_path, book) = self.open_book()?;
+            let page_version_reader = book.chapter_reader(PAGE_VERSIONS_CHAPTER)?;
+            let inner = self.load()?;
+
+            // Scan the metadata BTreeMap backwards, starting from the given entry.
+            let minkey = (blknum, Lsn(0));
+            let maxkey = (blknum, lsn);
+            let mut iter = inner
+                .page_version_metas
+                .range((Included(&minkey), Included(&maxkey)));
+            while let Some(((_blknum, _entry_lsn), entry)) = iter.next_back() {
+                if let Some(img_range) = &entry.page_image_range {
+                    // Found a page image, return it
+                    let img = Bytes::from(read_blob(&page_version_reader, img_range)?);
+                    reconstruct_data.page_img = Some(img);
+                    need_image = false;
+                    break;
+                } else if let Some(rec_range) = &entry.record_range {
+                    let rec = WALRecord::des(&read_blob(&page_version_reader, rec_range)?)?;
+                    let will_init = rec.will_init;
+                    reconstruct_data.records.push(rec);
+                    if will_init {
+                        // This WAL record initializes the page, so no need to go further back
+                        need_image = false;
+                        break;
+                    }
+                } else {
+                    // No base image, and no WAL record. Huh?
+                    bail!("no page image or WAL record for requested page");
+                }
+            }
+
+            // release metadata lock and close the file
+        }
+
+        // If an older page image is needed to reconstruct the page, let the
+        // caller know about the predecessor layer.
+        if need_image {
+            if let Some(cont_layer) = &self.predecessor {
+                Ok(PageReconstructResult::Continue(
+                    self.start_lsn,
+                    Arc::clone(cont_layer),
+                ))
+            } else {
+                Ok(PageReconstructResult::Missing(self.start_lsn))
+            }
+        } else {
+            Ok(PageReconstructResult::Complete)
+        }
+    }
+
+    /// Get size of the relation at given LSN
+    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
+        assert!(lsn >= self.start_lsn);
+
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let inner = self.load()?;
+        let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn)));
+
+        let result;
+        if let Some((_entry_lsn, entry)) = iter.next_back() {
+            result = *entry;
+        // Use the base image if needed
+        } else if let Some(predecessor) = &self.predecessor {
+            result = predecessor.get_seg_size(lsn)?;
+        } else {
+            result = 0;
+        }
+        Ok(result)
+    }
+
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
+        // Is the requested LSN after the rel was dropped?
+        if self.dropped && lsn >= self.end_lsn {
+            return Ok(false);
+        }
+
+        // Otherwise, it exists.
+        Ok(true)
+    }
+
+    ///
+    /// Release most of the memory used by this layer. If it's accessed again later,
+    /// it will need to be loaded back.
+    ///
+    fn unload(&self) -> Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.page_version_metas = BTreeMap::new();
+        inner.relsizes = BTreeMap::new();
+        inner.loaded = false;
+        Ok(())
+    }
+
+    fn delete(&self) -> Result<()> {
+        // delete underlying file
+        if let Some(path) = self.path() {
+            fs::remove_file(path)?;
+        }
+        Ok(())
+    }
+
+    fn is_incremental(&self) -> bool {
+        true
+    }
+
+    /// debugging function to print out the contents of the layer
+    fn dump(&self) -> Result<()> {
+        println!(
+            "----- delta layer for ten {} tli {} seg {} {}-{} ----",
+            self.tenantid, self.timelineid, self.seg, self.start_lsn, self.end_lsn
+        );
+
+        println!("--- relsizes ---");
+        let inner = self.load()?;
+        for (k, v) in inner.relsizes.iter() {
+            println!("  {}: {}", k, v);
+        }
+        println!("--- page versions ---");
+        let (_path, book) = self.open_book()?;
+        let chapter = book.chapter_reader(PAGE_VERSIONS_CHAPTER)?;
+        for (k, v) in inner.page_version_metas.iter() {
+            let mut desc = String::new();
+
+            if let Some(page_image_range) = v.page_image_range.as_ref() {
+                let image = read_blob(&chapter, page_image_range)?;
+                write!(&mut desc, " img {} bytes", image.len())?;
+            }
+            if let Some(record_range) = v.record_range.as_ref() {
+                let record_bytes = read_blob(&chapter, record_range)?;
+                let rec = WALRecord::des(&record_bytes)?;
+                let wal_desc = waldecoder::describe_wal_record(&rec.rec);
+                write!(
+                    &mut desc,
+                    " rec {} bytes will_init: {} {}",
+                    rec.rec.len(),
+                    rec.will_init,
+                    wal_desc
+                )?;
+            }
+            println!("  blk {} at {}: {}", k.0, k.1, desc);
+        }
+
+        Ok(())
+    }
+}
+
+impl DeltaLayer {
+    fn path_for(
+        path_or_conf: &PathOrConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        fname: &DeltaFileName,
+    ) -> PathBuf {
+        match path_or_conf {
+            PathOrConf::Path(path) => path.clone(),
+            PathOrConf::Conf(conf) => conf
+                .timeline_path(&timelineid, &tenantid)
+                .join(fname.to_string()),
+        }
+    }
+
+    /// Create a new delta file, using the given btreemaps containing the page versions and
+    /// relsizes.
+    ///
+    /// This is used to write the in-memory layer to disk. The in-memory layer uses the same
+    /// data structure with two btreemaps as we do, so passing the btreemaps is currently
+    /// expedient.
+    #[allow(clippy::too_many_arguments)]
+    pub fn create(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        seg: SegmentTag,
+        start_lsn: Lsn,
+        end_lsn: Lsn,
+        dropped: bool,
+        predecessor: Option<Arc<dyn Layer>>,
+        page_versions: BTreeMap<(u32, Lsn), PageVersion>,
+        relsizes: BTreeMap<Lsn, u32>,
+    ) -> Result<DeltaLayer> {
+        let delta_layer = DeltaLayer {
+            path_or_conf: PathOrConf::Conf(conf),
+            timelineid,
+            tenantid,
+            seg,
+            start_lsn,
+            end_lsn,
+            dropped,
+            inner: Mutex::new(DeltaLayerInner {
+                loaded: true,
+                page_version_metas: BTreeMap::new(),
+                relsizes,
+            }),
+            predecessor,
+        };
+        let mut inner = delta_layer.inner.lock().unwrap();
+
+        // Write the in-memory btreemaps into a file
+        let path = delta_layer
+            .path()
+            .expect("DeltaLayer is supposed to have a layer path on disk");
+
+        // Note: This overwrites any existing file. There shouldn't be any.
+        // FIXME: throw an error instead?
+        let file = File::create(&path)?;
+        let buf_writer = BufWriter::new(file);
+        let book = BookWriter::new(buf_writer, DELTA_FILE_MAGIC)?;
+
+        let mut page_version_writer = BlobWriter::new(book, PAGE_VERSIONS_CHAPTER);
+
+        for (key, page_version) in page_versions {
+            let page_image_range = page_version
+                .page_image
+                .map(|page_image| page_version_writer.write_blob(page_image.as_ref()))
+                .transpose()?;
+
+            let record_range = page_version
+                .record
+                .map(|record| {
+                    let buf = WALRecord::ser(&record)?;
+                    page_version_writer.write_blob(&buf)
+                })
+                .transpose()?;
+
+            let old = inner.page_version_metas.insert(
+                key,
+                PageVersionMeta {
+                    page_image_range,
+                    record_range,
+                },
+            );
+
+            assert!(old.is_none());
+        }
+
+        let book = page_version_writer.close()?;
+
+        // Write out page versions
+        let mut chapter = book.new_chapter(PAGE_VERSION_METAS_CHAPTER);
+        let buf = BTreeMap::ser(&inner.page_version_metas)?;
+        chapter.write_all(&buf)?;
+        let book = chapter.close()?;
+
+        // and relsizes to separate chapter
+        let mut chapter = book.new_chapter(REL_SIZES_CHAPTER);
+        let buf = BTreeMap::ser(&inner.relsizes)?;
+        chapter.write_all(&buf)?;
+        let book = chapter.close()?;
+
+        let mut chapter = book.new_chapter(SUMMARY_CHAPTER);
+        let summary = Summary {
+            tenantid,
+            timelineid,
+            seg,
+
+            start_lsn,
+            end_lsn,
+
+            dropped,
+        };
+        Summary::ser_into(&summary, &mut chapter)?;
+        let book = chapter.close()?;
+
+        // This flushes the underlying 'buf_writer'.
+        book.close()?;
+
+        trace!("saved {}", &path.display());
+
+        drop(inner);
+
+        Ok(delta_layer)
+    }
+
+    fn open_book(&self) -> Result<(PathBuf, Book<File>)> {
+        let path = Self::path_for(
+            &self.path_or_conf,
+            self.timelineid,
+            self.tenantid,
+            &DeltaFileName {
+                seg: self.seg,
+                start_lsn: self.start_lsn,
+                end_lsn: self.end_lsn,
+                dropped: self.dropped,
+            },
+        );
+
+        let file = File::open(&path)?;
+        let book = Book::new(file)?;
+
+        Ok((path, book))
+    }
+
+    ///
+    /// Load the contents of the file into memory
+    ///
+    fn load(&self) -> Result<MutexGuard<DeltaLayerInner>> {
+        // quick exit if already loaded
+        let mut inner = self.inner.lock().unwrap();
+
+        if inner.loaded {
+            return Ok(inner);
+        }
+
+        let (path, book) = self.open_book()?;
+
+        match &self.path_or_conf {
+            PathOrConf::Conf(_) => {
+                let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
+                let actual_summary = Summary::des(&chapter)?;
+
+                let expected_summary = Summary::from(self);
+
+                if actual_summary != expected_summary {
+                    bail!("in-file summary does not match expected summary. actual = {:?} expected = {:?}", actual_summary, expected_summary);
+                }
+            }
+            PathOrConf::Path(path) => {
+                let actual_filename = Path::new(path.file_name().unwrap());
+                let expected_filename = self.filename();
+
+                if actual_filename != expected_filename {
+                    println!(
+                        "warning: filename does not match what is expected from in-file summary"
+                    );
+                    println!("actual: {:?}", actual_filename);
+                    println!("expected: {:?}", expected_filename);
+                }
+            }
+        }
+
+        let chapter = book.read_chapter(PAGE_VERSION_METAS_CHAPTER)?;
+        let page_version_metas = BTreeMap::des(&chapter)?;
+
+        let chapter = book.read_chapter(REL_SIZES_CHAPTER)?;
+        let relsizes = BTreeMap::des(&chapter)?;
+
+        debug!("loaded from {}", &path.display());
+
+        *inner = DeltaLayerInner {
+            loaded: true,
+            page_version_metas,
+            relsizes,
+        };
+
+        Ok(inner)
+    }
+
+    /// Create a DeltaLayer struct representing an existing file on disk.
+    pub fn new(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        filename: &DeltaFileName,
+        predecessor: Option<Arc<dyn Layer>>,
+    ) -> DeltaLayer {
+        DeltaLayer {
+            path_or_conf: PathOrConf::Conf(conf),
+            timelineid,
+            tenantid,
+            seg: filename.seg,
+            start_lsn: filename.start_lsn,
+            end_lsn: filename.end_lsn,
+            dropped: filename.dropped,
+            inner: Mutex::new(DeltaLayerInner {
+                loaded: false,
+                page_version_metas: BTreeMap::new(),
+                relsizes: BTreeMap::new(),
+            }),
+            predecessor,
+        }
+    }
+
+    /// Create a DeltaLayer struct representing an existing file on disk.
+    ///
+    /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary.
+    pub fn new_for_path(path: &Path, book: &Book<File>) -> Result<Self> {
+        let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
+        let summary = Summary::des(&chapter)?;
+
+        Ok(DeltaLayer {
+            path_or_conf: PathOrConf::Path(path.to_path_buf()),
+            timelineid: summary.timelineid,
+            tenantid: summary.tenantid,
+            seg: summary.seg,
+            start_lsn: summary.start_lsn,
+            end_lsn: summary.end_lsn,
+            dropped: summary.dropped,
+            inner: Mutex::new(DeltaLayerInner {
+                loaded: false,
+                page_version_metas: BTreeMap::new(),
+                relsizes: BTreeMap::new(),
+            }),
+            predecessor: None,
+        })
+    }
+}
--- a/pageserver/src/layered_repository/filename.rs
+++ b/pageserver/src/layered_repository/filename.rs
@@ -0,0 +1,313 @@
+//!
+//! Helper functions for dealing with filenames of the image and delta layer files.
+//!
+use crate::layered_repository::storage_layer::SegmentTag;
+use crate::relish::*;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
+use std::fmt;
+use std::fs;
+use std::path::PathBuf;
+
+use anyhow::Result;
+use log::*;
+use zenith_utils::lsn::Lsn;
+
+// Note: LayeredTimeline::load_layer_map() relies on this sort order
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+pub struct DeltaFileName {
+    pub seg: SegmentTag,
+    pub start_lsn: Lsn,
+    pub end_lsn: Lsn,
+    pub dropped: bool,
+}
+
+/// Represents the filename of a DeltaLayer
+///
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>
+///
+/// or if it was dropped:
+///
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>_DROPPED
+///
+impl DeltaFileName {
+    ///
+    /// Parse a string as a delta file name. Returns None if the filename does not
+    /// match the expected pattern.
+    ///
+    pub fn from_str(fname: &str) -> Option<Self> {
+        let rel;
+        let mut parts;
+        if let Some(rest) = fname.strip_prefix("rel_") {
+            parts = rest.split('_');
+            rel = RelishTag::Relation(RelTag {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+                relnode: parts.next()?.parse::<u32>().ok()?,
+                forknum: parts.next()?.parse::<u8>().ok()?,
+            });
+        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
+            parts = rest.split('_');
+            rel = RelishTag::FileNodeMap {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
+            parts = rest.split('_');
+            rel = RelishTag::TwoPhase {
+                xid: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
+            parts = rest.split('_');
+            rel = RelishTag::Checkpoint;
+        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
+            parts = rest.split('_');
+            rel = RelishTag::ControlFile;
+        } else {
+            return None;
+        }
+
+        let segno = parts.next()?.parse::<u32>().ok()?;
+
+        let seg = SegmentTag { rel, segno };
+
+        let start_lsn = Lsn::from_hex(parts.next()?).ok()?;
+        let end_lsn = Lsn::from_hex(parts.next()?).ok()?;
+
+        let mut dropped = false;
+        if let Some(suffix) = parts.next() {
+            if suffix == "DROPPED" {
+                dropped = true;
+            } else {
+                return None;
+            }
+        }
+        if parts.next().is_some() {
+            return None;
+        }
+
+        Some(DeltaFileName {
+            seg,
+            start_lsn,
+            end_lsn,
+            dropped,
+        })
+    }
+}
+
+impl fmt::Display for DeltaFileName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let basename = match self.seg.rel {
+            RelishTag::Relation(reltag) => format!(
+                "rel_{}_{}_{}_{}",
+                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
+            ),
+            RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno,
+            } => format!("pg_xact_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno,
+            } => format!("pg_multixact_members_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno,
+            } => format!("pg_multixact_offsets_{:04X}", segno),
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
+            RelishTag::Checkpoint => "pg_control_checkpoint".to_string(),
+            RelishTag::ControlFile => "pg_control".to_string(),
+        };
+
+        write!(
+            f,
+            "{}_{}_{:016X}_{:016X}{}",
+            basename,
+            self.seg.segno,
+            u64::from(self.start_lsn),
+            u64::from(self.end_lsn),
+            if self.dropped { "_DROPPED" } else { "" }
+        )
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+pub struct ImageFileName {
+    pub seg: SegmentTag,
+    pub lsn: Lsn,
+}
+
+///
+/// Represents the filename of an ImageLayer
+///
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<LSN>
+///
+impl ImageFileName {
+    ///
+    /// Parse a string as an image file name. Returns None if the filename does not
+    /// match the expected pattern.
+    ///
+    pub fn from_str(fname: &str) -> Option<Self> {
+        let rel;
+        let mut parts;
+        if let Some(rest) = fname.strip_prefix("rel_") {
+            parts = rest.split('_');
+            rel = RelishTag::Relation(RelTag {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+                relnode: parts.next()?.parse::<u32>().ok()?,
+                forknum: parts.next()?.parse::<u8>().ok()?,
+            });
+        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
+            parts = rest.split('_');
+            rel = RelishTag::FileNodeMap {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
+            parts = rest.split('_');
+            rel = RelishTag::TwoPhase {
+                xid: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
+            parts = rest.split('_');
+            rel = RelishTag::Checkpoint;
+        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
+            parts = rest.split('_');
+            rel = RelishTag::ControlFile;
+        } else {
+            return None;
+        }
+
+        let segno = parts.next()?.parse::<u32>().ok()?;
+
+        let seg = SegmentTag { rel, segno };
+
+        let lsn = Lsn::from_hex(parts.next()?).ok()?;
+
+        if parts.next().is_some() {
+            return None;
+        }
+
+        Some(ImageFileName { seg, lsn })
+    }
+}
+
+impl fmt::Display for ImageFileName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let basename = match self.seg.rel {
+            RelishTag::Relation(reltag) => format!(
+                "rel_{}_{}_{}_{}",
+                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
+            ),
+            RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno,
+            } => format!("pg_xact_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno,
+            } => format!("pg_multixact_members_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno,
+            } => format!("pg_multixact_offsets_{:04X}", segno),
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
+            RelishTag::Checkpoint => "pg_control_checkpoint".to_string(),
+            RelishTag::ControlFile => "pg_control".to_string(),
+        };
+
+        write!(
+            f,
+            "{}_{}_{:016X}",
+            basename,
+            self.seg.segno,
+            u64::from(self.lsn),
+        )
+    }
+}
+
+/// Scan timeline directory and create ImageFileName and DeltaFilename
+/// structs representing all files on disk
+///
+/// TODO: returning an Iterator would be more idiomatic
+pub fn list_files(
+    conf: &'static PageServerConf,
+    timelineid: ZTimelineId,
+    tenantid: ZTenantId,
+) -> Result<(Vec<ImageFileName>, Vec<DeltaFileName>)> {
+    let path = conf.timeline_path(&timelineid, &tenantid);
+
+    let mut deltafiles: Vec<DeltaFileName> = Vec::new();
+    let mut imgfiles: Vec<ImageFileName> = Vec::new();
+    for direntry in fs::read_dir(path)? {
+        let fname = direntry?.file_name();
+        let fname = fname.to_str().unwrap();
+
+        if let Some(deltafilename) = DeltaFileName::from_str(fname) {
+            deltafiles.push(deltafilename);
+        } else if let Some(imgfilename) = ImageFileName::from_str(fname) {
+            imgfiles.push(imgfilename);
+        } else if fname == "wal" || fname == "metadata" || fname == "ancestor" {
+            // ignore these
+        } else {
+            warn!("unrecognized filename in timeline dir: {}", fname);
+        }
+    }
+    Ok((imgfiles, deltafiles))
+}
+
+/// Helper enum to hold a PageServerConf, or a path
+///
+/// This is used by DeltaLayer and ImageLayer. Normally, this holds a reference to the
+/// global config, and paths to layer files are constructed using the tenant/timeline
+/// path from the config. But in the 'dump_layerfile' binary, we need to construct a Layer
+/// struct for a file on disk, without having a page server running, so that we have no
+/// config. In that case, we use the Path variant to hold the full path to the file on
+/// disk.
+pub enum PathOrConf {
+    Path(PathBuf),
+    Conf(&'static PageServerConf),
+}
--- a/pageserver/src/layered_repository/image_layer.rs
+++ b/pageserver/src/layered_repository/image_layer.rs
@@ -0,0 +1,501 @@
+//! An ImageLayer represents an image or a snapshot of a segment at one particular LSN.
+//! It is stored in a file on disk.
+//!
+//! On disk, the image files are stored in timelines/<timelineid> directory.
+//! Currently, there are no subdirectories, and each image layer file is named like this:
+//!
+//! Note that segno is
+//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<LSN>
+//!
+//! For example:
+//!
+//!    1663_13990_2609_0_5_000000000169C348
+//!
+//! An image file is constructed using the 'bookfile' crate.
+//!
+//! Only metadata is loaded into memory by the load function.
+//! When images are needed, they are read directly from disk.
+//!
+//! For blocky relishes, the images are stored in BLOCKY_IMAGES_CHAPTER.
+//! All the images are required to be BLOCK_SIZE, which allows for random access.
+//!
+//! For non-blocky relishes, the image can be found in NONBLOCKY_IMAGE_CHAPTER.
+//!
+use crate::layered_repository::filename::{ImageFileName, PathOrConf};
+use crate::layered_repository::storage_layer::{
+    Layer, PageReconstructData, PageReconstructResult, SegmentTag,
+};
+use crate::layered_repository::LayeredTimeline;
+use crate::layered_repository::RELISH_SEG_SIZE;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
+use anyhow::{anyhow, bail, ensure, Result};
+use bytes::Bytes;
+use log::*;
+use serde::{Deserialize, Serialize};
+use std::convert::TryInto;
+use std::fs;
+use std::fs::File;
+use std::io::{BufWriter, Write};
+use std::path::{Path, PathBuf};
+use std::sync::{Mutex, MutexGuard};
+
+use bookfile::{Book, BookWriter};
+
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
+
+// Magic constant to identify a Zenith segment image file
+pub const IMAGE_FILE_MAGIC: u32 = 0x5A616E01 + 1;
+
+/// Contains each block in block # order
+const BLOCKY_IMAGES_CHAPTER: u64 = 1;
+const NONBLOCKY_IMAGE_CHAPTER: u64 = 2;
+
+/// Contains the [`Summary`] struct
+const SUMMARY_CHAPTER: u64 = 3;
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+struct Summary {
+    tenantid: ZTenantId,
+    timelineid: ZTimelineId,
+    seg: SegmentTag,
+
+    lsn: Lsn,
+}
+
+impl From<&ImageLayer> for Summary {
+    fn from(layer: &ImageLayer) -> Self {
+        Self {
+            tenantid: layer.tenantid,
+            timelineid: layer.timelineid,
+            seg: layer.seg,
+
+            lsn: layer.lsn,
+        }
+    }
+}
+
+const BLOCK_SIZE: usize = 8192;
+
+///
+/// ImageLayer is the in-memory data structure associated with an on-disk image
+/// file.  We keep an ImageLayer in memory for each file, in the LayerMap. If a
+/// layer is in "loaded" state, we have a copy of the file in memory, in 'inner'.
+/// Otherwise the struct is just a placeholder for a file that exists on disk,
+/// and it needs to be loaded before using it in queries.
+///
+pub struct ImageLayer {
+    path_or_conf: PathOrConf,
+    pub tenantid: ZTenantId,
+    pub timelineid: ZTimelineId,
+    pub seg: SegmentTag,
+
+    // This entry contains an image of all pages as of this LSN
+    pub lsn: Lsn,
+
+    inner: Mutex<ImageLayerInner>,
+}
+
+#[derive(Clone)]
+enum ImageType {
+    Blocky { num_blocks: u32 },
+    NonBlocky,
+}
+
+pub struct ImageLayerInner {
+    /// If false, the 'image_type' has not been
+    /// loaded into memory yet.
+    loaded: bool,
+
+    /// Derived from filename and bookfile chapter metadata
+    image_type: ImageType,
+}
+
+impl Layer for ImageLayer {
+    fn filename(&self) -> PathBuf {
+        PathBuf::from(
+            ImageFileName {
+                seg: self.seg,
+                lsn: self.lsn,
+            }
+            .to_string(),
+        )
+    }
+
+    fn path(&self) -> Option<PathBuf> {
+        Some(Self::path_for(
+            &self.path_or_conf,
+            self.timelineid,
+            self.tenantid,
+            &ImageFileName {
+                seg: self.seg,
+                lsn: self.lsn,
+            },
+        ))
+    }
+
+    fn get_timeline_id(&self) -> ZTimelineId {
+        self.timelineid
+    }
+
+    fn get_seg_tag(&self) -> SegmentTag {
+        self.seg
+    }
+
+    fn is_dropped(&self) -> bool {
+        false
+    }
+
+    fn get_start_lsn(&self) -> Lsn {
+        self.lsn
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
+        self.lsn
+    }
+
+    /// Look up given page in the file
+    fn get_page_reconstruct_data(
+        &self,
+        blknum: u32,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<PageReconstructResult> {
+        assert!(lsn >= self.lsn);
+
+        let inner = self.load()?;
+
+        let base_blknum = blknum % RELISH_SEG_SIZE;
+
+        let (_path, book) = self.open_book()?;
+
+        let buf = match &inner.image_type {
+            ImageType::Blocky { num_blocks } => {
+                if base_blknum >= *num_blocks {
+                    return Ok(PageReconstructResult::Missing(lsn));
+                }
+
+                let mut buf = vec![0u8; BLOCK_SIZE];
+                let offset = BLOCK_SIZE as u64 * base_blknum as u64;
+
+                let chapter = book.chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
+                chapter.read_exact_at(&mut buf, offset)?;
+
+                buf
+            }
+            ImageType::NonBlocky => {
+                ensure!(base_blknum == 0);
+                book.read_chapter(NONBLOCKY_IMAGE_CHAPTER)?.into_vec()
+            }
+        };
+
+        reconstruct_data.page_img = Some(Bytes::from(buf));
+        Ok(PageReconstructResult::Complete)
+    }
+
+    /// Get size of the segment
+    fn get_seg_size(&self, _lsn: Lsn) -> Result<u32> {
+        let inner = self.load()?;
+        match inner.image_type {
+            ImageType::Blocky { num_blocks } => Ok(num_blocks),
+            ImageType::NonBlocky => Err(anyhow!("get_seg_size called for non-blocky segment")),
+        }
+    }
+
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, _lsn: Lsn) -> Result<bool> {
+        Ok(true)
+    }
+
+    ///
+    /// Release most of the memory used by this layer. If it's accessed again later,
+    /// it will need to be loaded back.
+    ///
+    fn unload(&self) -> Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.image_type = ImageType::Blocky { num_blocks: 0 };
+        inner.loaded = false;
+        Ok(())
+    }
+
+    fn delete(&self) -> Result<()> {
+        // delete underlying file
+        if let Some(path) = self.path() {
+            fs::remove_file(path)?;
+        }
+        Ok(())
+    }
+
+    fn is_incremental(&self) -> bool {
+        false
+    }
+
+    /// debugging function to print out the contents of the layer
+    fn dump(&self) -> Result<()> {
+        println!(
+            "----- image layer for ten {} tli {} seg {} at {} ----",
+            self.tenantid, self.timelineid, self.seg, self.lsn
+        );
+
+        let inner = self.load()?;
+
+        match inner.image_type {
+            ImageType::Blocky { num_blocks } => println!("({}) blocks ", num_blocks),
+            ImageType::NonBlocky => {
+                let (_path, book) = self.open_book()?;
+                let chapter = book.read_chapter(NONBLOCKY_IMAGE_CHAPTER)?;
+                println!("non-blocky ({} bytes)", chapter.len());
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl ImageLayer {
+    fn path_for(
+        path_or_conf: &PathOrConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        fname: &ImageFileName,
+    ) -> PathBuf {
+        match path_or_conf {
+            PathOrConf::Path(path) => path.to_path_buf(),
+            PathOrConf::Conf(conf) => conf
+                .timeline_path(&timelineid, &tenantid)
+                .join(fname.to_string()),
+        }
+    }
+
+    /// Create a new image file, using the given array of pages.
+    fn create(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        seg: SegmentTag,
+        lsn: Lsn,
+        base_images: Vec<Bytes>,
+    ) -> Result<ImageLayer> {
+        let image_type = if seg.rel.is_blocky() {
+            let num_blocks: u32 = base_images.len().try_into()?;
+            ImageType::Blocky { num_blocks }
+        } else {
+            assert_eq!(base_images.len(), 1);
+            ImageType::NonBlocky
+        };
+
+        let layer = ImageLayer {
+            path_or_conf: PathOrConf::Conf(conf),
+            timelineid,
+            tenantid,
+            seg,
+            lsn,
+            inner: Mutex::new(ImageLayerInner {
+                loaded: true,
+                image_type: image_type.clone(),
+            }),
+        };
+        let inner = layer.inner.lock().unwrap();
+
+        // Write the images into a file
+        let path = layer
+            .path()
+            .expect("ImageLayer is supposed to have a layer path on disk");
+        // Note: This overwrites any existing file. There shouldn't be any.
+        // FIXME: throw an error instead?
+        let file = File::create(&path)?;
+        let buf_writer = BufWriter::new(file);
+        let book = BookWriter::new(buf_writer, IMAGE_FILE_MAGIC)?;
+
+        let book = match &image_type {
+            ImageType::Blocky { .. } => {
+                let mut chapter = book.new_chapter(BLOCKY_IMAGES_CHAPTER);
+                for block_bytes in base_images {
+                    assert_eq!(block_bytes.len(), BLOCK_SIZE);
+                    chapter.write_all(&block_bytes)?;
+                }
+                chapter.close()?
+            }
+            ImageType::NonBlocky => {
+                let mut chapter = book.new_chapter(NONBLOCKY_IMAGE_CHAPTER);
+                chapter.write_all(&base_images[0])?;
+                chapter.close()?
+            }
+        };
+
+        let mut chapter = book.new_chapter(SUMMARY_CHAPTER);
+        let summary = Summary {
+            tenantid,
+            timelineid,
+            seg,
+
+            lsn,
+        };
+        Summary::ser_into(&summary, &mut chapter)?;
+        let book = chapter.close()?;
+
+        // This flushes the underlying 'buf_writer'.
+        book.close()?;
+
+        trace!("saved {}", &path.display());
+
+        drop(inner);
+
+        Ok(layer)
+    }
+
+    // Create a new image file by materializing every page in a source layer
+    // at given LSN.
+    pub fn create_from_src(
+        conf: &'static PageServerConf,
+        timeline: &LayeredTimeline,
+        src: &dyn Layer,
+        lsn: Lsn,
+    ) -> Result<ImageLayer> {
+        let seg = src.get_seg_tag();
+        let timelineid = timeline.timelineid;
+
+        let startblk;
+        let size;
+        if seg.rel.is_blocky() {
+            size = src.get_seg_size(lsn)?;
+            startblk = seg.segno * RELISH_SEG_SIZE;
+        } else {
+            size = 1;
+            startblk = 0;
+        }
+
+        trace!(
+            "creating new ImageLayer for {} on timeline {} at {}",
+            seg,
+            timelineid,
+            lsn,
+        );
+
+        let mut base_images: Vec<Bytes> = Vec::new();
+        for blknum in startblk..(startblk + size) {
+            let img = timeline.materialize_page(seg, blknum, lsn, &*src)?;
+
+            base_images.push(img);
+        }
+
+        Self::create(conf, timelineid, timeline.tenantid, seg, lsn, base_images)
+    }
+
+    ///
+    /// Load the contents of the file into memory
+    ///
+    fn load(&self) -> Result<MutexGuard<ImageLayerInner>> {
+        // quick exit if already loaded
+        let mut inner = self.inner.lock().unwrap();
+
+        if inner.loaded {
+            return Ok(inner);
+        }
+
+        let (path, book) = self.open_book()?;
+
+        match &self.path_or_conf {
+            PathOrConf::Conf(_) => {
+                let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
+                let actual_summary = Summary::des(&chapter)?;
+
+                let expected_summary = Summary::from(self);
+
+                if actual_summary != expected_summary {
+                    bail!("in-file summary does not match expected summary. actual = {:?} expected = {:?}", actual_summary, expected_summary);
+                }
+            }
+            PathOrConf::Path(path) => {
+                let actual_filename = Path::new(path.file_name().unwrap());
+                let expected_filename = self.filename();
+
+                if actual_filename != expected_filename {
+                    println!(
+                        "warning: filename does not match what is expected from in-file summary"
+                    );
+                    println!("actual: {:?}", actual_filename);
+                    println!("expected: {:?}", expected_filename);
+                }
+            }
+        }
+
+        let image_type = if self.seg.rel.is_blocky() {
+            let chapter = book.chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
+            let images_len = chapter.len();
+            ensure!(images_len % BLOCK_SIZE as u64 == 0);
+            let num_blocks: u32 = (images_len / BLOCK_SIZE as u64).try_into()?;
+            ImageType::Blocky { num_blocks }
+        } else {
+            let _chapter = book.chapter_reader(NONBLOCKY_IMAGE_CHAPTER)?;
+            ImageType::NonBlocky
+        };
+
+        debug!("loaded from {}", &path.display());
+
+        *inner = ImageLayerInner {
+            loaded: true,
+            image_type,
+        };
+
+        Ok(inner)
+    }
+
+    fn open_book(&self) -> Result<(PathBuf, Book<File>)> {
+        let path = Self::path_for(
+            &self.path_or_conf,
+            self.timelineid,
+            self.tenantid,
+            &ImageFileName {
+                seg: self.seg,
+                lsn: self.lsn,
+            },
+        );
+
+        let file = File::open(&path)?;
+        let book = Book::new(file)?;
+
+        Ok((path, book))
+    }
+
+    /// Create an ImageLayer struct representing an existing file on disk
+    pub fn new(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        filename: &ImageFileName,
+    ) -> ImageLayer {
+        ImageLayer {
+            path_or_conf: PathOrConf::Conf(conf),
+            timelineid,
+            tenantid,
+            seg: filename.seg,
+            lsn: filename.lsn,
+            inner: Mutex::new(ImageLayerInner {
+                loaded: false,
+                image_type: ImageType::Blocky { num_blocks: 0 },
+            }),
+        }
+    }
+
+    /// Create an ImageLayer struct representing an existing file on disk.
+    ///
+    /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary.
+    pub fn new_for_path(path: &Path, book: &Book<File>) -> Result<ImageLayer> {
+        let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
+        let summary = Summary::des(&chapter)?;
+
+        Ok(ImageLayer {
+            path_or_conf: PathOrConf::Path(path.to_path_buf()),
+            timelineid: summary.timelineid,
+            tenantid: summary.tenantid,
+            seg: summary.seg,
+            lsn: summary.lsn,
+            inner: Mutex::new(ImageLayerInner {
+                loaded: false,
+                image_type: ImageType::Blocky { num_blocks: 0 },
+            }),
+        })
+    }
+}
--- a/pageserver/src/layered_repository/inmemory_layer.rs
+++ b/pageserver/src/layered_repository/inmemory_layer.rs
@@ -2,11 +2,13 @@
 //! An in-memory layer stores recently received page versions in memory. The page versions
 //! are held in a BTreeMap, and there's another BTreeMap to track the size of the relation.
 //!
+use crate::layered_repository::filename::DeltaFileName;
 use crate::layered_repository::storage_layer::{
-    Layer, PageReconstructData, PageVersion, SegmentTag, RELISH_SEG_SIZE,
+    Layer, PageReconstructData, PageReconstructResult, PageVersion, SegmentTag, RELISH_SEG_SIZE,
 };
 use crate::layered_repository::LayeredTimeline;
-use crate::layered_repository::SnapshotLayer;
+use crate::layered_repository::ZERO_PAGE;
+use crate::layered_repository::{DeltaLayer, ImageLayer};
 use crate::repository::WALRecord;
 use crate::PageServerConf;
 use crate::{ZTenantId, ZTimelineId};
@@ -15,6 +17,7 @@ use bytes::Bytes;
 use log::*;
 use std::collections::BTreeMap;
 use std::ops::Bound::Included;
+use std::path::PathBuf;
 use std::sync::{Arc, Mutex};

 use zenith_utils::lsn::Lsn;
@@ -27,11 +30,16 @@ pub struct InMemoryLayer {

    ///
    /// This layer contains all the changes from 'start_lsn'. The
-    /// start is inclusive. There is no end LSN; we only use in-memory
-    /// layer at the end of a timeline.
+    /// start is inclusive.
    ///
    start_lsn: Lsn,

+    /// Frozen in-memory layers have an inclusive end LSN.
+    end_lsn: Option<Lsn>,
+
+    /// LSN of the oldest page version stored in this layer
+    oldest_pending_lsn: Lsn,
+
    /// The above fields never change. The parts that do change are in 'inner',
    /// and protected by mutex.
    inner: Mutex<InMemoryLayerInner>,
@@ -51,22 +59,84 @@ pub struct InMemoryLayerInner {
    /// `segsizes` tracks the size of the segment at different points in time.
    ///
    segsizes: BTreeMap<Lsn, u32>,
+
+    /// Writes are only allowed when true.
+    /// Set to false when this layer is in the process of being replaced.
+    writeable: bool,
+
+    /// Predecessor layer
+    predecessor: Option<Arc<dyn Layer>>,
+}
+
+impl InMemoryLayerInner {
+    fn check_writeable(&self) -> WriteResult<()> {
+        if self.writeable {
+            Ok(())
+        } else {
+            Err(NonWriteableError)
+        }
+    }
+
+    fn get_seg_size(&self, lsn: Lsn) -> u32 {
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let mut iter = self.segsizes.range((Included(&Lsn(0)), Included(&lsn)));
+
+        if let Some((_entry_lsn, entry)) = iter.next_back() {
+            *entry
+        } else {
+            0
+        }
+    }
 }

 impl Layer for InMemoryLayer {
+    // An in-memory layer doesn't really have a filename as it's not stored on disk,
+    // but we construct a filename as if it was a delta layer
+    fn filename(&self) -> PathBuf {
+        let inner = self.inner.lock().unwrap();
+
+        let end_lsn;
+        let dropped;
+        if let Some(drop_lsn) = inner.drop_lsn {
+            end_lsn = drop_lsn;
+            dropped = true;
+        } else {
+            end_lsn = Lsn(u64::MAX);
+            dropped = false;
+        }
+
+        let delta_filename = DeltaFileName {
+            seg: self.seg,
+            start_lsn: self.start_lsn,
+            end_lsn,
+            dropped,
+        }
+        .to_string();
+
+        PathBuf::from(format!("inmem-{}", delta_filename))
+    }
+
+    fn path(&self) -> Option<PathBuf> {
+        None
+    }
+
    fn get_timeline_id(&self) -> ZTimelineId {
-        return self.timelineid;
+        self.timelineid
    }

    fn get_seg_tag(&self) -> SegmentTag {
-        return self.seg;
+        self.seg
    }

    fn get_start_lsn(&self) -> Lsn {
-        return self.start_lsn;
+        self.start_lsn
    }

    fn get_end_lsn(&self) -> Lsn {
+        if let Some(end_lsn) = self.end_lsn {
+            return Lsn(end_lsn.0 + 1);
+        }
+
        let inner = self.inner.lock().unwrap();

        if let Some(drop_lsn) = inner.drop_lsn {
@@ -87,32 +157,33 @@ impl Layer for InMemoryLayer {
        blknum: u32,
        lsn: Lsn,
        reconstruct_data: &mut PageReconstructData,
-    ) -> Result<Option<Lsn>> {
-        // Scan the BTreeMap backwards, starting from reconstruct_data.lsn.
-        let mut need_base_image_lsn: Option<Lsn> = Some(lsn);
+    ) -> Result<PageReconstructResult> {
+        let mut need_image = true;

        assert!(self.seg.blknum_in_seg(blknum));

+        let predecessor: Option<Arc<dyn Layer>>;
+
        {
            let inner = self.inner.lock().unwrap();
+
+            // Scan the BTreeMap backwards, starting from reconstruct_data.lsn.
            let minkey = (blknum, Lsn(0));
            let maxkey = (blknum, lsn);
            let mut iter = inner
                .page_versions
                .range((Included(&minkey), Included(&maxkey)));
-            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
+            while let Some(((_blknum, _entry_lsn), entry)) = iter.next_back() {
                if let Some(img) = &entry.page_image {
                    reconstruct_data.page_img = Some(img.clone());
-                    need_base_image_lsn = None;
+                    need_image = false;
                    break;
                } else if let Some(rec) = &entry.record {
                    reconstruct_data.records.push(rec.clone());
                    if rec.will_init {
                        // This WAL record initializes the page, so no need to go further back
-                        need_base_image_lsn = None;
+                        need_image = false;
                        break;
-                    } else {
-                        need_base_image_lsn = Some(*entry_lsn);
                    }
                } else {
                    // No base image, and no WAL record. Huh?
@@ -120,32 +191,40 @@ impl Layer for InMemoryLayer {
                }
            }

-            // release lock on 'page_versions'
+            predecessor = inner.predecessor.clone();
+            // release lock on 'inner'
        }

-        Ok(need_base_image_lsn)
+        // If an older page image is needed to reconstruct the page, let the
+        // caller know about the predecessor layer.
+        if need_image {
+            if let Some(cont_layer) = predecessor {
+                Ok(PageReconstructResult::Continue(self.start_lsn, cont_layer))
+            } else {
+                Ok(PageReconstructResult::Missing(self.start_lsn))
+            }
+        } else {
+            Ok(PageReconstructResult::Complete)
+        }
    }

    /// Get size of the relation at given LSN
    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let inner = self.inner.lock().unwrap();
-        let mut iter = inner.segsizes.range((Included(&Lsn(0)), Included(&lsn)));
+        assert!(lsn >= self.start_lsn);

-        if let Some((_entry_lsn, entry)) = iter.next_back() {
-            let result = *entry;
-            drop(inner);
-            trace!("get_seg_size: {} at {} -> {}", self.seg, lsn, result);
-            Ok(result)
-        } else {
-            bail!("No size found for {} at {} in memory", self.seg, lsn);
-        }
+        let inner = self.inner.lock().unwrap();
+        Ok(inner.get_seg_size(lsn))
    }

    /// Does this segment exist at given LSN?
    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
        let inner = self.inner.lock().unwrap();

+        // If the segment created after requested LSN,
+        // it doesn't exist in the layer. But we shouldn't
+        // have requested it in the first place.
+        assert!(lsn >= self.start_lsn);
+
        // Is the requested LSN after the segment was dropped?
        if let Some(drop_lsn) = inner.drop_lsn {
            if lsn >= drop_lsn {
@@ -156,9 +235,83 @@ impl Layer for InMemoryLayer {
        // Otherwise, it exists
        Ok(true)
    }
+
+    /// Cannot unload anything in an in-memory layer, since there's no backing
+    /// store. To release memory used by an in-memory layer, use 'freeze' to turn
+    /// it into an on-disk layer.
+    fn unload(&self) -> Result<()> {
+        Ok(())
+    }
+
+    /// Nothing to do here. When you drop the last reference to the layer, it will
+    /// be deallocated.
+    fn delete(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn is_incremental(&self) -> bool {
+        let inner = self.inner.lock().unwrap();
+        inner.predecessor.is_some()
+    }
+
+    /// debugging function to print out the contents of the layer
+    fn dump(&self) -> Result<()> {
+        let inner = self.inner.lock().unwrap();
+
+        let end_str = inner
+            .drop_lsn
+            .as_ref()
+            .map(|drop_lsn| drop_lsn.to_string())
+            .unwrap_or_default();
+
+        println!(
+            "----- in-memory layer for tli {} seg {} {}-{} ----",
+            self.timelineid, self.seg, self.start_lsn, end_str
+        );
+
+        for (k, v) in inner.segsizes.iter() {
+            println!("segsizes {}: {}", k, v);
+        }
+
+        for (k, v) in inner.page_versions.iter() {
+            println!(
+                "blk {} at {}: {}/{}\n",
+                k.0,
+                k.1,
+                v.page_image.is_some(),
+                v.record.is_some()
+            );
+        }
+
+        Ok(())
+    }
+}
+
+/// Write failed because the layer is in process of being replaced.
+/// See [`LayeredTimeline::perform_write_op`] for how to handle this error.
+#[derive(Debug)]
+pub struct NonWriteableError;
+
+pub type WriteResult<T> = std::result::Result<T, NonWriteableError>;
+
+/// Helper struct to cleanup `InMemoryLayer::freeze` return signature.
+pub struct FreezeLayers {
+    /// Replacement layer for the layer which freeze was called on.
+    pub frozen: Arc<InMemoryLayer>,
+    /// New open layer containing leftover data.
+    pub open: Option<Arc<InMemoryLayer>>,
 }

 impl InMemoryLayer {
+    fn assert_not_frozen(&self) {
+        assert!(self.end_lsn.is_none());
+    }
+
+    /// Return the oldest page version that's stored in this layer
+    pub fn get_oldest_pending_lsn(&self) -> Lsn {
+        self.oldest_pending_lsn
+    }
+
    ///
    /// Create a new, empty, in-memory layer
    ///
@@ -168,6 +321,7 @@ impl InMemoryLayer {
        tenantid: ZTenantId,
        seg: SegmentTag,
        start_lsn: Lsn,
+        oldest_pending_lsn: Lsn,
    ) -> Result<InMemoryLayer> {
        trace!(
            "initializing new empty InMemoryLayer for writing {} on timeline {} at {}",
@@ -182,10 +336,14 @@ impl InMemoryLayer {
            tenantid,
            seg,
            start_lsn,
+            end_lsn: None,
+            oldest_pending_lsn,
            inner: Mutex::new(InMemoryLayerInner {
                drop_lsn: None,
                page_versions: BTreeMap::new(),
                segsizes: BTreeMap::new(),
+                writeable: true,
+                predecessor: None,
            }),
        })
    }
@@ -193,7 +351,7 @@ impl InMemoryLayer {
    // Write operations

    /// Remember new page version, as a WAL record over previous version
-    pub fn put_wal_record(&self, blknum: u32, rec: WALRecord) -> Result<()> {
+    pub fn put_wal_record(&self, blknum: u32, rec: WALRecord) -> WriteResult<u32> {
        self.put_page_version(
            blknum,
            rec.lsn,
@@ -205,7 +363,7 @@ impl InMemoryLayer {
    }

    /// Remember new page version, as a full page image
-    pub fn put_page_image(&self, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()> {
+    pub fn put_page_image(&self, blknum: u32, lsn: Lsn, img: Bytes) -> WriteResult<u32> {
        self.put_page_version(
            blknum,
            lsn,
@@ -218,7 +376,8 @@ impl InMemoryLayer {

    /// Common subroutine of the public put_wal_record() and put_page_image() functions.
    /// Adds the page version to the in-memory tree
-    pub fn put_page_version(&self, blknum: u32, lsn: Lsn, pv: PageVersion) -> Result<()> {
+    pub fn put_page_version(&self, blknum: u32, lsn: Lsn, pv: PageVersion) -> WriteResult<u32> {
+        self.assert_not_frozen();
        assert!(self.seg.blknum_in_seg(blknum));

        trace!(
@@ -230,6 +389,8 @@ impl InMemoryLayer {
        );
        let mut inner = self.inner.lock().unwrap();

+        inner.check_writeable()?;
+
        let old = inner.page_versions.insert((blknum, lsn), pv);

        if old.is_some() {
@@ -244,15 +405,9 @@ impl InMemoryLayer {
        if self.seg.rel.is_blocky() {
            let newsize = blknum - self.seg.segno * RELISH_SEG_SIZE + 1;

-            let mut iter = inner.segsizes.range((Included(&Lsn(0)), Included(&lsn)));
-
-            let oldsize;
-            if let Some((_entry_lsn, entry)) = iter.next_back() {
-                oldsize = *entry;
-            } else {
-                oldsize = 0;
-                //bail!("No old size found for {} at {}", self.tag, lsn);
-            }
+            // use inner get_seg_size, since calling self.get_seg_size will try to acquire self.inner.lock
+            // which we've just acquired above
+            let oldsize = inner.get_seg_size(lsn);
            if newsize > oldsize {
                trace!(
                    "enlarging segment {} from {} to {} blocks at {}",
@@ -261,16 +416,55 @@ impl InMemoryLayer {
                    newsize,
                    lsn
                );
+
+                // If we are extending the relation by more than one page, initialize the "gap"
+                // with zeros
+                //
+                // XXX: What if the caller initializes the gap with subsequent call with same LSN?
+                // I don't think that can happen currently, but that is highly dependent on how
+                // PostgreSQL writes its WAL records and there's no guarantee of it. If it does
+                // happen, we would hit the "page version already exists" warning above on the
+                // subsequent call to initialize the gap page.
+                let gapstart = self.seg.segno * RELISH_SEG_SIZE + oldsize;
+                for gapblknum in gapstart..blknum {
+                    let zeropv = PageVersion {
+                        page_image: Some(ZERO_PAGE.clone()),
+                        record: None,
+                    };
+                    trace!(
+                        "filling gap blk {} with zeros for write of {}",
+                        gapblknum,
+                        blknum
+                    );
+                    let old = inner.page_versions.insert((gapblknum, lsn), zeropv);
+                    // We already had an entry for this LSN. That's odd..
+
+                    if old.is_some() {
+                        warn!(
+                            "Page version of rel {} blk {} at {} already exists",
+                            self.seg.rel, blknum, lsn
+                        );
+                    }
+                }
+
                inner.segsizes.insert(lsn, newsize);
+                return Ok(newsize - oldsize);
            }
        }
-
-        Ok(())
+        Ok(0)
    }

    /// Remember that the relation was truncated at given LSN
-    pub fn put_truncation(&self, lsn: Lsn, segsize: u32) -> anyhow::Result<()> {
+    pub fn put_truncation(&self, lsn: Lsn, segsize: u32) -> WriteResult<()> {
+        self.assert_not_frozen();
+
        let mut inner = self.inner.lock().unwrap();
+        inner.check_writeable()?;
+
+        // check that this we truncate to a smaller size than segment was before the truncation
+        let oldsize = inner.get_seg_size(lsn);
+        assert!(segsize < oldsize);
+
        let old = inner.segsizes.insert(lsn, segsize);

        if old.is_some() {
@@ -282,9 +476,13 @@ impl InMemoryLayer {
    }

    /// Remember that the segment was dropped at given LSN
-    pub fn put_unlink(&self, lsn: Lsn) -> anyhow::Result<()> {
+    pub fn drop_segment(&self, lsn: Lsn) -> WriteResult<()> {
+        self.assert_not_frozen();
+
        let mut inner = self.inner.lock().unwrap();

+        inner.check_writeable()?;
+
        assert!(inner.drop_lsn.is_none());
        inner.drop_lsn = Some(lsn);

@@ -297,82 +495,63 @@ impl InMemoryLayer {
    /// Initialize a new InMemoryLayer for, by copying the state at the given
    /// point in time from given existing layer.
    ///
-    pub fn copy_snapshot(
+    pub fn create_successor_layer(
        conf: &'static PageServerConf,
-        timeline: &LayeredTimeline,
-        src: &dyn Layer,
+        src: Arc<dyn Layer>,
        timelineid: ZTimelineId,
        tenantid: ZTenantId,
-        lsn: Lsn,
+        start_lsn: Lsn,
+        oldest_pending_lsn: Lsn,
    ) -> Result<InMemoryLayer> {
-        trace!(
-            "initializing new InMemoryLayer for writing {} on timeline {} at {}",
-            src.get_seg_tag(),
-            timelineid,
-            lsn
-        );
-        let mut page_versions = BTreeMap::new();
-        let mut segsizes = BTreeMap::new();
-
        let seg = src.get_seg_tag();

-        let startblk;
-        let size;
-        if seg.rel.is_blocky() {
-            size = src.get_seg_size(lsn)?;
-            segsizes.insert(lsn, size);
-            startblk = seg.segno * RELISH_SEG_SIZE;
-        } else {
-            size = 1;
-            startblk = 0;
-        }
+        trace!(
+            "initializing new InMemoryLayer for writing {} on timeline {} at {}",
+            seg,
+            timelineid,
+            start_lsn,
+        );

-        for blknum in startblk..(startblk + size) {
-            let img = timeline.materialize_page(seg, blknum, lsn, src)?;
-            let pv = PageVersion {
-                page_image: Some(img),
-                record: None,
-            };
-            page_versions.insert((blknum, lsn), pv);
+        // For convenience, copy the segment size from the predecessor layer
+        let mut segsizes = BTreeMap::new();
+        if seg.rel.is_blocky() {
+            let size = src.get_seg_size(start_lsn)?;
+            segsizes.insert(start_lsn, size);
        }

        Ok(InMemoryLayer {
            conf,
            timelineid,
            tenantid,
-            seg: src.get_seg_tag(),
-            start_lsn: lsn,
+            seg,
+            start_lsn,
+            end_lsn: None,
+            oldest_pending_lsn,
            inner: Mutex::new(InMemoryLayerInner {
                drop_lsn: None,
-                page_versions: page_versions,
-                segsizes: segsizes,
+                page_versions: BTreeMap::new(),
+                segsizes,
+                writeable: true,
+                predecessor: Some(src),
            }),
        })
    }

-    ///
-    /// Write the this in-memory layer to disk, as a snapshot layer.
-    ///
-    /// The cutoff point for the layer that's written to disk is 'end_lsn'.
-    ///
-    /// Returns new layers that replace this one. Always returns a
-    /// SnapshotLayer containing the page versions that were written to disk,
-    /// but if there were page versions newer than 'end_lsn', also return a new
-    /// in-memory layer containing those page versions. The caller replaces
-    /// this layer with the returned layers in the layer map.
-    ///
-    pub fn freeze(
-        &self,
-        cutoff_lsn: Lsn,
-        // This is needed just to call materialize_page()
-        timeline: &LayeredTimeline,
-    ) -> Result<(Option<Arc<SnapshotLayer>>, Option<Arc<InMemoryLayer>>)> {
+    /// Splits `self` into two InMemoryLayers: `frozen` and `open`.
+    /// All data up to and including `cutoff_lsn` (or the drop LSN, if dropped)
+    /// is copied to `frozen`, while the remaining data is copied to `open`.
+    /// After completion, self is non-writeable, but not frozen.
+    pub fn freeze(&self, cutoff_lsn: Lsn) -> Result<FreezeLayers> {
        info!(
            "freezing in memory layer for {} on timeline {} at {}",
            self.seg, self.timelineid, cutoff_lsn
        );

-        let inner = self.inner.lock().unwrap();
+        self.assert_not_frozen();
+
+        let mut inner = self.inner.lock().unwrap();
+        assert!(inner.writeable);
+        inner.writeable = false;

        // Normally, use the cutoff LSN as the end of the frozen layer.
        // But if the relation was dropped, we know that there are no
@@ -419,73 +598,124 @@ impl InMemoryLayer {
            after_page_versions = BTreeMap::new();
        }

-        // we can release the lock now.
-        drop(inner);
+        let frozen = Arc::new(InMemoryLayer {
+            conf: self.conf,
+            tenantid: self.tenantid,
+            timelineid: self.timelineid,
+            seg: self.seg,
+            start_lsn: self.start_lsn,
+            end_lsn: Some(end_lsn),
+            oldest_pending_lsn: self.start_lsn,
+            inner: Mutex::new(InMemoryLayerInner {
+                drop_lsn: inner.drop_lsn,
+                page_versions: before_page_versions,
+                segsizes: before_segsizes,
+                writeable: false,
+                predecessor: inner.predecessor.clone(),
+            }),
+        });

-        // Write the page versions before the cutoff to disk.
-        let snapfile = SnapshotLayer::create(
-            self.conf,
-            self.timelineid,
-            self.tenantid,
-            self.seg,
-            self.start_lsn,
-            end_lsn,
-            dropped,
-            before_page_versions,
-            before_segsizes,
-        )?;
-
-        // If there were any "new" page versions, initialize a new in-memory layer to hold
-        // them
-        let new_open = if !after_segsizes.is_empty() || !after_page_versions.is_empty() {
-            info!("created new in-mem layer for {} {}-", self.seg, end_lsn);
-
-            let new_open = Self::copy_snapshot(
+        let open = if !dropped && (!after_segsizes.is_empty() || !after_page_versions.is_empty()) {
+            let mut new_open = Self::create_successor_layer(
                self.conf,
-                timeline,
-                &snapfile,
+                frozen.clone(),
                self.timelineid,
                self.tenantid,
                end_lsn,
+                end_lsn,
            )?;
-            let mut new_inner = new_open.inner.lock().unwrap();
+
+            let new_inner = new_open.inner.get_mut().unwrap();
            new_inner.page_versions.append(&mut after_page_versions);
            new_inner.segsizes.append(&mut after_segsizes);
-            drop(new_inner);

            Some(Arc::new(new_open))
        } else {
            None
        };

-        let new_historic = Some(Arc::new(snapfile));
-
-        Ok((new_historic, new_open))
+        // TODO could we avoid creating the `frozen` if it contains no data
+        Ok(FreezeLayers { frozen, open })
    }

-    /// debugging function to print out the contents of the layer
-    #[allow(unused)]
-    pub fn dump(&self) -> String {
-        let mut result = format!(
-            "----- inmemory layer for {} {}-> ----\n",
-            self.seg, self.start_lsn
-        );
+    /// Write the this frozen in-memory layer to disk.
+    ///
+    /// Returns new layers that replace this one.
+    /// If not dropped, returns a new image layer containing the page versions
+    /// at the `end_lsn`. Can also return a DeltaLayer that includes all the
+    /// WAL records between start and end LSN. (The delta layer is not needed
+    /// when a new relish is created with a single LSN, so that the start and
+    /// end LSN are the same.)
+    pub fn write_to_disk(&self, timeline: &LayeredTimeline) -> Result<Vec<Arc<dyn Layer>>> {
+        let end_lsn = self.end_lsn.expect("can only write frozen layers to disk");

        let inner = self.inner.lock().unwrap();

-        for (k, v) in inner.segsizes.iter() {
-            result += &format!("{}: {}\n", k, v);
-        }
-        for (k, v) in inner.page_versions.iter() {
-            result += &format!(
-                "blk {} at {}: {}/{}\n",
-                k.0,
-                k.1,
-                v.page_image.is_some(),
-                v.record.is_some()
-            );
+        let drop_lsn = inner.drop_lsn;
+        let predecessor = inner.predecessor.clone();
+
+        let mut before_page_versions;
+        let mut before_segsizes;
+        if inner.drop_lsn.is_none() {
+            before_segsizes = BTreeMap::new();
+            for (lsn, size) in inner.segsizes.iter() {
+                if *lsn <= end_lsn {
+                    before_segsizes.insert(*lsn, *size);
+                }
+            }
+
+            before_page_versions = BTreeMap::new();
+            for ((blknum, lsn), pv) in inner.page_versions.iter() {
+                if *lsn < end_lsn {
+                    before_page_versions.insert((*blknum, *lsn), pv.clone());
+                }
+            }
+        } else {
+            before_page_versions = inner.page_versions.clone();
+            before_segsizes = inner.segsizes.clone();
        }

-        result
+        drop(inner);
+
+        let mut frozen_layers: Vec<Arc<dyn Layer>> = Vec::new();
+
+        if self.start_lsn != end_lsn {
+            // Write the page versions before the cutoff to disk.
+            let delta_layer = DeltaLayer::create(
+                self.conf,
+                self.timelineid,
+                self.tenantid,
+                self.seg,
+                self.start_lsn,
+                end_lsn,
+                drop_lsn.is_some(),
+                predecessor,
+                before_page_versions,
+                before_segsizes,
+            )?;
+            frozen_layers.push(Arc::new(delta_layer));
+            trace!(
+                "freeze: created delta layer {} {}-{}",
+                self.seg,
+                self.start_lsn,
+                end_lsn
+            );
+        } else {
+            assert!(before_page_versions.is_empty());
+        }
+
+        if drop_lsn.is_none() {
+            // Write a new base image layer at the cutoff point
+            let image_layer = ImageLayer::create_from_src(self.conf, timeline, self, end_lsn)?;
+            frozen_layers.push(Arc::new(image_layer));
+            trace!("freeze: created image layer {} at {}", self.seg, end_lsn);
+        }
+
+        Ok(frozen_layers)
+    }
+
+    pub fn update_predecessor(&self, predecessor: Arc<dyn Layer>) -> Option<Arc<dyn Layer>> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.predecessor.replace(predecessor)
    }
 }
--- a/pageserver/src/layered_repository/layer_map.rs
+++ b/pageserver/src/layered_repository/layer_map.rs
@@ -1,67 +1,63 @@
 //!
-//! The layer map tracks what layers exist for all the relations in a timeline.
+//! The layer map tracks what layers exist for all the relishes in a timeline.
 //!
-//! When the timeline is first accessed, the server lists of all snapshot files
+//! When the timeline is first accessed, the server lists of all layer files
 //! in the timelines/<timelineid> directory, and populates this map with
-//! SnapshotLayers corresponding to each file. When new WAL is received,
-//! we create InMemoryLayers to hold the incoming records. Now and then,
-//! in the checkpoint() function, the in-memory layers are frozen, forming
-//! new snapshot layers and corresponding files are written to disk.
+//! ImageLayer and DeltaLayer structs corresponding to each file. When new WAL
+//! is received, we create InMemoryLayers to hold the incoming records. Now and
+//! then, in the checkpoint() function, the in-memory layers are frozen, forming
+//! new image and delta layers and corresponding files are written to disk.
 //!

 use crate::layered_repository::storage_layer::{Layer, SegmentTag};
-use crate::layered_repository::{InMemoryLayer, SnapshotLayer};
+use crate::layered_repository::InMemoryLayer;
 use crate::relish::*;
 use anyhow::Result;
-use log::*;
-use std::collections::HashSet;
-use std::collections::{BTreeMap, HashMap};
-use std::ops::Bound::Included;
+use lazy_static::lazy_static;
+use std::cmp::Ordering;
+use std::collections::{BTreeMap, BinaryHeap, HashMap};
 use std::sync::Arc;
+use zenith_metrics::{register_int_gauge, IntGauge};
 use zenith_utils::lsn::Lsn;

-///
-/// LayerMap tracks what layers exist or a timeline. The last layer that is
-/// open for writes is always an InMemoryLayer, and is tracked separately
-/// because there can be only one for each segment. The older layers,
-/// stored on disk, are kept in a BTreeMap keyed by the layer's start LSN.
-///
-pub struct LayerMap {
-    segs: HashMap<SegmentTag, SegEntry>,
+lazy_static! {
+    static ref NUM_INMEMORY_LAYERS: IntGauge =
+        register_int_gauge!("pageserver_inmemory_layers", "Number of layers in memory")
+            .expect("failed to define a metric");
+    static ref NUM_ONDISK_LAYERS: IntGauge =
+        register_int_gauge!("pageserver_ondisk_layers", "Number of layers on-disk")
+            .expect("failed to define a metric");
 }

-struct SegEntry {
-    pub open: Option<Arc<InMemoryLayer>>,
-    pub historic: BTreeMap<Lsn, Arc<SnapshotLayer>>,
+///
+/// LayerMap tracks what layers exist on a timeline.
+///
+#[derive(Default)]
+pub struct LayerMap {
+    /// All the layers keyed by segment tag
+    segs: HashMap<SegmentTag, SegEntry>,
+
+    /// All in-memory layers, ordered by 'oldest_pending_lsn' and generation
+    /// of each layer. This allows easy access to the in-memory layer that
+    /// contains the oldest WAL record.
+    open_layers: BinaryHeap<OpenLayerEntry>,
+
+    /// Generation number, used to distinguish newly inserted entries in the
+    /// binary heap from older entries during checkpoint.
+    current_generation: u64,
 }

 impl LayerMap {
    ///
-    /// Look up using the given segment tag and LSN. This differs from a plain
-    /// key-value lookup in that if there is any layer that covers the
+    /// Look up a layer using the given segment tag and LSN. This differs from a
+    /// plain key-value lookup in that if there is any layer that covers the
    /// given LSN, or precedes the given LSN, it is returned. In other words,
    /// you don't need to know the exact start LSN of the layer.
    ///
    pub fn get(&self, tag: &SegmentTag, lsn: Lsn) -> Option<Arc<dyn Layer>> {
        let segentry = self.segs.get(tag)?;

-        if let Some(open) = &segentry.open {
-            if open.get_start_lsn() <= lsn {
-                let x: Arc<dyn Layer> = Arc::clone(&open) as _;
-                return Some(x);
-            }
-        }
-
-        if let Some((_k, v)) = segentry
-            .historic
-            .range((Included(Lsn(0)), Included(lsn)))
-            .next_back()
-        {
-            let x: Arc<dyn Layer> = Arc::clone(&v) as _;
-            Some(x)
-        } else {
-            None
-        }
+        segentry.get(lsn)
    }

    ///
@@ -71,52 +67,53 @@ impl LayerMap {
    pub fn get_open(&self, tag: &SegmentTag) -> Option<Arc<InMemoryLayer>> {
        let segentry = self.segs.get(tag)?;

-        if let Some(open) = &segentry.open {
-            Some(Arc::clone(open))
-        } else {
-            None
-        }
+        segentry.open.as_ref().map(Arc::clone)
    }

    ///
    /// Insert an open in-memory layer
    ///
    pub fn insert_open(&mut self, layer: Arc<InMemoryLayer>) {
-        let tag = layer.get_seg_tag();
+        let segentry = self.segs.entry(layer.get_seg_tag()).or_default();

-        if let Some(segentry) = self.segs.get_mut(&tag) {
-            if let Some(_old) = &segentry.open {
-                // FIXME: shouldn't exist, but check
-            }
-            segentry.open = Some(layer);
-        } else {
-            let segentry = SegEntry {
-                open: Some(layer),
-                historic: BTreeMap::new(),
-            };
-            self.segs.insert(tag, segentry);
-        }
+        segentry.insert_open(Arc::clone(&layer));
+
+        // Also add it to the binary heap
+        let open_layer_entry = OpenLayerEntry {
+            oldest_pending_lsn: layer.get_oldest_pending_lsn(),
+            layer,
+            generation: self.current_generation,
+        };
+        self.open_layers.push(open_layer_entry);
+
+        NUM_INMEMORY_LAYERS.inc();
+    }
+
+    /// Remove the oldest in-memory layer
+    pub fn pop_oldest_open(&mut self) {
+        // Pop it from the binary heap
+        let oldest_entry = self.open_layers.pop().unwrap();
+        let segtag = oldest_entry.layer.get_seg_tag();
+
+        // Also remove it from the SegEntry of this segment
+        let mut segentry = self.segs.get_mut(&segtag).unwrap();
+        assert!(Arc::ptr_eq(
+            segentry.open.as_ref().unwrap(),
+            &oldest_entry.layer
+        ));
+        segentry.open = None;
+
+        NUM_INMEMORY_LAYERS.dec();
    }

    ///
    /// Insert an on-disk layer
    ///
-    pub fn insert_historic(&mut self, layer: Arc<SnapshotLayer>) {
-        let tag = layer.get_seg_tag();
-        let start_lsn = layer.get_start_lsn();
+    pub fn insert_historic(&mut self, layer: Arc<dyn Layer>) {
+        let segentry = self.segs.entry(layer.get_seg_tag()).or_default();
+        segentry.insert_historic(layer);

-        if let Some(segentry) = self.segs.get_mut(&tag) {
-            segentry.historic.insert(start_lsn, layer);
-        } else {
-            let mut historic = BTreeMap::new();
-            historic.insert(start_lsn, layer);
-
-            let segentry = SegEntry {
-                open: None,
-                historic,
-            };
-            self.segs.insert(tag, segentry);
-        }
+        NUM_ONDISK_LAYERS.inc();
    }

    ///
@@ -124,83 +121,72 @@ impl LayerMap {
    ///
    /// This should be called when the corresponding file on disk has been deleted.
    ///
-    pub fn remove_historic(&mut self, layer: &SnapshotLayer) {
+    pub fn remove_historic(&mut self, layer: &dyn Layer) {
        let tag = layer.get_seg_tag();
        let start_lsn = layer.get_start_lsn();

        if let Some(segentry) = self.segs.get_mut(&tag) {
            segentry.historic.remove(&start_lsn);
        }
+        NUM_ONDISK_LAYERS.dec();
    }

-    pub fn list_rels(&self, spcnode: u32, dbnode: u32) -> Result<HashSet<RelTag>> {
-        let mut rels: HashSet<RelTag> = HashSet::new();
+    // List relations along with a flag that marks if they exist at the given lsn.
+    // spcnode 0 and dbnode 0 have special meanings and mean all tabespaces/databases.
+    // Pass Tag if we're only interested in some relations.
+    pub fn list_relishes(&self, tag: Option<RelTag>, lsn: Lsn) -> Result<HashMap<RelishTag, bool>> {
+        let mut rels: HashMap<RelishTag, bool> = HashMap::new();

-        for (seg, _entry) in self.segs.iter() {
-            if let RelishTag::Relation(reltag) = seg.rel {
-                // FIXME: skip if it was dropped before the requested LSN. But there is no
-                // LSN argument
-
-                if (spcnode == 0 || reltag.spcnode == spcnode)
-                    && (dbnode == 0 || reltag.dbnode == dbnode)
-                {
-                    rels.insert(reltag);
+        for (seg, segentry) in self.segs.iter() {
+            match seg.rel {
+                RelishTag::Relation(reltag) => {
+                    if let Some(request_rel) = tag {
+                        if (request_rel.spcnode == 0 || reltag.spcnode == request_rel.spcnode)
+                            && (request_rel.dbnode == 0 || reltag.dbnode == request_rel.dbnode)
+                        {
+                            if let Some(exists) = segentry.exists_at_lsn(lsn) {
+                                rels.insert(seg.rel, exists);
+                            }
+                        }
+                    }
+                }
+                _ => {
+                    if tag == None {
+                        if let Some(exists) = segentry.exists_at_lsn(lsn) {
+                            rels.insert(seg.rel, exists);
+                        }
+                    }
                }
            }
        }
        Ok(rels)
    }

-    pub fn list_nonrels(&self, _lsn: Lsn) -> Result<HashSet<RelishTag>> {
-        let mut rels: HashSet<RelishTag> = HashSet::new();
-
-        // Scan the timeline directory to get all rels in this timeline.
-        for (seg, _entry) in self.segs.iter() {
-            // FIXME: skip if it was dropped before the requested LSN.
-
-            if let RelishTag::Relation(_) = seg.rel {
-            } else {
-                rels.insert(seg.rel);
-            }
-        }
-        Ok(rels)
-    }
-
-    /// Is there a newer layer for given segment?
-    pub fn newer_layer_exists(&self, seg: SegmentTag, lsn: Lsn) -> bool {
+    /// Is there a newer image layer for given segment?
+    ///
+    /// This is used for garbage collection, to determine if an old layer can
+    /// be deleted.
+    pub fn newer_image_layer_exists(&self, seg: SegmentTag, lsn: Lsn) -> bool {
        if let Some(segentry) = self.segs.get(&seg) {
-            if let Some(_open) = &segentry.open {
-                return true;
-            }
-
-            for (newer_lsn, layer) in segentry
-                .historic
-                .range((Included(lsn), Included(Lsn(u64::MAX))))
-            {
-                if layer.get_end_lsn() > lsn {
-                    trace!(
-                        "found later layer for {}, {} {}-{}",
-                        seg,
-                        lsn,
-                        newer_lsn,
-                        layer.get_end_lsn()
-                    );
-                    return true;
-                } else {
-                    trace!("found singleton layer for {}, {} {}", seg, lsn, newer_lsn);
-                    continue;
-                }
-            }
+            segentry.newer_image_layer_exists(lsn)
+        } else {
+            false
        }
-        trace!("no later layer found for {}, {}", seg, lsn);
-        false
    }

-    pub fn iter_open_layers(&mut self) -> OpenLayerIter {
-        OpenLayerIter {
-            last: None,
-            segiter: self.segs.iter_mut(),
-        }
+    /// Return the oldest in-memory layer, along with its generation number.
+    pub fn peek_oldest_open(&self) -> Option<(Arc<InMemoryLayer>, u64)> {
+        self.open_layers
+            .peek()
+            .map(|oldest_entry| (Arc::clone(&oldest_entry.layer), oldest_entry.generation))
+    }
+
+    /// Increment the generation number used to stamp open in-memory layers. Layers
+    /// added with `insert_open` after this call will be associated with the new
+    /// generation. Returns the new generation number.
+    pub fn increment_generation(&mut self) -> u64 {
+        self.current_generation += 1;
+        self.current_generation
    }

    pub fn iter_historic_layers(&self) -> HistoricLayerIter {
@@ -209,59 +195,141 @@ impl LayerMap {
            iter: None,
        }
    }
-}

-impl Default for LayerMap {
-    fn default() -> Self {
-        LayerMap {
-            segs: HashMap::new(),
-        }
-    }
-}
+    /// debugging function to print out the contents of the layer map
+    #[allow(unused)]
+    pub fn dump(&self) -> Result<()> {
+        println!("Begin dump LayerMap");
+        for (seg, segentry) in self.segs.iter() {
+            if let Some(open) = &segentry.open {
+                open.dump()?;
+            }

-pub struct OpenLayerIter<'a> {
-    last: Option<&'a mut SegEntry>,
-
-    segiter: std::collections::hash_map::IterMut<'a, SegmentTag, SegEntry>,
-}
-
-impl<'a> OpenLayerIter<'a> {
-    pub fn replace(&mut self, replacement: Option<Arc<InMemoryLayer>>) {
-        let segentry = self.last.as_mut().unwrap();
-        segentry.open = replacement;
-    }
-
-    pub fn insert_historic(&mut self, new_layer: Arc<SnapshotLayer>) {
-        let start_lsn = new_layer.get_start_lsn();
-
-        let segentry = self.last.as_mut().unwrap();
-        segentry.historic.insert(start_lsn, new_layer);
-    }
-}
-
-impl<'a> Iterator for OpenLayerIter<'a> {
-    type Item = Arc<InMemoryLayer>;
-
-    fn next(&mut self) -> std::option::Option<<Self as std::iter::Iterator>::Item> {
-        while let Some((_seg, entry)) = self.segiter.next() {
-            if let Some(open) = &entry.open {
-                let op = Arc::clone(&open);
-                self.last = Some(entry);
-                return Some(op);
+            for (_, layer) in segentry.historic.iter() {
+                layer.dump()?;
            }
        }
-        self.last = None;
-        None
+        println!("End dump LayerMap");
+        Ok(())
    }
 }

+///
+/// Per-segment entry in the LayerMap::segs hash map. Holds all the layers
+/// associated with the segment.
+///
+/// The last layer that is open for writes is always an InMemoryLayer,
+/// and is kept in a separate field, because there can be only one for
+/// each segment. The older layers, stored on disk, are kept in a
+/// BTreeMap keyed by the layer's start LSN.
+#[derive(Default)]
+struct SegEntry {
+    pub open: Option<Arc<InMemoryLayer>>,
+    pub historic: BTreeMap<Lsn, Arc<dyn Layer>>,
+}
+
+impl SegEntry {
+    /// Does the segment exist at given LSN?
+    /// Return None if object is not found in this SegEntry.
+    fn exists_at_lsn(&self, lsn: Lsn) -> Option<bool> {
+        if let Some(layer) = &self.open {
+            if layer.get_start_lsn() <= lsn && lsn <= layer.get_end_lsn() {
+                let exists = layer.get_seg_exists(lsn).ok()?;
+                return Some(exists);
+            }
+        } else if let Some((_, layer)) = self.historic.range(..=lsn).next_back() {
+            let exists = layer.get_seg_exists(lsn).ok()?;
+            return Some(exists);
+        }
+        None
+    }
+
+    pub fn get(&self, lsn: Lsn) -> Option<Arc<dyn Layer>> {
+        if let Some(open) = &self.open {
+            if open.get_start_lsn() <= lsn {
+                let x: Arc<dyn Layer> = Arc::clone(open) as _;
+                return Some(x);
+            }
+        }
+
+        if let Some((_start_lsn, layer)) = self.historic.range(..=lsn).next_back() {
+            Some(Arc::clone(layer))
+        } else {
+            None
+        }
+    }
+
+    pub fn newer_image_layer_exists(&self, lsn: Lsn) -> bool {
+        // We only check on-disk layers, because
+        // in-memory layers are not durable
+
+        for (_newer_lsn, layer) in self.historic.range(lsn..) {
+            // Ignore incremental layers.
+            if layer.is_incremental() {
+                continue;
+            }
+            if layer.get_end_lsn() > lsn {
+                return true;
+            } else {
+                continue;
+            }
+        }
+        false
+    }
+
+    pub fn insert_open(&mut self, layer: Arc<InMemoryLayer>) {
+        assert!(self.open.is_none());
+        self.open = Some(layer);
+    }
+
+    pub fn insert_historic(&mut self, layer: Arc<dyn Layer>) {
+        let start_lsn = layer.get_start_lsn();
+
+        self.historic.insert(start_lsn, layer);
+    }
+}
+
+/// Entry held in LayerMap::open_layers, with boilerplate comparison routines
+/// to implement a min-heap ordered by 'oldest_pending_lsn' and 'generation'
+///
+/// The generation number associated with each entry can be used to distinguish
+/// recently-added entries (i.e after last call to increment_generation()) from older
+/// entries with the same 'oldest_pending_lsn'.
+struct OpenLayerEntry {
+    pub oldest_pending_lsn: Lsn, // copy of layer.get_oldest_pending_lsn()
+    pub generation: u64,
+    pub layer: Arc<InMemoryLayer>,
+}
+impl Ord for OpenLayerEntry {
+    fn cmp(&self, other: &Self) -> Ordering {
+        // BinaryHeap is a max-heap, and we want a min-heap. Reverse the ordering here
+        // to get that. Entries with identical oldest_pending_lsn are ordered by generation
+        other
+            .oldest_pending_lsn
+            .cmp(&self.oldest_pending_lsn)
+            .then_with(|| other.generation.cmp(&self.generation))
+    }
+}
+impl PartialOrd for OpenLayerEntry {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+impl PartialEq for OpenLayerEntry {
+    fn eq(&self, other: &Self) -> bool {
+        self.cmp(other) == Ordering::Equal
+    }
+}
+impl Eq for OpenLayerEntry {}
+
+/// Iterator returned by LayerMap::iter_historic_layers()
 pub struct HistoricLayerIter<'a> {
    segiter: std::collections::hash_map::Iter<'a, SegmentTag, SegEntry>,
-    iter: Option<std::collections::btree_map::Iter<'a, Lsn, Arc<SnapshotLayer>>>,
+    iter: Option<std::collections::btree_map::Iter<'a, Lsn, Arc<dyn Layer>>>,
 }

 impl<'a> Iterator for HistoricLayerIter<'a> {
-    type Item = Arc<SnapshotLayer>;
+    type Item = Arc<dyn Layer>;

    fn next(&mut self) -> std::option::Option<<Self as std::iter::Iterator>::Item> {
        loop {
@@ -279,3 +347,78 @@ impl<'a> Iterator for HistoricLayerIter<'a> {
        }
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::PageServerConf;
+    use std::str::FromStr;
+    use zenith_utils::zid::{ZTenantId, ZTimelineId};
+
+    /// Arbitrary relation tag, for testing.
+    const TESTREL_A: RelishTag = RelishTag::Relation(RelTag {
+        spcnode: 0,
+        dbnode: 111,
+        relnode: 1000,
+        forknum: 0,
+    });
+
+    /// Construct a dummy InMemoryLayer for testing
+    fn dummy_inmem_layer(
+        conf: &'static PageServerConf,
+        segno: u32,
+        start_lsn: Lsn,
+        oldest_pending_lsn: Lsn,
+    ) -> Arc<InMemoryLayer> {
+        Arc::new(
+            InMemoryLayer::create(
+                conf,
+                ZTimelineId::from_str("00000000000000000000000000000000").unwrap(),
+                ZTenantId::from_str("00000000000000000000000000000000").unwrap(),
+                SegmentTag {
+                    rel: TESTREL_A,
+                    segno,
+                },
+                start_lsn,
+                oldest_pending_lsn,
+            )
+            .unwrap(),
+        )
+    }
+
+    #[test]
+    fn test_open_layers() -> Result<()> {
+        let conf = PageServerConf::dummy_conf(PageServerConf::test_repo_dir("dummy_inmem_layer"));
+        let conf = Box::leak(Box::new(conf));
+
+        let mut layers = LayerMap::default();
+
+        let gen1 = layers.increment_generation();
+        layers.insert_open(dummy_inmem_layer(conf, 0, Lsn(100), Lsn(100)));
+        layers.insert_open(dummy_inmem_layer(conf, 1, Lsn(100), Lsn(200)));
+        layers.insert_open(dummy_inmem_layer(conf, 2, Lsn(100), Lsn(120)));
+        layers.insert_open(dummy_inmem_layer(conf, 3, Lsn(100), Lsn(110)));
+
+        let gen2 = layers.increment_generation();
+        layers.insert_open(dummy_inmem_layer(conf, 4, Lsn(100), Lsn(110)));
+        layers.insert_open(dummy_inmem_layer(conf, 5, Lsn(100), Lsn(100)));
+
+        // A helper function (closure) to pop the next oldest open entry from the layer map,
+        // and assert that it is what we'd expect
+        let mut assert_pop_layer = |expected_segno: u32, expected_generation: u64| {
+            let (l, generation) = layers.peek_oldest_open().unwrap();
+            assert!(l.get_seg_tag().segno == expected_segno);
+            assert!(generation == expected_generation);
+            layers.pop_oldest_open();
+        };
+
+        assert_pop_layer(0, gen1); // 100
+        assert_pop_layer(5, gen2); // 100
+        assert_pop_layer(3, gen1); // 110
+        assert_pop_layer(4, gen2); // 110
+        assert_pop_layer(2, gen1); // 120
+        assert_pop_layer(1, gen1); // 200
+
+        Ok(())
+    }
+}
--- a/pageserver/src/layered_repository/snapshot_layer.rs
+++ b/pageserver/src/layered_repository/snapshot_layer.rs
@@ -1,547 +0,0 @@
-//!
-//! A SnapshotLayer represents one snapshot file on disk. One file holds all page
-//! version and size information of one relation, in a range of LSN.
-//! The name "snapshot file" is a bit of a misnomer because a snapshot file doesn't
-//! contain a snapshot at a specific LSN, but rather all the page versions in a range
-//! of LSNs.
-//!
-//! Currently, a snapshot file contains full information needed to reconstruct any
-//! page version in the LSN range, without consulting any other snapshot files. When
-//! a new snapshot file is created for writing, the full contents of relation are
-//! materialized as it is at the beginning of the LSN range. That can be very expensive,
-//! we should find a way to store differential files. But this keeps the read-side
-//! of things simple. You can find the correct snapshot file based on RelishTag and
-//! timeline+LSN, and once you've located it, you have all the data you need to in that
-//! file.
-//!
-//! When a snapshot file needs to be accessed, we slurp the whole file into memory, into
-//! the SnapshotLayer struct. See load() and unload() functions.
-//!
-//! On disk, the snapshot files are stored in timelines/<timelineid> directory.
-//! Currently, there are no subdirectories, and each snapshot file is named like this:
-//!
-//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<start LSN>_<end LSN>
-//!
-//! For example:
-//!
-//!    1663_13990_2609_0_000000000169C348_000000000169C349
-//!
-//! If a relation is dropped, we add a '_DROPPED' to the end of the filename to indicate that.
-//! So the above example would become:
-//!
-//!    1663_13990_2609_0_000000000169C348_000000000169C349_DROPPED
-//!
-//! The end LSN indicates when it was dropped in that case, we don't store it in the
-//! file contents in any way.
-//!
-//! A snapshot file is constructed using the 'bookfile' crate. Each file consists of two
-//! parts: the page versions and the relation sizes. They are stored as separate chapters.
-//!
-use crate::layered_repository::storage_layer::{
-    Layer, PageReconstructData, PageVersion, SegmentTag,
-};
-use crate::relish::*;
-use crate::PageServerConf;
-use crate::{ZTenantId, ZTimelineId};
-use anyhow::{bail, Result};
-use log::*;
-use std::collections::BTreeMap;
-use std::fmt;
-use std::fs;
-use std::fs::File;
-use std::io::Write;
-use std::ops::Bound::Included;
-use std::path::PathBuf;
-use std::sync::{Arc, Mutex, MutexGuard};
-
-use bookfile::{Book, BookWriter};
-
-use zenith_utils::bin_ser::BeSer;
-use zenith_utils::lsn::Lsn;
-
-// Magic constant to identify a Zenith snapshot file
-static SNAPSHOT_FILE_MAGIC: u32 = 0x5A616E01;
-
-static PAGE_VERSIONS_CHAPTER: u64 = 1;
-static REL_SIZES_CHAPTER: u64 = 2;
-
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
-struct SnapshotFileName {
-    seg: SegmentTag,
-    start_lsn: Lsn,
-    end_lsn: Lsn,
-    dropped: bool,
-}
-
-impl SnapshotFileName {
-    fn from_str(fname: &str) -> Option<Self> {
-        // Split the filename into parts
-        //
-        //    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>
-        //
-        // or if it was dropped:
-        //
-        //    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>_DROPPED
-        //
-        let rel;
-        let mut parts;
-        if let Some(rest) = fname.strip_prefix("rel_") {
-            parts = rest.split('_');
-            rel = RelishTag::Relation(RelTag {
-                spcnode: parts.next()?.parse::<u32>().ok()?,
-                dbnode: parts.next()?.parse::<u32>().ok()?,
-                relnode: parts.next()?.parse::<u32>().ok()?,
-                forknum: parts.next()?.parse::<u8>().ok()?,
-            });
-        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::Clog,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::MultiXactMembers,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::MultiXactOffsets,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
-            parts = rest.split('_');
-            rel = RelishTag::FileNodeMap {
-                spcnode: parts.next()?.parse::<u32>().ok()?,
-                dbnode: parts.next()?.parse::<u32>().ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
-            parts = rest.split('_');
-            rel = RelishTag::TwoPhase {
-                xid: parts.next()?.parse::<u32>().ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
-            parts = rest.split('_');
-            rel = RelishTag::Checkpoint;
-        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
-            parts = rest.split('_');
-            rel = RelishTag::ControlFile;
-        } else {
-            return None;
-        }
-
-        let segno = parts.next()?.parse::<u32>().ok()?;
-
-        let seg = SegmentTag { rel, segno };
-
-        let start_lsn = Lsn::from_hex(parts.next()?).ok()?;
-        let end_lsn = Lsn::from_hex(parts.next()?).ok()?;
-
-        let mut dropped = false;
-        if let Some(suffix) = parts.next() {
-            if suffix == "DROPPED" {
-                dropped = true;
-            } else {
-                warn!("unrecognized filename in timeline dir: {}", fname);
-                return None;
-            }
-        }
-        if parts.next().is_some() {
-            warn!("unrecognized filename in timeline dir: {}", fname);
-            return None;
-        }
-
-        Some(SnapshotFileName {
-            seg,
-            start_lsn,
-            end_lsn,
-            dropped,
-        })
-    }
-
-    fn to_string(&self) -> String {
-        let basename = match self.seg.rel {
-            RelishTag::Relation(reltag) => format!(
-                "rel_{}_{}_{}_{}",
-                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
-            ),
-            RelishTag::Slru {
-                slru: SlruKind::Clog,
-                segno,
-            } => format!("pg_xact_{:04X}", segno),
-            RelishTag::Slru {
-                slru: SlruKind::MultiXactMembers,
-                segno,
-            } => format!("pg_multixact_members_{:04X}", segno),
-            RelishTag::Slru {
-                slru: SlruKind::MultiXactOffsets,
-                segno,
-            } => format!("pg_multixact_offsets_{:04X}", segno),
-            RelishTag::FileNodeMap { spcnode, dbnode } => {
-                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
-            }
-            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
-            RelishTag::Checkpoint => format!("pg_control_checkpoint"),
-            RelishTag::ControlFile => format!("pg_control"),
-        };
-
-        format!(
-            "{}_{}_{:016X}_{:016X}{}",
-            basename,
-            self.seg.segno,
-            u64::from(self.start_lsn),
-            u64::from(self.end_lsn),
-            if self.dropped { "_DROPPED" } else { "" }
-        )
-    }
-}
-
-impl fmt::Display for SnapshotFileName {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}", self.to_string())
-    }
-}
-
-///
-/// SnapshotLayer is the in-memory data structure associated with an
-/// on-disk snapshot file.  We keep a SnapshotLayer in memory for each
-/// file, in the LayerMap. If a layer is in "loaded" state, we have a
-/// copy of the file in memory, in 'inner'. Otherwise the struct is
-/// just a placeholder for a file that exists on disk, and it needs to
-/// be loaded before using it in queries.
-///
-pub struct SnapshotLayer {
-    conf: &'static PageServerConf,
-    pub tenantid: ZTenantId,
-    pub timelineid: ZTimelineId,
-    pub seg: SegmentTag,
-
-    //
-    // This entry contains all the changes from 'start_lsn' to 'end_lsn'. The
-    // start is inclusive, and end is exclusive.
-    pub start_lsn: Lsn,
-    pub end_lsn: Lsn,
-
-    dropped: bool,
-
-    inner: Mutex<SnapshotLayerInner>,
-}
-
-pub struct SnapshotLayerInner {
-    /// If false, the 'page_versions' and 'relsizes' have not been
-    /// loaded into memory yet.
-    loaded: bool,
-
-    /// All versions of all pages in the file are are kept here.
-    /// Indexed by block number and LSN.
-    page_versions: BTreeMap<(u32, Lsn), PageVersion>,
-
-    /// `relsizes` tracks the size of the relation at different points in time.
-    relsizes: BTreeMap<Lsn, u32>,
-}
-
-impl Layer for SnapshotLayer {
-    fn get_timeline_id(&self) -> ZTimelineId {
-        return self.timelineid;
-    }
-
-    fn get_seg_tag(&self) -> SegmentTag {
-        return self.seg;
-    }
-
-    fn is_dropped(&self) -> bool {
-        return self.dropped;
-    }
-
-    fn get_start_lsn(&self) -> Lsn {
-        return self.start_lsn;
-    }
-
-    fn get_end_lsn(&self) -> Lsn {
-        return self.end_lsn;
-    }
-
-    /// Look up given page in the cache.
-    fn get_page_reconstruct_data(
-        &self,
-        blknum: u32,
-        lsn: Lsn,
-        reconstruct_data: &mut PageReconstructData,
-    ) -> Result<Option<Lsn>> {
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let mut need_base_image_lsn: Option<Lsn> = Some(lsn);
-        {
-            let inner = self.load()?;
-            let minkey = (blknum, Lsn(0));
-            let maxkey = (blknum, lsn);
-            let mut iter = inner
-                .page_versions
-                .range((Included(&minkey), Included(&maxkey)));
-            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
-                if let Some(img) = &entry.page_image {
-                    reconstruct_data.page_img = Some(img.clone());
-                    need_base_image_lsn = None;
-                    break;
-                } else if let Some(rec) = &entry.record {
-                    reconstruct_data.records.push(rec.clone());
-                    if rec.will_init {
-                        // This WAL record initializes the page, so no need to go further back
-                        need_base_image_lsn = None;
-                        break;
-                    } else {
-                        need_base_image_lsn = Some(*entry_lsn);
-                    }
-                } else {
-                    // No base image, and no WAL record. Huh?
-                    bail!("no page image or WAL record for requested page");
-                }
-            }
-
-            // release lock on 'inner'
-        }
-
-        Ok(need_base_image_lsn)
-    }
-
-    /// Get size of the relation at given LSN
-    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let inner = self.load()?;
-        let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn)));
-
-        if let Some((_entry_lsn, entry)) = iter.next_back() {
-            let result = *entry;
-            drop(inner);
-            trace!("get_seg_size: {} at {} -> {}", self.seg, lsn, result);
-            Ok(result)
-        } else {
-            error!(
-                "No size found for {} at {} in snapshot layer {} {}-{}",
-                self.seg, lsn, self.seg, self.start_lsn, self.end_lsn
-            );
-            bail!(
-                "No size found for {} at {} in snapshot layer",
-                self.seg,
-                lsn
-            );
-        }
-    }
-
-    /// Does this segment exist at given LSN?
-    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
-        // Is the requested LSN after the rel was dropped?
-        if self.dropped && lsn >= self.end_lsn {
-            return Ok(false);
-        }
-
-        // Otherwise, it exists.
-        Ok(true)
-    }
-}
-
-impl SnapshotLayer {
-    fn path(&self) -> PathBuf {
-        Self::path_for(
-            self.conf,
-            self.timelineid,
-            self.tenantid,
-            &SnapshotFileName {
-                seg: self.seg,
-                start_lsn: self.start_lsn,
-                end_lsn: self.end_lsn,
-                dropped: self.dropped,
-            },
-        )
-    }
-
-    fn path_for(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        fname: &SnapshotFileName,
-    ) -> PathBuf {
-        conf.timeline_path(&timelineid, &tenantid)
-            .join(fname.to_string())
-    }
-
-    /// Create a new snapshot file, using the given btreemaps containing the page versions and
-    /// relsizes.
-    ///
-    /// This is used to write the in-memory layer to disk. The in-memory layer uses the same
-    /// data structure with two btreemaps as we do, so passing the btreemaps is currently
-    /// expedient.
-    pub fn create(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        seg: SegmentTag,
-        start_lsn: Lsn,
-        end_lsn: Lsn,
-        dropped: bool,
-        page_versions: BTreeMap<(u32, Lsn), PageVersion>,
-        relsizes: BTreeMap<Lsn, u32>,
-    ) -> Result<SnapshotLayer> {
-        let snapfile = SnapshotLayer {
-            conf: conf,
-            timelineid: timelineid,
-            tenantid: tenantid,
-            seg: seg,
-            start_lsn: start_lsn,
-            end_lsn,
-            dropped,
-            inner: Mutex::new(SnapshotLayerInner {
-                loaded: true,
-                page_versions: page_versions,
-                relsizes: relsizes,
-            }),
-        };
-        let inner = snapfile.inner.lock().unwrap();
-
-        // Write the in-memory btreemaps into a file
-        let path = snapfile.path();
-
-        // Note: This overwrites any existing file. There shouldn't be any.
-        // FIXME: throw an error instead?
-        let file = File::create(&path)?;
-        let book = BookWriter::new(file, SNAPSHOT_FILE_MAGIC)?;
-
-        // Write out page versions
-        let mut chapter = book.new_chapter(PAGE_VERSIONS_CHAPTER);
-        let buf = BTreeMap::ser(&inner.page_versions)?;
-        chapter.write_all(&buf)?;
-        let book = chapter.close()?;
-
-        // and relsizes to separate chapter
-        let mut chapter = book.new_chapter(REL_SIZES_CHAPTER);
-        let buf = BTreeMap::ser(&inner.relsizes)?;
-        chapter.write_all(&buf)?;
-        let book = chapter.close()?;
-
-        book.close()?;
-
-        trace!("saved {}", &path.display());
-
-        drop(inner);
-
-        Ok(snapfile)
-    }
-
-    ///
-    /// Load the contents of the file into memory
-    ///
-    fn load(&self) -> Result<MutexGuard<SnapshotLayerInner>> {
-        // quick exit if already loaded
-        let mut inner = self.inner.lock().unwrap();
-
-        if inner.loaded {
-            return Ok(inner);
-        }
-
-        let path = Self::path_for(
-            self.conf,
-            self.timelineid,
-            self.tenantid,
-            &SnapshotFileName {
-                seg: self.seg,
-                start_lsn: self.start_lsn,
-                end_lsn: self.end_lsn,
-                dropped: self.dropped,
-            },
-        );
-
-        let file = File::open(&path)?;
-        let book = Book::new(file)?;
-
-        let chapter = book.read_chapter(PAGE_VERSIONS_CHAPTER)?;
-        let page_versions = BTreeMap::des(&chapter)?;
-
-        let chapter = book.read_chapter(REL_SIZES_CHAPTER)?;
-        let relsizes = BTreeMap::des(&chapter)?;
-
-        debug!("loaded from {}", &path.display());
-
-        *inner = SnapshotLayerInner {
-            loaded: true,
-            page_versions,
-            relsizes,
-        };
-
-        Ok(inner)
-    }
-
-    /// Create SnapshotLayers representing all files on disk
-    ///
-    // TODO: returning an Iterator would be more idiomatic
-    pub fn list_snapshot_files(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-    ) -> Result<Vec<Arc<SnapshotLayer>>> {
-        let path = conf.timeline_path(&timelineid, &tenantid);
-
-        let mut snapfiles: Vec<Arc<SnapshotLayer>> = Vec::new();
-        for direntry in fs::read_dir(path)? {
-            let fname = direntry?.file_name();
-            let fname = fname.to_str().unwrap();
-
-            if let Some(snapfilename) = SnapshotFileName::from_str(fname) {
-                let snapfile = SnapshotLayer {
-                    conf,
-                    timelineid,
-                    tenantid,
-                    seg: snapfilename.seg,
-                    start_lsn: snapfilename.start_lsn,
-                    end_lsn: snapfilename.end_lsn,
-                    dropped: snapfilename.dropped,
-                    inner: Mutex::new(SnapshotLayerInner {
-                        loaded: false,
-                        page_versions: BTreeMap::new(),
-                        relsizes: BTreeMap::new(),
-                    }),
-                };
-
-                snapfiles.push(Arc::new(snapfile));
-            }
-        }
-        return Ok(snapfiles);
-    }
-
-    pub fn delete(&self) -> Result<()> {
-        // delete underlying file
-        fs::remove_file(self.path())?;
-        Ok(())
-    }
-
-    ///
-    /// Release most of the memory used by this layer. If it's accessed again later,
-    /// it will need to be loaded back.
-    ///
-    pub fn unload(&self) -> Result<()> {
-        let mut inner = self.inner.lock().unwrap();
-        inner.page_versions = BTreeMap::new();
-        inner.relsizes = BTreeMap::new();
-        inner.loaded = false;
-        Ok(())
-    }
-
-    /// debugging function to print out the contents of the layer
-    #[allow(unused)]
-    pub fn dump(&self) -> String {
-        let mut result = format!(
-            "----- snapshot layer for {} {}-{} ----\n",
-            self.seg, self.start_lsn, self.end_lsn
-        );
-
-        let inner = self.inner.lock().unwrap();
-        for (k, v) in inner.relsizes.iter() {
-            result += &format!("{}: {}\n", k, v);
-        }
-        //for (k, v) in inner.page_versions.iter() {
-        //    result += &format!("blk {} at {}: {}/{}\n", k.0, k.1, v.page_image.is_some(), v.record.is_some());
-        //}
-
-        result
-    }
-}
--- a/pageserver/src/layered_repository/storage_layer.rs
+++ b/pageserver/src/layered_repository/storage_layer.rs
@@ -9,6 +9,8 @@ use anyhow::Result;
 use bytes::Bytes;
 use serde::{Deserialize, Serialize};
 use std::fmt;
+use std::path::PathBuf;
+use std::sync::Arc;

 use zenith_utils::lsn::Lsn;

@@ -19,7 +21,7 @@ pub const RELISH_SEG_SIZE: u32 = 10 * 1024 * 1024 / 8192;
 /// Each relish stored in the repository is divided into fixed-sized "segments",
 /// with 10 MB of key-space, or 1280 8k pages each.
 ///
-#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy, Serialize, Deserialize)]
 pub struct SegmentTag {
    pub rel: RelishTag,
    pub segno: u32,
@@ -81,48 +83,98 @@ pub struct PageReconstructData {
    pub page_img: Option<Bytes>,
 }

+/// Return value from Layer::get_page_reconstruct_data
+pub enum PageReconstructResult {
+    /// Got all the data needed to reconstruct the requested page
+    Complete,
+    /// This layer didn't contain all the required data, the caller should collect
+    /// more data from the returned predecessor layer at the returned LSN.
+    Continue(Lsn, Arc<dyn Layer>),
+    /// This layer didn't contain data needed to reconstruct the page version at
+    /// the returned LSN. This is usually considered an error, but might be OK
+    /// in some circumstances.
+    Missing(Lsn),
+}
+
 ///
-/// A Layer holds all page versions for one segment of a relish, in a range of LSNs.
-/// There are two kinds of layers, in-memory and snapshot layers. In-memory
+/// A Layer corresponds to one RELISH_SEG_SIZE slice of a relish in a range of LSNs.
+/// There are two kinds of layers, in-memory and on-disk layers. In-memory
 /// layers are used to ingest incoming WAL, and provide fast access
-/// to the recent page versions. Snaphot layers are stored on disk, and
+/// to the recent page versions. On-disk layers are stored as files on disk, and
 /// are immutable. This trait presents the common functionality of
-/// in-memory and snapshot layers.
-///
-/// Each layer contains a full snapshot of the segment at the start
-/// LSN. In addition to that, it contains WAL (or more page images)
-/// needed to recontruct any page version up to the end LSN.
+/// in-memory and on-disk layers.
 ///
 pub trait Layer: Send + Sync {
-    // These functions identify the relish segment and the LSN range
-    // that this Layer holds.
+    /// Identify the timeline this relish belongs to
    fn get_timeline_id(&self) -> ZTimelineId;
+
+    /// Identify the relish segment
    fn get_seg_tag(&self) -> SegmentTag;
+
+    /// Inclusive start bound of the LSN range that this layer hold
    fn get_start_lsn(&self) -> Lsn;
+
+    /// 'end_lsn' meaning depends on the layer kind:
+    /// - in-memory layer is either unbounded (end_lsn = MAX_LSN) or dropped (end_lsn = drop_lsn)
+    /// - image layer represents snapshot at one LSN, so end_lsn = lsn
+    /// - delta layer has end_lsn
+    ///
+    /// TODO Is end_lsn always exclusive for all layer kinds?
    fn get_end_lsn(&self) -> Lsn;
+
+    /// Is the segment represented by this layer dropped by PostgreSQL?
    fn is_dropped(&self) -> bool;

+    /// Gets the physical location of the layer on disk.
+    /// Some layers, such as in-memory, might not have the location.
+    fn path(&self) -> Option<PathBuf>;
+
+    /// Filename used to store this layer on disk. (Even in-memory layers
+    /// implement this, to print a handy unique identifier for the layer for
+    /// log messages, even though they're never not on disk.)
+    fn filename(&self) -> PathBuf;
+
    ///
    /// Return data needed to reconstruct given page at LSN.
    ///
    /// It is up to the caller to collect more data from previous layer and
    /// perform WAL redo, if necessary.
    ///
-    /// If returns Some, the returned data is not complete. The caller needs
-    /// to continue with the returned 'lsn'.
-    ///
    /// Note that the 'blknum' is the offset of the page from the beginning
    /// of the *relish*, not the beginning of the segment. The requested
    /// 'blknum' must be covered by this segment.
+    ///
+    /// See PageReconstructResult for possible return values. The collected data
+    /// is appended to reconstruct_data; the caller should pass an empty struct
+    /// on first call. If this returns PageReconstructResult::Continue, call
+    /// again on the returned predecessor layer with the same 'reconstruct_data'
+    /// to collect more data.
    fn get_page_reconstruct_data(
        &self,
        blknum: u32,
        lsn: Lsn,
        reconstruct_data: &mut PageReconstructData,
-    ) -> Result<Option<Lsn>>;
+    ) -> Result<PageReconstructResult>;

-    // Functions that correspond to the Timeline trait functions.
+    /// Return size of the segment at given LSN. (Only for blocky relations.)
    fn get_seg_size(&self, lsn: Lsn) -> Result<u32>;

+    /// Does the segment exist at given LSN? Or was it dropped before it.
    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool>;
+
+    /// Does this layer only contain some data for the segment (incremental),
+    /// or does it contain a version of every page? This is important to know
+    /// for garbage collecting old layers: an incremental layer depends on
+    /// the previous non-incremental layer.
+    fn is_incremental(&self) -> bool;
+
+    /// Release memory used by this layer. There is no corresponding 'load'
+    /// function, that's done implicitly when you call one of the get-functions.
+    fn unload(&self) -> Result<()>;
+
+    /// Permanently remove this layer from disk.
+    fn delete(&self) -> Result<()>;
+
+    /// Dump summary of the contents of the layer to stdout
+    fn dump(&self) -> Result<()>;
 }
--- a/pageserver/src/lib.rs
+++ b/pageserver/src/lib.rs
@@ -9,21 +9,38 @@ use zenith_metrics::{register_int_gauge_vec, IntGaugeVec};

 pub mod basebackup;
 pub mod branches;
+pub mod http;
 pub mod layered_repository;
-pub mod logger;
-pub mod object_key;
-pub mod object_repository;
-pub mod object_store;
-pub mod page_cache;
 pub mod page_service;
 pub mod relish;
+mod relish_storage;
 pub mod repository;
 pub mod restore_local_repo;
-pub mod rocksdb_storage;
+pub mod tenant_mgr;
 pub mod waldecoder;
 pub mod walreceiver;
 pub mod walredo;

+pub mod defaults {
+    use std::time::Duration;
+
+    pub const DEFAULT_PG_LISTEN_PORT: u16 = 64000;
+    pub const DEFAULT_PG_LISTEN_ADDR: &str = "127.0.0.1:64000"; // can't format! const yet...
+    pub const DEFAULT_HTTP_LISTEN_PORT: u16 = 9898;
+    pub const DEFAULT_HTTP_LISTEN_ADDR: &str = "127.0.0.1:9898";
+
+    // FIXME: This current value is very low. I would imagine something like 1 GB or 10 GB
+    // would be more appropriate. But a low value forces the code to be exercised more,
+    // which is good for now to trigger bugs.
+    pub const DEFAULT_CHECKPOINT_DISTANCE: u64 = 256 * 1024 * 1024;
+    pub const DEFAULT_CHECKPOINT_PERIOD: Duration = Duration::from_secs(1);
+
+    pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
+    pub const DEFAULT_GC_PERIOD: Duration = Duration::from_secs(100);
+
+    pub const DEFAULT_SUPERUSER: &str = "zenith_admin";
+}
+
 lazy_static! {
    static ref LIVE_CONNECTIONS_COUNT: IntGaugeVec = register_int_gauge_vec!(
        "pageserver_live_connections_count",
@@ -33,11 +50,19 @@ lazy_static! {
    .expect("failed to define a metric");
 }

+pub const LOG_FILE_NAME: &str = "pageserver.log";
+
 #[derive(Debug, Clone)]
 pub struct PageServerConf {
    pub daemonize: bool,
-    pub listen_addr: String,
-    pub http_endpoint_addr: String,
+    pub listen_pg_addr: String,
+    pub listen_http_addr: String,
+    // Flush out an inmemory layer, if it's holding WAL older than this
+    // This puts a backstop on how much WAL needs to be re-digested if the
+    // page server crashes.
+    pub checkpoint_distance: u64,
+    pub checkpoint_period: Duration,
+
    pub gc_horizon: u64,
    pub gc_period: Duration,
    pub superuser: String,
@@ -55,14 +80,7 @@ pub struct PageServerConf {
    pub auth_type: AuthType,

    pub auth_validation_public_key_path: Option<PathBuf>,
-
-    pub repository_format: RepositoryFormat,
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub enum RepositoryFormat {
-    Layered,
-    RocksDb,
+    pub relish_storage_config: Option<RelishStorageConfig>,
 }

 impl PageServerConf {
@@ -121,4 +139,54 @@ impl PageServerConf {
    pub fn pg_lib_dir(&self) -> PathBuf {
        self.pg_distrib_dir.join("lib")
    }
+
+    #[cfg(test)]
+    fn test_repo_dir(test_name: &str) -> PathBuf {
+        PathBuf::from(format!("../tmp_check/test_{}", test_name))
+    }
+
+    #[cfg(test)]
+    fn dummy_conf(repo_dir: PathBuf) -> Self {
+        PageServerConf {
+            daemonize: false,
+            checkpoint_distance: defaults::DEFAULT_CHECKPOINT_DISTANCE,
+            checkpoint_period: Duration::from_secs(10),
+            gc_horizon: defaults::DEFAULT_GC_HORIZON,
+            gc_period: Duration::from_secs(10),
+            listen_pg_addr: "127.0.0.1:5430".to_string(),
+            listen_http_addr: "127.0.0.1:9898".to_string(),
+            superuser: "zenith_admin".to_string(),
+            workdir: repo_dir,
+            pg_distrib_dir: "".into(),
+            auth_type: AuthType::Trust,
+            auth_validation_public_key_path: None,
+            relish_storage_config: None,
+        }
+    }
+}
+
+/// External relish storage configuration, enough for creating a client for that storage.
+#[derive(Debug, Clone)]
+pub enum RelishStorageConfig {
+    /// Root folder to place all stored relish data into.
+    LocalFs(PathBuf),
+    AwsS3(S3Config),
+}
+
+/// AWS S3 bucket coordinates and access credentials to manage the bucket contents (read and write).
+#[derive(Clone)]
+pub struct S3Config {
+    pub bucket_name: String,
+    pub bucket_region: String,
+    pub access_key_id: Option<String>,
+    pub secret_access_key: Option<String>,
+}
+
+impl std::fmt::Debug for S3Config {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("S3Config")
+            .field("bucket_name", &self.bucket_name)
+            .field("bucket_region", &self.bucket_region)
+            .finish()
+    }
 }
--- a/pageserver/src/logger.rs
+++ b/pageserver/src/logger.rs
@@ -1,45 +0,0 @@
-use crate::PageServerConf;
-
-use anyhow::{Context, Result};
-use slog::{Drain, FnValue};
-use std::fs::{File, OpenOptions};
-
-pub fn init_logging(
-    _conf: &PageServerConf,
-    log_filename: &str,
-) -> Result<(slog_scope::GlobalLoggerGuard, File)> {
-    // Don't open the same file for output multiple times;
-    // the different fds could overwrite each other's output.
-    let log_file = OpenOptions::new()
-        .create(true)
-        .append(true)
-        .open(&log_filename)
-        .with_context(|| format!("failed to open {:?}", &log_filename))?;
-
-    let logger_file = log_file.try_clone().unwrap();
-
-    let decorator = slog_term::PlainSyncDecorator::new(logger_file);
-    let drain = slog_term::FullFormat::new(decorator).build();
-    let drain = slog::Filter::new(drain, |record: &slog::Record| {
-        if record.level().is_at_least(slog::Level::Info) {
-            return true;
-        }
-        false
-    });
-    let drain = std::sync::Mutex::new(drain).fuse();
-    let logger = slog::Logger::root(
-        drain,
-        slog::o!(
-            "location" =>
-                FnValue(move |record| {
-                    format!("{}, {}:{}",
-                            record.module(),
-                            record.file(),
-                            record.line()
-                    )
-                }
-                )
-        ),
-    );
-    Ok((slog_scope::set_global_logger(logger), log_file))
-}
--- a/pageserver/src/object_key.rs
+++ b/pageserver/src/object_key.rs
@@ -1,49 +0,0 @@
-//!
-//! Common structs shared by object_repository.rs and object_store.rs.
-//!
-
-use crate::relish::RelishTag;
-use serde::{Deserialize, Serialize};
-use zenith_utils::zid::ZTimelineId;
-
-///
-/// ObjectKey is the key type used to identify objects stored in an object
-/// repository. It is shared between object_repository.rs and object_store.rs.
-/// It is mostly opaque to ObjectStore, it just stores and retrieves objects
-/// using the key given by the caller.
-///
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ObjectKey {
-    pub timeline: ZTimelineId,
-    pub tag: ObjectTag,
-}
-
-///
-/// ObjectTag is a part of ObjectKey that is specific to the type of
-/// the stored object.
-///
-/// NB: the order of the enum values is significant!  In particular,
-/// rocksdb_storage.rs assumes that TimelineMetadataTag is first
-///
-/// Buffer is the kind of object that is accessible by the public
-/// get_page_at_lsn() / put_page_image() / put_wal_record() functions in
-/// the repository.rs interface. The rest are internal objects stored in
-/// the key-value store, to store various metadata. They're not directly
-/// accessible outside object_repository.rs
-///
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
-pub enum ObjectTag {
-    // dummy tag preceeding all other keys
-    FirstTag,
-
-    // Metadata about a timeline. Not versioned.
-    TimelineMetadataTag,
-
-    // These objects store metadata about one relish. Currently it's used
-    // just to track the relish's size. It's not used for non-blocky relishes
-    // at all.
-    RelationMetadata(RelishTag),
-
-    // These are the pages exposed in the public Repository/Timeline interface.
-    Buffer(RelishTag, u32),
-}
--- a/pageserver/src/object_repository.rs
+++ b/pageserver/src/object_repository.rs
--- a/pageserver/src/object_store.rs
+++ b/pageserver/src/object_store.rs
@@ -1,92 +0,0 @@
-//! Low-level key-value storage abstraction.
-//!
-use crate::object_key::*;
-use crate::relish::*;
-use anyhow::Result;
-use std::collections::HashSet;
-use std::iter::Iterator;
-use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::ZTimelineId;
-
-///
-/// Low-level storage abstraction.
-///
-/// All the data in the repository is stored in a key-value store. This trait
-/// abstracts the details of the key-value store.
-///
-/// A simple key-value store would support just GET and PUT operations with
-/// a key, but the upper layer needs slightly complicated read operations
-///
-/// The most frequently used function is 'object_versions'. It is used
-/// to look up a page version. It is LSN aware, in that the caller
-/// specifies an LSN, and the function returns all values for that
-/// block with the same or older LSN.
-///
-pub trait ObjectStore: Send + Sync {
-    ///
-    /// Store a value with given key.
-    ///
-    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()>;
-
-    /// Read entry with the exact given key.
-    ///
-    /// This is used for retrieving metadata with special key that doesn't
-    /// correspond to any real relation.
-    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>>;
-
-    /// Read key greater or equal than specified
-    fn get_next_key(&self, key: &ObjectKey) -> Result<Option<ObjectKey>>;
-
-    /// Iterate through all page versions of one object.
-    ///
-    /// Returns all page versions in descending LSN order, along with the LSN
-    /// of each page version.
-    fn object_versions<'a>(
-        &'a self,
-        key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>>;
-
-    /// Iterate through versions of all objects in a timeline.
-    ///
-    /// Returns objects in increasing key-version order.
-    /// Returns all versions up to and including the specified LSN.
-    fn objects<'a>(
-        &'a self,
-        timeline: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = Result<(ObjectTag, Lsn, Vec<u8>)>> + 'a>>;
-
-    /// Iterate through all keys with given tablespace and database ID, and LSN <= 'lsn'.
-    /// Both dbnode and spcnode can be InvalidId (0) which means get all relations in tablespace/cluster
-    ///
-    /// This is used to implement 'create database'
-    fn list_rels(
-        &self,
-        timelineid: ZTimelineId,
-        spcnode: u32,
-        dbnode: u32,
-        lsn: Lsn,
-    ) -> Result<HashSet<RelTag>>;
-
-    /// Iterate through non-rel relishes
-    ///
-    /// This is used to prepare tarball for new node startup.
-    fn list_nonrels<'a>(&'a self, timelineid: ZTimelineId, lsn: Lsn) -> Result<HashSet<RelishTag>>;
-
-    /// Iterate through objects tags. If nonrel_only, then only non-relationa data is iterated.
-    ///
-    /// This is used to implement GC and preparing tarball for new node startup
-    /// Returns objects in increasing key-version order.
-    fn list_objects<'a>(
-        &'a self,
-        timelineid: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = ObjectTag> + 'a>>;
-
-    /// Unlink object (used by GC). This mehod may actually delete object or just mark it for deletion.
-    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()>;
-
-    // Compact storage and remove versions marged for deletion
-    fn compact(&self);
-}
--- a/pageserver/src/page_service.rs
+++ b/pageserver/src/page_service.rs
@@ -15,7 +15,6 @@ use bytes::{Buf, BufMut, Bytes, BytesMut};
 use lazy_static::lazy_static;
 use log::*;
 use regex::Regex;
-use std::io::Write;
 use std::net::TcpListener;
 use std::str;
 use std::str::FromStr;
@@ -23,86 +22,123 @@ use std::sync::Arc;
 use std::thread;
 use std::{io, net::TcpStream};
 use zenith_metrics::{register_histogram_vec, HistogramVec};
-use zenith_utils::auth::JwtAuth;
+use zenith_utils::auth::{self, JwtAuth};
 use zenith_utils::auth::{Claims, Scope};
+use zenith_utils::lsn::Lsn;
 use zenith_utils::postgres_backend::PostgresBackend;
 use zenith_utils::postgres_backend::{self, AuthType};
 use zenith_utils::pq_proto::{
    BeMessage, FeMessage, RowDescriptor, HELLO_WORLD_ROW, SINGLE_COL_ROWDESC,
 };
 use zenith_utils::zid::{ZTenantId, ZTimelineId};
-use zenith_utils::{bin_ser::BeSer, lsn::Lsn};

 use crate::basebackup;
 use crate::branches;
-use crate::page_cache;
 use crate::relish::*;
-use crate::repository::Modification;
+use crate::repository::Timeline;
+use crate::tenant_mgr;
 use crate::walreceiver;
 use crate::PageServerConf;

 // Wrapped in libpq CopyData
 enum PagestreamFeMessage {
-    Exists(PagestreamRequest),
-    Nblocks(PagestreamRequest),
-    Read(PagestreamRequest),
+    Exists(PagestreamExistsRequest),
+    Nblocks(PagestreamNblocksRequest),
+    GetPage(PagestreamGetPageRequest),
 }

 // Wrapped in libpq CopyData
 enum PagestreamBeMessage {
-    Status(PagestreamStatusResponse),
-    Nblocks(PagestreamStatusResponse),
-    Read(PagestreamReadResponse),
+    Exists(PagestreamExistsResponse),
+    Nblocks(PagestreamNblocksResponse),
+    GetPage(PagestreamGetPageResponse),
+    Error(PagestreamErrorResponse),
 }

 #[derive(Debug)]
-struct PagestreamRequest {
-    spcnode: u32,
-    dbnode: u32,
-    relnode: u32,
-    forknum: u8,
-    blkno: u32,
+struct PagestreamExistsRequest {
+    latest: bool,
    lsn: Lsn,
+    rel: RelTag,
 }

 #[derive(Debug)]
-struct PagestreamStatusResponse {
-    ok: bool,
+struct PagestreamNblocksRequest {
+    latest: bool,
+    lsn: Lsn,
+    rel: RelTag,
+}
+
+#[derive(Debug)]
+struct PagestreamGetPageRequest {
+    latest: bool,
+    lsn: Lsn,
+    rel: RelTag,
+    blkno: u32,
+}
+
+#[derive(Debug)]
+struct PagestreamExistsResponse {
+    exists: bool,
+}
+
+#[derive(Debug)]
+struct PagestreamNblocksResponse {
    n_blocks: u32,
 }

 #[derive(Debug)]
-struct PagestreamReadResponse {
-    ok: bool,
-    n_blocks: u32,
+struct PagestreamGetPageResponse {
    page: Bytes,
 }

+#[derive(Debug)]
+struct PagestreamErrorResponse {
+    message: String,
+}
+
 impl PagestreamFeMessage {
    fn parse(mut body: Bytes) -> anyhow::Result<PagestreamFeMessage> {
        // TODO these gets can fail

-        let smgr_tag = body.get_u8();
-        let zreq = PagestreamRequest {
-            spcnode: body.get_u32(),
-            dbnode: body.get_u32(),
-            relnode: body.get_u32(),
-            forknum: body.get_u8(),
-            blkno: body.get_u32(),
-            lsn: Lsn::from(body.get_u64()),
-        };
-
+        // these correspond to the ZenithMessageTag enum in pagestore_client.h
+        //
        // TODO: consider using protobuf or serde bincode for less error prone
        // serialization.
-        match smgr_tag {
-            0 => Ok(PagestreamFeMessage::Exists(zreq)),
-            1 => Ok(PagestreamFeMessage::Nblocks(zreq)),
-            2 => Ok(PagestreamFeMessage::Read(zreq)),
-            _ => Err(anyhow!(
-                "unknown smgr message tag: {},'{:?}'",
-                smgr_tag,
-                body
-            )),
+        let msg_tag = body.get_u8();
+        match msg_tag {
+            0 => Ok(PagestreamFeMessage::Exists(PagestreamExistsRequest {
+                latest: body.get_u8() != 0,
+                lsn: Lsn::from(body.get_u64()),
+                rel: RelTag {
+                    spcnode: body.get_u32(),
+                    dbnode: body.get_u32(),
+                    relnode: body.get_u32(),
+                    forknum: body.get_u8(),
+                },
+            })),
+            1 => Ok(PagestreamFeMessage::Nblocks(PagestreamNblocksRequest {
+                latest: body.get_u8() != 0,
+                lsn: Lsn::from(body.get_u64()),
+                rel: RelTag {
+                    spcnode: body.get_u32(),
+                    dbnode: body.get_u32(),
+                    relnode: body.get_u32(),
+                    forknum: body.get_u8(),
+                },
+            })),
+            2 => Ok(PagestreamFeMessage::GetPage(PagestreamGetPageRequest {
+                latest: body.get_u8() != 0,
+                lsn: Lsn::from(body.get_u64()),
+                rel: RelTag {
+                    spcnode: body.get_u32(),
+                    dbnode: body.get_u32(),
+                    relnode: body.get_u32(),
+                    forknum: body.get_u8(),
+                },
+                blkno: body.get_u32(),
+            })),
+            _ => bail!("unknown smgr message tag: {},'{:?}'", msg_tag, body),
        }
    }
 }
@@ -112,24 +148,26 @@ impl PagestreamBeMessage {
        let mut bytes = BytesMut::new();

        match self {
-            Self::Status(resp) => {
+            Self::Exists(resp) => {
                bytes.put_u8(100); /* tag from pagestore_client.h */
-                bytes.put_u8(resp.ok as u8);
-                bytes.put_u32(resp.n_blocks);
+                bytes.put_u8(resp.exists as u8);
            }

            Self::Nblocks(resp) => {
                bytes.put_u8(101); /* tag from pagestore_client.h */
-                bytes.put_u8(resp.ok as u8);
                bytes.put_u32(resp.n_blocks);
            }

-            Self::Read(resp) => {
+            Self::GetPage(resp) => {
                bytes.put_u8(102); /* tag from pagestore_client.h */
-                bytes.put_u8(resp.ok as u8);
-                bytes.put_u32(resp.n_blocks);
                bytes.put(&resp.page[..]);
            }
+
+            Self::Error(resp) => {
+                bytes.put_u8(103); /* tag from pagestore_client.h */
+                bytes.put(resp.message.as_bytes());
+                bytes.put_u8(0); // null terminator
+            }
        }

        bytes.into()
@@ -145,7 +183,7 @@ impl PagestreamBeMessage {
 ///
 pub fn thread_main(
    conf: &'static PageServerConf,
-    auth: Arc<Option<JwtAuth>>,
+    auth: Option<Arc<JwtAuth>>,
    listener: TcpListener,
    auth_type: AuthType,
 ) -> anyhow::Result<()> {
@@ -153,7 +191,7 @@ pub fn thread_main(
        let (socket, peer_addr) = listener.accept()?;
        debug!("accepted connection from {}", peer_addr);
        socket.set_nodelay(true).unwrap();
-        let local_auth = Arc::clone(&auth);
+        let local_auth = auth.clone();
        thread::spawn(move || {
            if let Err(err) = page_service_conn_main(conf, local_auth, socket, auth_type) {
                error!("error: {}", err);
@@ -164,7 +202,7 @@ pub fn thread_main(

 fn page_service_conn_main(
    conf: &'static PageServerConf,
-    auth: Arc<Option<JwtAuth>>,
+    auth: Option<Arc<JwtAuth>>,
    socket: TcpStream,
    auth_type: AuthType,
 ) -> anyhow::Result<()> {
@@ -178,14 +216,14 @@ fn page_service_conn_main(
    }

    let mut conn_handler = PageServerHandler::new(conf, auth);
-    let pgbackend = PostgresBackend::new(socket, auth_type)?;
+    let pgbackend = PostgresBackend::new(socket, auth_type, None)?;
    pgbackend.run(&mut conn_handler)
 }

 #[derive(Debug)]
 struct PageServerHandler {
    conf: &'static PageServerConf,
-    auth: Arc<Option<JwtAuth>>,
+    auth: Option<Arc<JwtAuth>>,
    claims: Option<Claims>,
 }

@@ -208,7 +246,7 @@ lazy_static! {
 }

 impl PageServerHandler {
-    pub fn new(conf: &'static PageServerConf, auth: Arc<Option<JwtAuth>>) -> Self {
+    pub fn new(conf: &'static PageServerConf, auth: Option<Arc<JwtAuth>>) -> Self {
        PageServerHandler {
            conf,
            auth,
@@ -216,14 +254,6 @@ impl PageServerHandler {
        }
    }

-    fn handle_controlfile(&self, pgb: &mut PostgresBackend) -> io::Result<()> {
-        pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
-            .write_message_noflush(&BeMessage::ControlFile)?
-            .write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;
-
-        Ok(())
-    }
-
    fn handle_pagerequests(
        &self,
        pgb: &mut PostgresBackend,
@@ -231,13 +261,7 @@ impl PageServerHandler {
        tenantid: ZTenantId,
    ) -> anyhow::Result<()> {
        // Check that the timeline exists
-        let repository = page_cache::get_repository_for_tenant(&tenantid)?;
-        let timeline = repository.get_timeline(timelineid).map_err(|_| {
-            anyhow!(
-                "client requested pagestream on timeline {} which does not exist in page server",
-                timelineid
-            )
-        })?;
+        let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)?;

        /* switch client to COPYBOTH */
        pgb.write_message(&BeMessage::CopyBothResponse)?;
@@ -253,88 +277,134 @@ impl PageServerHandler {
            let zenith_fe_msg = PagestreamFeMessage::parse(copy_data_bytes)?;

            let response = match zenith_fe_msg {
-                PagestreamFeMessage::Exists(req) => {
-                    let rel = RelTag {
-                        spcnode: req.spcnode,
-                        dbnode: req.dbnode,
-                        relnode: req.relnode,
-                        forknum: req.forknum,
-                    };
-                    let tag = RelishTag::Relation(rel);
-
-                    let exist = SMGR_QUERY_TIME
-                        .with_label_values(&["get_rel_exists"])
-                        .observe_closure_duration(|| {
-                            timeline.get_rel_exists(tag, req.lsn).unwrap_or(false)
-                        });
-
-                    PagestreamBeMessage::Status(PagestreamStatusResponse {
-                        ok: exist,
-                        n_blocks: 0,
-                    })
-                }
-                PagestreamFeMessage::Nblocks(req) => {
-                    let rel = RelTag {
-                        spcnode: req.spcnode,
-                        dbnode: req.dbnode,
-                        relnode: req.relnode,
-                        forknum: req.forknum,
-                    };
-                    let tag = RelishTag::Relation(rel);
-
-                    let n_blocks = SMGR_QUERY_TIME
-                        .with_label_values(&["get_rel_size"])
-                        .observe_closure_duration(|| {
-                            // Return 0 if relation is not found.
-                            // This is what postgres smgr expects.
-                            timeline
-                                .get_relish_size(tag, req.lsn)
-                                .unwrap_or(Some(0))
-                                .unwrap_or(0)
-                        });
-
-                    PagestreamBeMessage::Nblocks(PagestreamStatusResponse { ok: true, n_blocks })
-                }
-                PagestreamFeMessage::Read(req) => {
-                    let rel = RelTag {
-                        spcnode: req.spcnode,
-                        dbnode: req.dbnode,
-                        relnode: req.relnode,
-                        forknum: req.forknum,
-                    };
-                    let tag = RelishTag::Relation(rel);
-
-                    let read_response = SMGR_QUERY_TIME
-                        .with_label_values(&["get_page_at_lsn"])
-                        .observe_closure_duration(|| {
-                            match timeline.get_page_at_lsn(tag, req.blkno, req.lsn) {
-                                Ok(p) => PagestreamReadResponse {
-                                    ok: true,
-                                    n_blocks: 0,
-                                    page: p,
-                                },
-                                Err(e) => {
-                                    const ZERO_PAGE: [u8; 8192] = [0; 8192];
-                                    error!("get_page_at_lsn: {}", e);
-                                    PagestreamReadResponse {
-                                        ok: false,
-                                        n_blocks: 0,
-                                        page: Bytes::from_static(&ZERO_PAGE),
-                                    }
-                                }
-                            }
-                        });
-
-                    PagestreamBeMessage::Read(read_response)
-                }
+                PagestreamFeMessage::Exists(req) => SMGR_QUERY_TIME
+                    .with_label_values(&["get_rel_exists"])
+                    .observe_closure_duration(|| {
+                        self.handle_get_rel_exists_request(&*timeline, &req)
+                    }),
+                PagestreamFeMessage::Nblocks(req) => SMGR_QUERY_TIME
+                    .with_label_values(&["get_rel_size"])
+                    .observe_closure_duration(|| self.handle_get_nblocks_request(&*timeline, &req)),
+                PagestreamFeMessage::GetPage(req) => SMGR_QUERY_TIME
+                    .with_label_values(&["get_page_at_lsn"])
+                    .observe_closure_duration(|| {
+                        self.handle_get_page_at_lsn_request(&*timeline, &req)
+                    }),
            };

+            let response = response.unwrap_or_else(|e| {
+                error!("error reading relation or page version: {}", e);
+                PagestreamBeMessage::Error(PagestreamErrorResponse {
+                    message: e.to_string(),
+                })
+            });
+
            pgb.write_message(&BeMessage::CopyData(&response.serialize()))?;
        }

        Ok(())
    }

+    /// Helper function to handle the LSN from client request.
+    ///
+    /// Each GetPage (and Exists and Nblocks) request includes information about
+    /// which version of the page is being requested. The client can request the
+    /// latest version of the page, or the version that's valid at a particular
+    /// LSN. The primary compute node will always request the latest page
+    /// version, while a standby will request a version at the LSN that it's
+    /// currently caught up to.
+    ///
+    /// In either case, if the page server hasn't received the WAL up to the
+    /// requested LSN yet, we will wait for it to arrive. The return value is
+    /// the LSN that should be used to look up the page versions.
+    fn wait_or_get_last_lsn(timeline: &dyn Timeline, lsn: Lsn, latest: bool) -> Result<Lsn> {
+        if latest {
+            // Latest page version was requested. If LSN is given, it is a hint
+            // to the page server that there have been no modifications to the
+            // page after that LSN. If we haven't received WAL up to that point,
+            // wait until it arrives.
+            let last_record_lsn = timeline.get_last_record_lsn();
+
+            // Note: this covers the special case that lsn == Lsn(0). That
+            // special case means "return the latest version whatever it is",
+            // and it's used for bootstrapping purposes, when the page server is
+            // connected directly to the compute node. That is needed because
+            // when you connect to the compute node, to receive the WAL, the
+            // walsender process will do a look up in the pg_authid catalog
+            // table for authentication. That poses a deadlock problem: the
+            // catalog table lookup will send a GetPage request, but the GetPage
+            // request will block in the page server because the recent WAL
+            // hasn't been received yet, and it cannot be received until the
+            // walsender completes the authentication and starts streaming the
+            // WAL.
+            if lsn <= last_record_lsn {
+                Ok(last_record_lsn)
+            } else {
+                timeline.wait_lsn(lsn)?;
+                // Since we waited for 'lsn' to arrive, that is now the last
+                // record LSN. (Or close enough for our purposes; the
+                // last-record LSN can advance immediately after we return
+                // anyway)
+                Ok(lsn)
+            }
+        } else {
+            if lsn == Lsn(0) {
+                bail!("invalid LSN(0) in request");
+            }
+            timeline.wait_lsn(lsn)?;
+            Ok(lsn)
+        }
+    }
+
+    fn handle_get_rel_exists_request(
+        &self,
+        timeline: &dyn Timeline,
+        req: &PagestreamExistsRequest,
+    ) -> Result<PagestreamBeMessage> {
+        let tag = RelishTag::Relation(req.rel);
+        let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest)?;
+
+        let exists = timeline.get_rel_exists(tag, lsn)?;
+
+        Ok(PagestreamBeMessage::Exists(PagestreamExistsResponse {
+            exists,
+        }))
+    }
+
+    fn handle_get_nblocks_request(
+        &self,
+        timeline: &dyn Timeline,
+        req: &PagestreamNblocksRequest,
+    ) -> Result<PagestreamBeMessage> {
+        let tag = RelishTag::Relation(req.rel);
+        let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest)?;
+
+        let n_blocks = timeline.get_relish_size(tag, lsn)?;
+
+        // Return 0 if relation is not found.
+        // This is what postgres smgr expects.
+        let n_blocks = n_blocks.unwrap_or(0);
+
+        Ok(PagestreamBeMessage::Nblocks(PagestreamNblocksResponse {
+            n_blocks,
+        }))
+    }
+
+    fn handle_get_page_at_lsn_request(
+        &self,
+        timeline: &dyn Timeline,
+        req: &PagestreamGetPageRequest,
+    ) -> Result<PagestreamBeMessage> {
+        let tag = RelishTag::Relation(req.rel);
+        let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest)?;
+
+        let page = timeline.get_page_at_lsn(tag, req.blkno, lsn)?;
+
+        Ok(PagestreamBeMessage::GetPage(PagestreamGetPageResponse {
+            page,
+        }))
+    }
+
    fn handle_basebackup_request(
        &self,
        pgb: &mut PostgresBackend,
@@ -343,30 +413,16 @@ impl PageServerHandler {
        tenantid: ZTenantId,
    ) -> anyhow::Result<()> {
        // check that the timeline exists
-        let repository = page_cache::get_repository_for_tenant(&tenantid)?;
-        let timeline = repository.get_timeline(timelineid).map_err(|e| {
-            error!("error fetching timeline: {:?}", e);
-            anyhow!(
-                "client requested basebackup on timeline {} which does not exist in page server",
-                timelineid
-            )
-        })?;
+        let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)?;
+
        /* switch client to COPYOUT */
        pgb.write_message(&BeMessage::CopyOutResponse)?;
        info!("sent CopyOut");

-        /* Send a tarball of the latest snapshot on the timeline */
-
-        let req_lsn = lsn.unwrap_or_else(|| timeline.get_last_valid_lsn());
-
+        /* Send a tarball of the latest layer on the timeline */
        {
            let mut writer = CopyDataSink { pgb };
-            let mut basebackup = basebackup::Basebackup::new(
-                &mut writer,
-                &timeline,
-                req_lsn,
-                timeline.get_prev_record_lsn(),
-            );
+            let mut basebackup = basebackup::Basebackup::new(&mut writer, &timeline, lsn)?;
            basebackup.send_tarball()?;
        }
        pgb.write_message(&BeMessage::CopyDone)?;
@@ -389,19 +445,7 @@ impl PageServerHandler {
            .claims
            .as_ref()
            .expect("claims presence already checked");
-        match (&claims.scope, tenantid) {
-            (Scope::Tenant, None) => {
-                bail!("Attempt to access management api with tenant scope. Permission denied")
-            }
-            (Scope::Tenant, Some(tenantid)) => {
-                if claims.tenant_id.unwrap() != tenantid {
-                    bail!("Tenant id mismatch. Permission denied")
-                }
-                Ok(())
-            }
-            (Scope::PageServerApi, None) => Ok(()), // access to management api for PageServerApi scope
-            (Scope::PageServerApi, Some(_)) => Ok(()), // access to tenant api using PageServerApi scope
-        }
+        auth::check_permission(claims, tenantid)
    }
 }

@@ -418,7 +462,7 @@ impl postgres_backend::Handler for PageServerHandler {
            .as_ref()
            .as_ref()
            .unwrap()
-            .decode(&str::from_utf8(jwt_response)?)?;
+            .decode(str::from_utf8(jwt_response)?)?;

        if matches!(data.claims.scope, Scope::Tenant) {
            ensure!(
@@ -450,11 +494,9 @@ impl postgres_backend::Handler for PageServerHandler {
        }
        let query_string = std::str::from_utf8(&query_string)?;

-        if query_string.starts_with("controlfile") {
-            self.handle_controlfile(pgb)?;
-        } else if query_string.starts_with("pagestream ") {
+        if query_string.starts_with("pagestream ") {
            let (_, params_raw) = query_string.split_at("pagestream ".len());
-            let params = params_raw.split(" ").collect::<Vec<_>>();
+            let params = params_raw.split(' ').collect::<Vec<_>>();
            ensure!(
                params.len() == 2,
                "invalid param number for pagestream command"
@@ -467,9 +509,10 @@ impl postgres_backend::Handler for PageServerHandler {
            self.handle_pagerequests(pgb, timelineid, tenantid)?;
        } else if query_string.starts_with("basebackup ") {
            let (_, params_raw) = query_string.split_at("basebackup ".len());
-            let params = params_raw.split(" ").collect::<Vec<_>>();
+            let params = params_raw.split_whitespace().collect::<Vec<_>>();
+
            ensure!(
-                params.len() == 2,
+                params.len() >= 2,
                "invalid param number for basebackup command"
            );

@@ -478,12 +521,12 @@ impl postgres_backend::Handler for PageServerHandler {

            self.check_permission(Some(tenantid))?;

-            // TODO are there any tests with lsn option?
            let lsn = if params.len() == 3 {
                Some(Lsn::from_str(params[2])?)
            } else {
                None
            };
+
            info!(
                "got basebackup command. tenantid=\"{}\" timelineid=\"{}\" lsn=\"{:#?}\"",
                tenantid, timelineid, lsn
@@ -507,12 +550,9 @@ impl postgres_backend::Handler for PageServerHandler {
            self.check_permission(Some(tenantid))?;

            // Check that the timeline exists
-            let repository = page_cache::get_repository_for_tenant(&tenantid)?;
-            if repository.get_timeline(timelineid).is_err() {
-                bail!("client requested callmemaybe on timeline {} which does not exist in page server", timelineid);
-            }
+            tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)?;

-            walreceiver::launch_wal_receiver(&self.conf, timelineid, &connstr, tenantid.to_owned());
+            walreceiver::launch_wal_receiver(self.conf, timelineid, &connstr, tenantid.to_owned());

            pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
        } else if query_string.starts_with("branch_create ") {
@@ -520,10 +560,10 @@ impl postgres_backend::Handler for PageServerHandler {

            // branch_create <tenantid> <branchname> <startpoint>
            // TODO lazy static
-            // TOOD: escaping, to allow branch names with spaces
+            // TODO: escaping, to allow branch names with spaces
            let re = Regex::new(r"^branch_create ([[:xdigit:]]+) (\S+) ([^\r\n\s;]+)[\r\n\s;]*;?$")
                .unwrap();
-            let caps = re.captures(&query_string).ok_or_else(err)?;
+            let caps = re.captures(query_string).ok_or_else(err)?;

            let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;
            let branchname = caps.get(2).ok_or_else(err)?.as_str().to_owned();
@@ -532,86 +572,12 @@ impl postgres_backend::Handler for PageServerHandler {
            self.check_permission(Some(tenantid))?;

            let branch =
-                branches::create_branch(&self.conf, &branchname, &startpoint_str, &tenantid)?;
+                branches::create_branch(self.conf, &branchname, &startpoint_str, &tenantid)?;
            let branch = serde_json::to_vec(&branch)?;

            pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
                .write_message_noflush(&BeMessage::DataRow(&[Some(&branch)]))?
                .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
-        } else if query_string.starts_with("push ") {
-            // push <zenith tenantid as hex string> <zenith timelineid as hex string>
-            let re = Regex::new(r"^push ([[:xdigit:]]+) ([[:xdigit:]]+)$").unwrap();
-
-            let caps = re
-                .captures(query_string)
-                .ok_or_else(|| anyhow!("invalid push: '{}'", query_string))?;
-
-            let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;
-            let timelineid = ZTimelineId::from_str(caps.get(2).unwrap().as_str())?;
-
-            self.check_permission(Some(tenantid))?;
-
-            let start_lsn = Lsn(0); // TODO this needs to come from the repo
-            let timeline = page_cache::get_repository_for_tenant(&tenantid)?
-                .create_empty_timeline(timelineid, start_lsn)?;
-
-            pgb.write_message(&BeMessage::CopyInResponse)?;
-
-            let mut last_lsn = Lsn(0);
-
-            while let Some(msg) = pgb.read_message()? {
-                match msg {
-                    FeMessage::CopyData(bytes) => {
-                        let modification = Modification::des(&bytes)?;
-
-                        last_lsn = modification.lsn;
-                        timeline.put_raw_data(
-                            modification.tag,
-                            modification.lsn,
-                            &modification.data,
-                        )?;
-                    }
-                    FeMessage::CopyDone => {
-                        timeline.advance_last_valid_lsn(last_lsn);
-                        break;
-                    }
-                    FeMessage::Sync => {}
-                    _ => bail!("unexpected message {:?}", msg),
-                }
-            }
-
-            pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
-        } else if query_string.starts_with("request_push ") {
-            // request_push <zenith tenantid as hex string> <zenith timelineid as hex string> <postgres_connection_uri>
-            let re = Regex::new(r"^request_push ([[:xdigit:]]+) ([[:xdigit:]]+) (.*)$").unwrap();
-
-            let caps = re
-                .captures(query_string)
-                .ok_or_else(|| anyhow!("invalid request_push: '{}'", query_string))?;
-
-            let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;
-            let timelineid = ZTimelineId::from_str(caps.get(2).unwrap().as_str())?;
-            let postgres_connection_uri = caps.get(3).unwrap().as_str();
-
-            self.check_permission(Some(tenantid))?;
-
-            let timeline =
-                page_cache::get_repository_for_tenant(&tenantid)?.get_timeline(timelineid)?;
-
-            let mut conn = postgres::Client::connect(postgres_connection_uri, postgres::NoTls)?;
-            let mut copy_in = conn.copy_in(format!("push {}", timelineid.to_string()).as_str())?;
-
-            let history = timeline.history()?;
-            for update_res in history {
-                let update = update_res?;
-                let update_bytes = update.ser()?;
-                copy_in.write_all(&update_bytes)?;
-                copy_in.flush()?; // ensure that messages are sent inside individual CopyData packets
-            }
-
-            copy_in.finish()?;
-
-            pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
        } else if query_string.starts_with("branch_list ") {
            // branch_list <zenith tenantid as hex string>
            let re = Regex::new(r"^branch_list ([[:xdigit:]]+)$").unwrap();
@@ -621,14 +587,14 @@ impl postgres_backend::Handler for PageServerHandler {

            let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;

-            let branches = crate::branches::get_branches(&self.conf, &tenantid)?;
+            let branches = crate::branches::get_branches(self.conf, &tenantid)?;
            let branches_buf = serde_json::to_vec(&branches)?;

            pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
                .write_message_noflush(&BeMessage::DataRow(&[Some(&branches_buf)]))?
                .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
        } else if query_string.starts_with("tenant_list") {
-            let tenants = crate::branches::get_tenants(&self.conf)?;
+            let tenants = crate::branches::get_tenants(self.conf)?;
            let tenants_buf = serde_json::to_vec(&tenants)?;

            pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
@@ -639,13 +605,13 @@ impl postgres_backend::Handler for PageServerHandler {

            // tenant_create <tenantid>
            let re = Regex::new(r"^tenant_create ([[:xdigit:]]+)$").unwrap();
-            let caps = re.captures(&query_string).ok_or_else(err)?;
+            let caps = re.captures(query_string).ok_or_else(err)?;

            self.check_permission(None)?;

            let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;

-            page_cache::create_repository_for_tenant(&self.conf, tenantid)?;
+            tenant_mgr::create_repository_for_tenant(self.conf, tenantid)?;

            pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
                .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
@@ -679,82 +645,59 @@ impl postgres_backend::Handler for PageServerHandler {
                .map(|h| h.as_str().parse())
                .unwrap_or(Ok(self.conf.gc_horizon))?;

-            let repo = page_cache::get_repository_for_tenant(&tenantid)?;
+            let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;

            let result = repo.gc_iteration(Some(timelineid), gc_horizon, true)?;

            pgb.write_message_noflush(&BeMessage::RowDescription(&[
-                RowDescriptor::int8_col(b"n_relations"),
-                RowDescriptor::int8_col(b"truncated"),
-                RowDescriptor::int8_col(b"deleted"),
-                RowDescriptor::int8_col(b"prep_deleted"),
-                RowDescriptor::int8_col(b"slru_deleted"),
-                RowDescriptor::int8_col(b"chkp_deleted"),
-                RowDescriptor::int8_col(b"control_deleted"),
-                RowDescriptor::int8_col(b"filenodemap_deleted"),
-                RowDescriptor::int8_col(b"dropped"),
-                RowDescriptor::int8_col(b"snapshot_relfiles_total"),
-                RowDescriptor::int8_col(b"snapshot_relfiles_needed_by_cutoff"),
-                RowDescriptor::int8_col(b"snapshot_relfiles_needed_by_branches"),
-                RowDescriptor::int8_col(b"snapshot_relfiles_not_updated"),
-                RowDescriptor::int8_col(b"snapshot_relfiles_removed"),
-                RowDescriptor::int8_col(b"snapshot_relfiles_dropped"),
-                RowDescriptor::int8_col(b"snapshot_nonrelfiles_total"),
-                RowDescriptor::int8_col(b"snapshot_nonrelfiles_needed_by_cutoff"),
-                RowDescriptor::int8_col(b"snapshot_nonrelfiles_needed_by_branches"),
-                RowDescriptor::int8_col(b"snapshot_nonrelfiles_not_updated"),
-                RowDescriptor::int8_col(b"snapshot_nonrelfiles_removed"),
-                RowDescriptor::int8_col(b"snapshot_nonrelfiles_dropped"),
+                RowDescriptor::int8_col(b"layer_relfiles_total"),
+                RowDescriptor::int8_col(b"layer_relfiles_needed_by_cutoff"),
+                RowDescriptor::int8_col(b"layer_relfiles_needed_by_branches"),
+                RowDescriptor::int8_col(b"layer_relfiles_not_updated"),
+                RowDescriptor::int8_col(b"layer_relfiles_removed"),
+                RowDescriptor::int8_col(b"layer_relfiles_dropped"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_total"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_needed_by_cutoff"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_needed_by_branches"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_not_updated"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_removed"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_dropped"),
                RowDescriptor::int8_col(b"elapsed"),
            ]))?
            .write_message_noflush(&BeMessage::DataRow(&[
-                Some(&result.n_relations.to_string().as_bytes()),
-                Some(&result.truncated.to_string().as_bytes()),
-                Some(&result.deleted.to_string().as_bytes()),
-                Some(&result.prep_deleted.to_string().as_bytes()),
-                Some(&result.slru_deleted.to_string().as_bytes()),
-                Some(&result.chkp_deleted.to_string().as_bytes()),
-                Some(&result.control_deleted.to_string().as_bytes()),
-                Some(&result.filenodemap_deleted.to_string().as_bytes()),
-                Some(&result.dropped.to_string().as_bytes()),
-                Some(&result.snapshot_relfiles_total.to_string().as_bytes()),
+                Some(result.ondisk_relfiles_total.to_string().as_bytes()),
                Some(
-                    &result
-                        .snapshot_relfiles_needed_by_cutoff
+                    result
+                        .ondisk_relfiles_needed_by_cutoff
                        .to_string()
                        .as_bytes(),
                ),
                Some(
-                    &result
-                        .snapshot_relfiles_needed_by_branches
+                    result
+                        .ondisk_relfiles_needed_by_branches
                        .to_string()
                        .as_bytes(),
                ),
-                Some(&result.snapshot_relfiles_not_updated.to_string().as_bytes()),
-                Some(&result.snapshot_relfiles_removed.to_string().as_bytes()),
-                Some(&result.snapshot_relfiles_dropped.to_string().as_bytes()),
-                Some(&result.snapshot_nonrelfiles_total.to_string().as_bytes()),
+                Some(result.ondisk_relfiles_not_updated.to_string().as_bytes()),
+                Some(result.ondisk_relfiles_removed.to_string().as_bytes()),
+                Some(result.ondisk_relfiles_dropped.to_string().as_bytes()),
+                Some(result.ondisk_nonrelfiles_total.to_string().as_bytes()),
                Some(
-                    &result
-                        .snapshot_nonrelfiles_needed_by_cutoff
+                    result
+                        .ondisk_nonrelfiles_needed_by_cutoff
                        .to_string()
                        .as_bytes(),
                ),
                Some(
-                    &result
-                        .snapshot_nonrelfiles_needed_by_branches
+                    result
+                        .ondisk_nonrelfiles_needed_by_branches
                        .to_string()
                        .as_bytes(),
                ),
-                Some(
-                    &result
-                        .snapshot_nonrelfiles_not_updated
-                        .to_string()
-                        .as_bytes(),
-                ),
-                Some(&result.snapshot_nonrelfiles_removed.to_string().as_bytes()),
-                Some(&result.snapshot_nonrelfiles_dropped.to_string().as_bytes()),
-                Some(&result.elapsed.as_millis().to_string().as_bytes()),
+                Some(result.ondisk_nonrelfiles_not_updated.to_string().as_bytes()),
+                Some(result.ondisk_nonrelfiles_removed.to_string().as_bytes()),
+                Some(result.ondisk_nonrelfiles_dropped.to_string().as_bytes()),
+                Some(result.elapsed.as_millis().to_string().as_bytes()),
            ]))?
            .write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;
        } else {
--- a/pageserver/src/relish.rs
+++ b/pageserver/src/relish.rs
@@ -125,11 +125,7 @@ impl RelishTag {

    // convenience function to check if this relish is a normal relation.
    pub const fn is_relation(&self) -> bool {
-        if let RelishTag::Relation(_) = self {
-            true
-        } else {
-            false
-        }
+        matches!(self, RelishTag::Relation(_))
    }
 }

--- a/pageserver/src/relish_storage.rs
+++ b/pageserver/src/relish_storage.rs
@@ -0,0 +1,54 @@
+//! Abstractions for the page server to store its relish layer data in the external storage.
+//!
+//! Main purpose of this module subtree is to provide a set of abstractions to manage the storage state
+//! in a way, optimal for page server.
+//!
+//! The abstractions hide multiple custom external storage API implementations,
+//! such as AWS S3, local filesystem, etc., located in the submodules.
+
+mod local_fs;
+mod rust_s3;
+/// A queue and the background machinery behind it to upload
+/// local page server layer files to external storage.
+pub mod storage_uploader;
+
+use std::path::Path;
+
+use anyhow::Context;
+
+/// Storage (potentially remote) API to manage its state.
+#[async_trait::async_trait]
+pub trait RelishStorage: Send + Sync {
+    type RelishStoragePath;
+
+    fn derive_destination(
+        page_server_workdir: &Path,
+        relish_local_path: &Path,
+    ) -> anyhow::Result<Self::RelishStoragePath>;
+
+    async fn list_relishes(&self) -> anyhow::Result<Vec<Self::RelishStoragePath>>;
+
+    async fn download_relish(
+        &self,
+        from: &Self::RelishStoragePath,
+        to: &Path,
+    ) -> anyhow::Result<()>;
+
+    async fn delete_relish(&self, path: &Self::RelishStoragePath) -> anyhow::Result<()>;
+
+    async fn upload_relish(&self, from: &Path, to: &Self::RelishStoragePath) -> anyhow::Result<()>;
+}
+
+fn strip_workspace_prefix<'a>(
+    page_server_workdir: &'a Path,
+    relish_local_path: &'a Path,
+) -> anyhow::Result<&'a Path> {
+    relish_local_path
+        .strip_prefix(page_server_workdir)
+        .with_context(|| {
+            format!(
+                "Unexpected: relish local path '{}' is not relevant to server workdir",
+                relish_local_path.display(),
+            )
+        })
+}
--- a/pageserver/src/relish_storage/local_fs.rs
+++ b/pageserver/src/relish_storage/local_fs.rs
@@ -0,0 +1,158 @@
+//! Local filesystem relish storage.
+//!
+//! Page server already stores layer data on the server, when freezing it.
+//! This storage serves a way to
+//!
+//! * test things locally simply
+//! * allow to compabre both binary sets
+//! * help validating the relish storage API
+
+use std::{
+    future::Future,
+    path::{Path, PathBuf},
+    pin::Pin,
+};
+
+use anyhow::{bail, Context};
+
+use super::{strip_workspace_prefix, RelishStorage};
+
+pub struct LocalFs {
+    root: PathBuf,
+}
+
+impl LocalFs {
+    /// Atetmpts to create local FS relish storage, also creates the directory provided, if not exists.
+    pub fn new(root: PathBuf) -> anyhow::Result<Self> {
+        if !root.exists() {
+            std::fs::create_dir_all(&root).with_context(|| {
+                format!(
+                    "Failed to create all directories in the given root path {}",
+                    root.display(),
+                )
+            })?;
+        }
+        Ok(Self { root })
+    }
+
+    fn resolve_in_storage(&self, path: &Path) -> anyhow::Result<PathBuf> {
+        if path.is_relative() {
+            Ok(self.root.join(path))
+        } else if path.starts_with(&self.root) {
+            Ok(path.to_path_buf())
+        } else {
+            bail!(
+                "Path '{}' does not belong to the current storage",
+                path.display()
+            )
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl RelishStorage for LocalFs {
+    type RelishStoragePath = PathBuf;
+
+    fn derive_destination(
+        page_server_workdir: &Path,
+        relish_local_path: &Path,
+    ) -> anyhow::Result<Self::RelishStoragePath> {
+        Ok(strip_workspace_prefix(page_server_workdir, relish_local_path)?.to_path_buf())
+    }
+
+    async fn list_relishes(&self) -> anyhow::Result<Vec<Self::RelishStoragePath>> {
+        Ok(get_all_files(&self.root).await?.into_iter().collect())
+    }
+
+    async fn download_relish(
+        &self,
+        from: &Self::RelishStoragePath,
+        to: &Path,
+    ) -> anyhow::Result<()> {
+        let file_path = self.resolve_in_storage(from)?;
+        if file_path.exists() && file_path.is_file() {
+            create_target_directory(to).await?;
+            tokio::fs::copy(file_path, to).await?;
+            Ok(())
+        } else {
+            bail!(
+                "File '{}' either does not exist or is not a file",
+                file_path.display()
+            )
+        }
+    }
+
+    async fn delete_relish(&self, path: &Self::RelishStoragePath) -> anyhow::Result<()> {
+        let file_path = self.resolve_in_storage(path)?;
+        if file_path.exists() && file_path.is_file() {
+            Ok(tokio::fs::remove_file(file_path).await?)
+        } else {
+            bail!(
+                "File '{}' either does not exist or is not a file",
+                file_path.display()
+            )
+        }
+    }
+
+    async fn upload_relish(&self, from: &Path, to: &Self::RelishStoragePath) -> anyhow::Result<()> {
+        let target_file_path = self.resolve_in_storage(to)?;
+        create_target_directory(&target_file_path).await?;
+
+        tokio::fs::copy(&from, &target_file_path)
+            .await
+            .with_context(|| {
+                format!(
+                    "Failed to upload relish '{}' to local storage",
+                    from.display(),
+                )
+            })?;
+        Ok(())
+    }
+}
+
+fn get_all_files<'a, P>(
+    directory_path: P,
+) -> Pin<Box<dyn Future<Output = anyhow::Result<Vec<PathBuf>>> + Send + Sync + 'a>>
+where
+    P: AsRef<Path> + Send + Sync + 'a,
+{
+    Box::pin(async move {
+        let directory_path = directory_path.as_ref();
+        if directory_path.exists() {
+            if directory_path.is_dir() {
+                let mut paths = Vec::new();
+                let mut dir_contents = tokio::fs::read_dir(directory_path).await?;
+                while let Some(dir_entry) = dir_contents.next_entry().await? {
+                    let file_type = dir_entry.file_type().await?;
+                    let entry_path = dir_entry.path();
+                    if file_type.is_symlink() {
+                        log::debug!("{:?} us a symlink, skipping", entry_path)
+                    } else if file_type.is_dir() {
+                        paths.extend(get_all_files(entry_path).await?.into_iter())
+                    } else {
+                        paths.push(dir_entry.path());
+                    }
+                }
+                Ok(paths)
+            } else {
+                bail!("Path '{}' is not a directory", directory_path.display())
+            }
+        } else {
+            Ok(Vec::new())
+        }
+    })
+}
+
+async fn create_target_directory(target_file_path: &Path) -> anyhow::Result<()> {
+    let target_dir = match target_file_path.parent() {
+        Some(parent_dir) => parent_dir,
+        None => bail!(
+            "Relish path '{}' has no parent directory",
+            target_file_path.display()
+        ),
+    };
+    if !target_dir.exists() {
+        tokio::fs::create_dir_all(target_dir).await?;
+    }
+    Ok(())
+}
--- a/pageserver/src/relish_storage/rust_s3.rs
+++ b/pageserver/src/relish_storage/rust_s3.rs
@@ -0,0 +1,144 @@
+//! A wrapper around AWS S3 client library `rust_s3` to be used a relish storage.
+
+use std::path::Path;
+
+use anyhow::Context;
+use s3::{bucket::Bucket, creds::Credentials, region::Region};
+
+use crate::{relish_storage::strip_workspace_prefix, S3Config};
+
+use super::RelishStorage;
+
+const S3_FILE_SEPARATOR: char = '/';
+
+#[derive(Debug)]
+pub struct S3ObjectKey(String);
+
+impl S3ObjectKey {
+    fn key(&self) -> &str {
+        &self.0
+    }
+}
+
+/// AWS S3 relish storage.
+pub struct RustS3 {
+    bucket: Bucket,
+}
+
+impl RustS3 {
+    /// Creates the relish storage, errors if incorrect AWS S3 configuration provided.
+    pub fn new(aws_config: &S3Config) -> anyhow::Result<Self> {
+        let region = aws_config
+            .bucket_region
+            .parse::<Region>()
+            .context("Failed to parse the s3 region from config")?;
+        let credentials = Credentials::new(
+            aws_config.access_key_id.as_deref(),
+            aws_config.secret_access_key.as_deref(),
+            None,
+            None,
+            None,
+        )
+        .context("Failed to create the s3 credentials")?;
+        Ok(Self {
+            bucket: Bucket::new_with_path_style(
+                aws_config.bucket_name.as_str(),
+                region,
+                credentials,
+            )
+            .context("Failed to create the s3 bucket")?,
+        })
+    }
+}
+
+#[async_trait::async_trait]
+impl RelishStorage for RustS3 {
+    type RelishStoragePath = S3ObjectKey;
+
+    fn derive_destination(
+        page_server_workdir: &Path,
+        relish_local_path: &Path,
+    ) -> anyhow::Result<Self::RelishStoragePath> {
+        let relative_path = strip_workspace_prefix(page_server_workdir, relish_local_path)?;
+        let mut key = String::new();
+        for segment in relative_path {
+            key.push(S3_FILE_SEPARATOR);
+            key.push_str(&segment.to_string_lossy());
+        }
+        Ok(S3ObjectKey(key))
+    }
+
+    async fn list_relishes(&self) -> anyhow::Result<Vec<Self::RelishStoragePath>> {
+        let list_response = self
+            .bucket
+            .list(String::new(), None)
+            .await
+            .context("Failed to list s3 objects")?;
+
+        Ok(list_response
+            .into_iter()
+            .flat_map(|response| response.contents)
+            .map(|s3_object| S3ObjectKey(s3_object.key))
+            .collect())
+    }
+
+    async fn download_relish(
+        &self,
+        from: &Self::RelishStoragePath,
+        to: &Path,
+    ) -> anyhow::Result<()> {
+        let mut target_file = std::fs::OpenOptions::new()
+            .write(true)
+            .open(to)
+            .with_context(|| format!("Failed to open target s3 destination at {}", to.display()))?;
+        let code = self
+            .bucket
+            .get_object_stream(from.key(), &mut target_file)
+            .await
+            .with_context(|| format!("Failed to download s3 object with key {}", from.key()))?;
+        if code != 200 {
+            Err(anyhow::format_err!(
+                "Received non-200 exit code during downloading object from directory, code: {}",
+                code
+            ))
+        } else {
+            Ok(())
+        }
+    }
+
+    async fn delete_relish(&self, path: &Self::RelishStoragePath) -> anyhow::Result<()> {
+        let (_, code) = self
+            .bucket
+            .delete_object(path.key())
+            .await
+            .with_context(|| format!("Failed to delete s3 object with key {}", path.key()))?;
+        if code != 200 {
+            Err(anyhow::format_err!(
+                "Received non-200 exit code during deleting object with key '{}', code: {}",
+                path.key(),
+                code
+            ))
+        } else {
+            Ok(())
+        }
+    }
+
+    async fn upload_relish(&self, from: &Path, to: &Self::RelishStoragePath) -> anyhow::Result<()> {
+        let mut local_file = tokio::fs::OpenOptions::new().read(true).open(from).await?;
+
+        let code = self
+            .bucket
+            .put_object_stream(&mut local_file, to.key())
+            .await
+            .with_context(|| format!("Failed to create s3 object with key {}", to.key()))?;
+        if code != 200 {
+            Err(anyhow::format_err!(
+                "Received non-200 exit code during creating object with key '{}', code: {}",
+                to.key(),
+                code
+            ))
+        } else {
+            Ok(())
+        }
+    }
+}
--- a/pageserver/src/relish_storage/storage_uploader.rs
+++ b/pageserver/src/relish_storage/storage_uploader.rs
@@ -0,0 +1,116 @@
+use std::{
+    collections::VecDeque,
+    path::{Path, PathBuf},
+    sync::{Arc, Mutex},
+    thread,
+};
+
+use zenith_utils::zid::ZTimelineId;
+
+use crate::{relish_storage::RelishStorage, RelishStorageConfig};
+
+use super::{local_fs::LocalFs, rust_s3::RustS3};
+
+pub struct QueueBasedRelishUploader {
+    upload_queue: Arc<Mutex<VecDeque<(ZTimelineId, PathBuf)>>>,
+}
+
+impl QueueBasedRelishUploader {
+    pub fn new(
+        config: &RelishStorageConfig,
+        page_server_workdir: &'static Path,
+    ) -> anyhow::Result<Self> {
+        let upload_queue = Arc::new(Mutex::new(VecDeque::new()));
+        let _handle = match config {
+            RelishStorageConfig::LocalFs(root) => {
+                let relish_storage = LocalFs::new(root.clone())?;
+                create_upload_thread(
+                    Arc::clone(&upload_queue),
+                    relish_storage,
+                    page_server_workdir,
+                )?
+            }
+            RelishStorageConfig::AwsS3(s3_config) => {
+                let relish_storage = RustS3::new(s3_config)?;
+                create_upload_thread(
+                    Arc::clone(&upload_queue),
+                    relish_storage,
+                    page_server_workdir,
+                )?
+            }
+        };
+
+        Ok(Self { upload_queue })
+    }
+
+    pub fn schedule_upload(&self, timeline_id: ZTimelineId, relish_path: PathBuf) {
+        self.upload_queue
+            .lock()
+            .unwrap()
+            .push_back((timeline_id, relish_path))
+    }
+}
+
+fn create_upload_thread<P, S: 'static + RelishStorage<RelishStoragePath = P>>(
+    upload_queue: Arc<Mutex<VecDeque<(ZTimelineId, PathBuf)>>>,
+    relish_storage: S,
+    page_server_workdir: &'static Path,
+) -> std::io::Result<thread::JoinHandle<()>> {
+    let runtime = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()?;
+    thread::Builder::new()
+        .name("Queue based relish uploader".to_string())
+        .spawn(move || loop {
+            runtime.block_on(async {
+                upload_loop_step(&upload_queue, &relish_storage, page_server_workdir).await;
+            })
+        })
+}
+
+async fn upload_loop_step<P, S: 'static + RelishStorage<RelishStoragePath = P>>(
+    upload_queue: &Mutex<VecDeque<(ZTimelineId, PathBuf)>>,
+    relish_storage: &S,
+    page_server_workdir: &Path,
+) {
+    let mut queue_accessor = upload_queue.lock().unwrap();
+    log::debug!("current upload queue length: {}", queue_accessor.len());
+    let next_upload = queue_accessor.pop_front();
+    drop(queue_accessor);
+
+    let (relish_timeline_id, relish_local_path) = match next_upload {
+        Some(data) => data,
+        None => {
+            // Don't spin and allow others to use the queue.
+            // In future, could be improved to be more clever about delays depending on relish upload stats
+            thread::sleep(std::time::Duration::from_secs(1));
+            return;
+        }
+    };
+
+    if let Err(e) = upload_relish(relish_storage, page_server_workdir, &relish_local_path).await {
+        log::error!(
+            "Failed to upload relish '{}' for timeline {}, reason: {}",
+            relish_local_path.display(),
+            relish_timeline_id,
+            e
+        );
+        upload_queue
+            .lock()
+            .unwrap()
+            .push_back((relish_timeline_id, relish_local_path))
+    } else {
+        log::debug!("Relish successfully uploaded");
+    }
+}
+
+async fn upload_relish<P, S: RelishStorage<RelishStoragePath = P>>(
+    relish_storage: &S,
+    page_server_workdir: &Path,
+    relish_local_path: &Path,
+) -> anyhow::Result<()> {
+    let destination = S::derive_destination(page_server_workdir, relish_local_path)?;
+    relish_storage
+        .upload_relish(relish_local_path, &destination)
+        .await
+}
--- a/pageserver/src/repository.rs
+++ b/pageserver/src/repository.rs
@@ -1,14 +1,12 @@
-use crate::object_key::*;
 use crate::relish::*;
 use anyhow::Result;
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
-use std::iter::Iterator;
 use std::ops::AddAssign;
 use std::sync::Arc;
 use std::time::Duration;
-use zenith_utils::lsn::Lsn;
+use zenith_utils::lsn::{Lsn, RecordLsn};
 use zenith_utils::zid::ZTimelineId;

 ///
@@ -19,11 +17,7 @@ pub trait Repository: Send + Sync {
    fn get_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>>;

    /// Create a new, empty timeline. The caller is responsible for loading data into it
-    fn create_empty_timeline(
-        &self,
-        timelineid: ZTimelineId,
-        start_lsn: Lsn,
-    ) -> Result<Arc<dyn Timeline>>;
+    fn create_empty_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>>;

    /// Branch a timeline
    fn branch_timeline(&self, src: ZTimelineId, dst: ZTimelineId, start_lsn: Lsn) -> Result<()>;
@@ -47,9 +41,6 @@ pub trait Repository: Send + Sync {
        horizon: u64,
        compact: bool,
    ) -> Result<GcResult>;
-
-    // TODO get timelines?
-    //fn get_stats(&self) -> RepositoryStats;
 }

 ///
@@ -57,58 +48,38 @@ pub trait Repository: Send + Sync {
 ///
 #[derive(Default)]
 pub struct GcResult {
-    // FIXME: These counters make sense for the ObjectRepository. They are not used
-    // by the LayeredRepository.
-    pub n_relations: u64,
-    pub inspected: u64,
-    pub truncated: u64,
-    pub deleted: u64,
-    pub prep_deleted: u64,        // RelishTag::Twophase
-    pub slru_deleted: u64,        // RelishTag::Slru
-    pub chkp_deleted: u64,        // RelishTag::Checkpoint
-    pub control_deleted: u64,     // RelishTag::ControlFile
-    pub filenodemap_deleted: u64, // RelishTag::FileNodeMap
-    pub dropped: u64,
+    pub ondisk_relfiles_total: u64,
+    pub ondisk_relfiles_needed_by_cutoff: u64,
+    pub ondisk_relfiles_needed_by_branches: u64,
+    pub ondisk_relfiles_not_updated: u64,
+    pub ondisk_relfiles_removed: u64, // # of layer files removed because they have been made obsolete by newer ondisk files.
+    pub ondisk_relfiles_dropped: u64, // # of layer files removed because the relation was dropped

-    // These are used for the LayeredRepository instead
-    pub snapshot_relfiles_total: u64,
-    pub snapshot_relfiles_needed_by_cutoff: u64,
-    pub snapshot_relfiles_needed_by_branches: u64,
-    pub snapshot_relfiles_not_updated: u64,
-    pub snapshot_relfiles_removed: u64, // # of snapshot files removed because they have been made obsolete by newer snapshot files.
-    pub snapshot_relfiles_dropped: u64, // # of snapshot files removed because the relation was dropped
-
-    pub snapshot_nonrelfiles_total: u64,
-    pub snapshot_nonrelfiles_needed_by_cutoff: u64,
-    pub snapshot_nonrelfiles_needed_by_branches: u64,
-    pub snapshot_nonrelfiles_not_updated: u64,
-    pub snapshot_nonrelfiles_removed: u64, // # of snapshot files removed because they have been made obsolete by newer snapshot files.
-    pub snapshot_nonrelfiles_dropped: u64, // # of snapshot files removed because the relation was dropped
+    pub ondisk_nonrelfiles_total: u64,
+    pub ondisk_nonrelfiles_needed_by_cutoff: u64,
+    pub ondisk_nonrelfiles_needed_by_branches: u64,
+    pub ondisk_nonrelfiles_not_updated: u64,
+    pub ondisk_nonrelfiles_removed: u64, // # of layer files removed because they have been made obsolete by newer ondisk files.
+    pub ondisk_nonrelfiles_dropped: u64, // # of layer files removed because the relation was dropped

    pub elapsed: Duration,
 }

 impl AddAssign for GcResult {
    fn add_assign(&mut self, other: Self) {
-        self.n_relations += other.n_relations;
-        self.truncated += other.truncated;
-        self.deleted += other.deleted;
-        self.dropped += other.dropped;
+        self.ondisk_relfiles_total += other.ondisk_relfiles_total;
+        self.ondisk_relfiles_needed_by_cutoff += other.ondisk_relfiles_needed_by_cutoff;
+        self.ondisk_relfiles_needed_by_branches += other.ondisk_relfiles_needed_by_branches;
+        self.ondisk_relfiles_not_updated += other.ondisk_relfiles_not_updated;
+        self.ondisk_relfiles_removed += other.ondisk_relfiles_removed;
+        self.ondisk_relfiles_dropped += other.ondisk_relfiles_dropped;

-        self.snapshot_relfiles_total += other.snapshot_relfiles_total;
-        self.snapshot_relfiles_needed_by_cutoff += other.snapshot_relfiles_needed_by_cutoff;
-        self.snapshot_relfiles_needed_by_branches += other.snapshot_relfiles_needed_by_branches;
-        self.snapshot_relfiles_not_updated += other.snapshot_relfiles_not_updated;
-        self.snapshot_relfiles_removed += other.snapshot_relfiles_removed;
-        self.snapshot_relfiles_dropped += other.snapshot_relfiles_dropped;
-
-        self.snapshot_nonrelfiles_total += other.snapshot_nonrelfiles_total;
-        self.snapshot_nonrelfiles_needed_by_cutoff += other.snapshot_nonrelfiles_needed_by_cutoff;
-        self.snapshot_nonrelfiles_needed_by_branches +=
-            other.snapshot_nonrelfiles_needed_by_branches;
-        self.snapshot_nonrelfiles_not_updated += other.snapshot_nonrelfiles_not_updated;
-        self.snapshot_nonrelfiles_removed += other.snapshot_nonrelfiles_removed;
-        self.snapshot_nonrelfiles_dropped += other.snapshot_nonrelfiles_dropped;
+        self.ondisk_nonrelfiles_total += other.ondisk_nonrelfiles_total;
+        self.ondisk_nonrelfiles_needed_by_cutoff += other.ondisk_nonrelfiles_needed_by_cutoff;
+        self.ondisk_nonrelfiles_needed_by_branches += other.ondisk_nonrelfiles_needed_by_branches;
+        self.ondisk_nonrelfiles_not_updated += other.ondisk_nonrelfiles_not_updated;
+        self.ondisk_nonrelfiles_removed += other.ondisk_nonrelfiles_removed;
+        self.ondisk_nonrelfiles_dropped += other.ondisk_nonrelfiles_dropped;

        self.elapsed += other.elapsed;
    }
@@ -119,11 +90,16 @@ pub trait Timeline: Send + Sync {
    // Public GET functions
    //------------------------------------------------------------------------------

-    /// Look up given page in the cache.
-    fn get_page_at_lsn(&self, tag: RelishTag, blknum: u32, lsn: Lsn) -> Result<Bytes>;
+    ///
+    /// Wait until WAL has been received and processed up to this LSN.
+    ///
+    /// You should call this before any of the other get_* or list_* functions. Calling
+    /// those functions with an LSN that has been processed yet is an error.
+    ///
+    fn wait_lsn(&self, lsn: Lsn) -> Result<()>;

-    /// Look up given page in the cache.
-    fn get_page_at_lsn_nowait(&self, tag: RelishTag, blknum: u32, lsn: Lsn) -> Result<Bytes>;
+    /// Look up given page version.
+    fn get_page_at_lsn(&self, tag: RelishTag, blknum: u32, lsn: Lsn) -> Result<Bytes>;

    /// Get size of a relish
    fn get_relish_size(&self, tag: RelishTag, lsn: Lsn) -> Result<Option<u32>>;
@@ -131,11 +107,15 @@ pub trait Timeline: Send + Sync {
    /// Does relation exist?
    fn get_rel_exists(&self, tag: RelishTag, lsn: Lsn) -> Result<bool>;

-    /// Get a list of all distinct relations in given tablespace and database.
-    fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashSet<RelTag>>;
+    /// Get a list of all existing relations
+    /// Pass RelTag to get relation objects or None to get nonrels.
+    fn list_relishes(&self, tag: Option<RelTag>, lsn: Lsn) -> Result<HashSet<RelishTag>>;

-    /// Get a list of non-relational objects
-    fn list_nonrels<'a>(&'a self, lsn: Lsn) -> Result<HashSet<RelishTag>>;
+    /// Get a list of all existing relations in given tablespace and database.
+    fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashSet<RelishTag>>;
+
+    /// Get a list of all existing non-relational objects
+    fn list_nonrels(&self, lsn: Lsn) -> Result<HashSet<RelishTag>>;

    //------------------------------------------------------------------------------
    // Public PUT functions, to update the repository with new page versions.
@@ -150,44 +130,25 @@ pub trait Timeline: Send + Sync {
    fn put_wal_record(&self, tag: RelishTag, blknum: u32, rec: WALRecord) -> Result<()>;

    /// Like put_wal_record, but with ready-made image of the page.
-    fn put_page_image(
-        &self,
-        tag: RelishTag,
-        blknum: u32,
-        lsn: Lsn,
-        img: Bytes,
-        update_meta: bool,
-    ) -> Result<()>;
+    fn put_page_image(&self, tag: RelishTag, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()>;

    /// Truncate relation
    fn put_truncation(&self, rel: RelishTag, lsn: Lsn, nblocks: u32) -> Result<()>;

-    /// Unlink relish.
-    /// This method is used for marking dropped relations and truncated SLRU segments
-    fn put_unlink(&self, tag: RelishTag, lsn: Lsn) -> Result<()>;
+    /// This method is used for marking dropped relations and truncated SLRU files and aborted two phase records
+    fn drop_relish(&self, tag: RelishTag, lsn: Lsn) -> Result<()>;

-    /// Put raw data
-    fn put_raw_data(&self, tag: ObjectTag, lsn: Lsn, data: &[u8]) -> Result<()>;
-
-    /// Remember the all WAL before the given LSN has been processed.
+    /// Track end of the latest digested WAL record.
    ///
-    /// The WAL receiver calls this after the put_* functions, to indicate that
-    /// all WAL before this point has been digested. Before that, if you call
-    /// GET on an earlier LSN, it will block.
-    fn advance_last_valid_lsn(&self, lsn: Lsn);
-    fn get_last_valid_lsn(&self) -> Lsn;
-    fn init_valid_lsn(&self, lsn: Lsn);
-
-    /// Like `advance_last_valid_lsn`, but this always points to the end of
-    /// a WAL record, not in the middle of one.
-    ///
-    /// This must be <= last valid LSN. This is tracked separately from last
-    /// valid LSN, so that the WAL receiver knows where to restart streaming.
+    /// Advance requires aligned LSN as an argument and would wake wait_lsn() callers.
+    /// Previous last record LSN is stored alongside the latest and can be read.
    fn advance_last_record_lsn(&self, lsn: Lsn);
+    /// Atomically get both last and prev.
+    fn get_last_record_rlsn(&self) -> RecordLsn;
+    /// Get last or prev record separately. Same as get_last_record_rlsn().last/prev.
    fn get_last_record_lsn(&self) -> Lsn;
-
-    // Like `advance_last_record_lsn`, but points to the start position of last record
    fn get_prev_record_lsn(&self) -> Lsn;
+    fn get_start_lsn(&self) -> Lsn;

    ///
    /// Flush to disk all data that was written with the put_* functions
@@ -196,47 +157,15 @@ pub trait Timeline: Send + Sync {
    /// know anything about them here in the repository.
    fn checkpoint(&self) -> Result<()>;

-    /// Events for all relations in the timeline.
-    /// Contains updates from start up to the last valid LSN
-    /// at time of history() call. This lsn can be read via the lsn() function.
+    /// Retrieve current logical size of the timeline
    ///
-    /// Relation size is increased implicitly and decreased with Truncate updates.
-    // TODO ordering guarantee?
-    fn history<'a>(&'a self) -> Result<Box<dyn History + 'a>>;
-}
+    /// NOTE: counted incrementally, includes ancestors,
+    /// doesnt support TwoPhase relishes yet
+    fn get_current_logical_size(&self) -> usize;

-pub trait History: Iterator<Item = Result<Modification>> {
-    /// The last_valid_lsn at the time of history() call.
-    fn lsn(&self) -> Lsn;
-}
-
-//
-// Structure representing any update operation of object storage.
-// It is used to copy object storage content in PUSH method.
-//
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Modification {
-    pub tag: ObjectTag,
-    pub lsn: Lsn,
-    pub data: Vec<u8>,
-}
-
-impl Modification {
-    pub fn new(entry: (ObjectTag, Lsn, Vec<u8>)) -> Modification {
-        Modification {
-            tag: entry.0,
-            lsn: entry.1,
-            data: entry.2,
-        }
-    }
-}
-
-#[derive(Clone)]
-pub struct RepositoryStats {
-    pub num_entries: Lsn,
-    pub num_page_images: Lsn,
-    pub num_wal_records: Lsn,
-    pub num_getpage_requests: Lsn,
+    /// Does the same as get_current_logical_size but counted on demand.
+    /// Used in tests to ensure thet incremental and non incremental variants match.
+    fn get_current_logical_size_non_incremental(&self, lsn: Lsn) -> Result<usize>;
 }

 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
@@ -276,22 +205,17 @@ impl WALRecord {
 ///
 /// Tests that should work the same with any Repository/Timeline implementation.
 ///
+#[allow(clippy::bool_assert_comparison)]
 #[cfg(test)]
 mod tests {
    use super::*;
    use crate::layered_repository::LayeredRepository;
-    use crate::object_repository::ObjectRepository;
-    use crate::object_repository::{ObjectValue, PageEntry, RelationSizeEntry};
-    use crate::rocksdb_storage::RocksObjectStore;
    use crate::walredo::{WalRedoError, WalRedoManager};
-    use crate::{PageServerConf, RepositoryFormat};
+    use crate::PageServerConf;
    use postgres_ffi::pg_constants;
+    use postgres_ffi::xlog_utils::SIZEOF_CHECKPOINT;
    use std::fs;
-    use std::path::PathBuf;
    use std::str::FromStr;
-    use std::time::Duration;
-    use zenith_utils::bin_ser::BeSer;
-    use zenith_utils::postgres_backend::AuthType;
    use zenith_utils::zid::ZTenantId;

    /// Arbitrary relation tag, for testing.
@@ -318,30 +242,24 @@ mod tests {
        buf.freeze()
    }

-    static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
+    fn assert_current_logical_size(timeline: &Arc<dyn Timeline>, lsn: Lsn) {
+        let incremental = timeline.get_current_logical_size();
+        let non_incremental = timeline
+            .get_current_logical_size_non_incremental(lsn)
+            .unwrap();
+        assert_eq!(incremental, non_incremental);
+    }

-    fn get_test_repo(
-        test_name: &str,
-        repository_format: RepositoryFormat,
-    ) -> Result<Box<dyn Repository>> {
-        let repo_dir = PathBuf::from(format!("../tmp_check/test_{}", test_name));
+    static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
+    static ZERO_CHECKPOINT: Bytes = Bytes::from_static(&[0u8; SIZEOF_CHECKPOINT]);
+
+    fn get_test_repo(test_name: &str) -> Result<Box<dyn Repository>> {
+        let repo_dir = PageServerConf::test_repo_dir(test_name);
        let _ = fs::remove_dir_all(&repo_dir);
        fs::create_dir_all(&repo_dir)?;
        fs::create_dir_all(&repo_dir.join("timelines"))?;

-        let conf = PageServerConf {
-            daemonize: false,
-            gc_horizon: 64 * 1024 * 1024,
-            gc_period: Duration::from_secs(10),
-            listen_addr: "127.0.0.1:5430".to_string(),
-            http_endpoint_addr: "127.0.0.1:9898".to_string(),
-            superuser: "zenith_admin".to_string(),
-            workdir: repo_dir,
-            pg_distrib_dir: "".into(),
-            auth_type: AuthType::Trust,
-            auth_validation_public_key_path: None,
-            repository_format,
-        };
+        let conf = PageServerConf::dummy_conf(repo_dir);
        // Make a static copy of the config. This can never be free'd, but that's
        // OK in a test.
        let conf: &'static PageServerConf = Box::leak(Box::new(conf));
@@ -350,153 +268,150 @@ mod tests {

        let walredo_mgr = TestRedoManager {};

-        let repo: Box<dyn Repository + Sync + Send> = match conf.repository_format {
-            RepositoryFormat::Layered => Box::new(LayeredRepository::new(
-                conf,
-                Arc::new(walredo_mgr),
-                tenantid,
-            )),
-            RepositoryFormat::RocksDb => {
-                let obj_store = RocksObjectStore::create(conf, &tenantid)?;
-
-                Box::new(ObjectRepository::new(
-                    conf,
-                    Arc::new(obj_store),
-                    Arc::new(walredo_mgr),
-                    tenantid,
-                ))
-            }
-        };
+        let repo = Box::new(LayeredRepository::new(
+            conf,
+            Arc::new(walredo_mgr),
+            tenantid,
+        ));

        Ok(repo)
    }

-    /// Test get_relsize() and truncation.
    #[test]
-    fn test_relsize_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_relsize_rocksdb", RepositoryFormat::RocksDb)?;
-        test_relsize(&*repo)
-    }
-
-    #[test]
-    fn test_relsize_layered() -> Result<()> {
-        let repo = get_test_repo("test_relsize_layered", RepositoryFormat::Layered)?;
-        test_relsize(&*repo)
-    }
-
-    fn test_relsize(repo: &dyn Repository) -> Result<()> {
+    fn test_relsize() -> Result<()> {
+        let repo = get_test_repo("test_relsize")?;
        // get_timeline() with non-existent timeline id should fail
        //repo.get_timeline("11223344556677881122334455667788");

        // Create timeline to work on
        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
+        let tline = repo.create_empty_timeline(timelineid)?;

-        tline.init_valid_lsn(Lsn(1));
-        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"), true)?;
-        tline.put_page_image(TESTREL_A, 1, Lsn(4), TEST_IMG("foo blk 1 at 4"), true)?;
-        tline.put_page_image(TESTREL_A, 2, Lsn(5), TEST_IMG("foo blk 2 at 5"), true)?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(0x30), TEST_IMG("foo blk 0 at 3"))?;
+        tline.put_page_image(TESTREL_A, 1, Lsn(0x40), TEST_IMG("foo blk 1 at 4"))?;
+        tline.put_page_image(TESTREL_A, 2, Lsn(0x50), TEST_IMG("foo blk 2 at 5"))?;

-        tline.advance_last_valid_lsn(Lsn(5));
+        tline.advance_last_record_lsn(Lsn(0x50));
+
+        assert_current_logical_size(&tline, Lsn(0x50));

        // The relation was created at LSN 2, not visible at LSN 1 yet.
-        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(1))?, false);
-        assert!(tline.get_relish_size(TESTREL_A, Lsn(1))?.is_none());
+        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(0x10))?, false);
+        assert!(tline.get_relish_size(TESTREL_A, Lsn(0x10))?.is_none());

-        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(2))?, true);
-        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(2))?.unwrap(), 1);
-        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(5))?.unwrap(), 3);
+        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(0x20))?, true);
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(0x20))?.unwrap(), 1);
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(0x50))?.unwrap(), 3);

        // Check page contents at each LSN
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(2))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x20))?,
            TEST_IMG("foo blk 0 at 2")
        );

        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(3))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x30))?,
            TEST_IMG("foo blk 0 at 3")
        );

        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(4))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x40))?,
            TEST_IMG("foo blk 0 at 3")
        );
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(4))?,
+            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x40))?,
            TEST_IMG("foo blk 1 at 4")
        );

        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(5))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x50))?,
            TEST_IMG("foo blk 0 at 3")
        );
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(5))?,
+            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x50))?,
            TEST_IMG("foo blk 1 at 4")
        );
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 2, Lsn(5))?,
+            tline.get_page_at_lsn(TESTREL_A, 2, Lsn(0x50))?,
            TEST_IMG("foo blk 2 at 5")
        );

        // Truncate last block
-        tline.put_truncation(TESTREL_A, Lsn(6), 2)?;
-        tline.advance_last_valid_lsn(Lsn(6));
+        tline.put_truncation(TESTREL_A, Lsn(0x60), 2)?;
+        tline.advance_last_record_lsn(Lsn(0x60));
+        assert_current_logical_size(&tline, Lsn(0x60));

        // Check reported size and contents after truncation
-        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(6))?.unwrap(), 2);
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(0x60))?.unwrap(), 2);
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(6))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x60))?,
            TEST_IMG("foo blk 0 at 3")
        );
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(6))?,
+            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x60))?,
            TEST_IMG("foo blk 1 at 4")
        );

        // should still see the truncated block with older LSN
-        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(5))?.unwrap(), 3);
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(0x50))?.unwrap(), 3);
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 2, Lsn(5))?,
+            tline.get_page_at_lsn(TESTREL_A, 2, Lsn(0x50))?,
            TEST_IMG("foo blk 2 at 5")
        );

+        // Truncate to zero length
+        tline.put_truncation(TESTREL_A, Lsn(0x68), 0)?;
+        tline.advance_last_record_lsn(Lsn(0x68));
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(0x68))?.unwrap(), 0);
+
+        // Extend from 0 to 2 blocks, leaving a gap
+        tline.put_page_image(TESTREL_A, 1, Lsn(0x70), TEST_IMG("foo blk 1"))?;
+        tline.advance_last_record_lsn(Lsn(0x70));
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(0x70))?.unwrap(), 2);
+        assert_eq!(tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x70))?, ZERO_PAGE);
+        assert_eq!(
+            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x70))?,
+            TEST_IMG("foo blk 1")
+        );
+
+        // Extend a lot more, leaving a big gap that spans across segments
+        // FIXME: This is currently broken, see https://github.com/zenithdb/zenith/issues/500
+        /*
+        tline.put_page_image(TESTREL_A, 1500, Lsn(0x80), TEST_IMG("foo blk 1500"))?;
+        tline.advance_last_record_lsn(Lsn(0x80));
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(0x80))?.unwrap(), 1501);
+        for blk in 2..1500 {
+            assert_eq!(
+                tline.get_page_at_lsn(TESTREL_A, blk, Lsn(0x80))?,
+                ZERO_PAGE);
+        }
+        assert_eq!(
+            tline.get_page_at_lsn(TESTREL_A, 1500, Lsn(0x80))?,
+            TEST_IMG("foo blk 1500"));
+         */
+
        Ok(())
    }

    /// Test get_relsize() and truncation with a file larger than 1 GB, so that it's
    /// split into multiple 1 GB segments in Postgres.
-    ///
-    /// This isn't very interesting with the RocksDb implementation, as we don't pay
-    /// any attention to Postgres segment boundaries there.
    #[test]
-    fn test_large_rel_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_large_rel_rocksdb", RepositoryFormat::RocksDb)?;
-        test_large_rel(&*repo)
-    }
-
-    #[test]
-    fn test_large_rel_layered() -> Result<()> {
-        let repo = get_test_repo("test_large_rel_layered", RepositoryFormat::Layered)?;
-        test_large_rel(&*repo)
-    }
-
-    fn test_large_rel(repo: &dyn Repository) -> Result<()> {
+    fn test_large_rel() -> Result<()> {
+        let repo = get_test_repo("test_large_rel")?;
        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
+        let tline = repo.create_empty_timeline(timelineid)?;

-        tline.init_valid_lsn(Lsn(1));
-
-        let mut lsn = 1;
+        let mut lsn = 0x10;
        for blknum in 0..pg_constants::RELSEG_SIZE + 1 {
            let img = TEST_IMG(&format!("foo blk {} at {}", blknum, Lsn(lsn)));
-            lsn += 1;
-            tline.put_page_image(TESTREL_A, blknum as u32, Lsn(lsn), img, true)?;
+            lsn += 0x10;
+            tline.put_page_image(TESTREL_A, blknum as u32, Lsn(lsn), img)?;
        }
-        tline.advance_last_valid_lsn(Lsn(lsn));
+        tline.advance_last_record_lsn(Lsn(lsn));
+
+        assert_current_logical_size(&tline, Lsn(lsn));

        assert_eq!(
            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
@@ -504,30 +419,32 @@ mod tests {
        );

        // Truncate one block
-        lsn += 1;
+        lsn += 0x10;
        tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE)?;
-        tline.advance_last_valid_lsn(Lsn(lsn));
+        tline.advance_last_record_lsn(Lsn(lsn));
        assert_eq!(
            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
            pg_constants::RELSEG_SIZE
        );
+        assert_current_logical_size(&tline, Lsn(lsn));

        // Truncate another block
-        lsn += 1;
+        lsn += 0x10;
        tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE - 1)?;
-        tline.advance_last_valid_lsn(Lsn(lsn));
+        tline.advance_last_record_lsn(Lsn(lsn));
        assert_eq!(
            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
            pg_constants::RELSEG_SIZE - 1
        );
+        assert_current_logical_size(&tline, Lsn(lsn));

        // Truncate to 1500, and then truncate all the way down to 0, one block at a time
        // This tests the behavior at segment boundaries
        let mut size: i32 = 3000;
        while size >= 0 {
-            lsn += 1;
+            lsn += 0x10;
            tline.put_truncation(TESTREL_A, Lsn(lsn), size as u32)?;
-            tline.advance_last_valid_lsn(Lsn(lsn));
+            tline.advance_last_record_lsn(Lsn(lsn));
            assert_eq!(
                tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
                size as u32
@@ -535,167 +452,122 @@ mod tests {

            size -= 1;
        }
+        assert_current_logical_size(&tline, Lsn(lsn));

        Ok(())
    }

-    fn skip_nonrel_objects<'a>(
-        snapshot: Box<dyn History + 'a>,
-    ) -> Result<impl Iterator<Item = <dyn History as Iterator>::Item> + 'a> {
-        Ok(snapshot.skip_while(|r| match r {
-            Ok(m) => match m.tag {
-                ObjectTag::RelationMetadata(_) => false,
-                _ => true,
-            },
-            _ => panic!("Iteration error"),
-        }))
-    }
-
+    ///
+    /// Test list_rels() function, with branches and dropped relations
+    ///
    #[test]
-    fn test_branch_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_branch_rocksdb", RepositoryFormat::RocksDb)?;
-        test_branch(&*repo)
-    }
+    // FIXME: The last assertion in this test is currently failing, see
+    // https://github.com/zenithdb/zenith/issues/502. Ignore the failure until that's fixed.
+    #[ignore]
+    fn test_list_rels_drop() -> Result<()> {
+        let repo = get_test_repo("test_list_rels_drop")?;
+        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
+        let tline = repo.create_empty_timeline(timelineid)?;
+        const TESTDB: u32 = 111;

-    #[test]
-    fn test_branch_layered() -> Result<()> {
-        let repo = get_test_repo("test_branch_layered", RepositoryFormat::Layered)?;
-        test_branch(&*repo)
+        // Import initial dummy checkpoint record, otherwise the get_timeline() call
+        // after branching fails below
+        tline.put_page_image(RelishTag::Checkpoint, 0, Lsn(0x10), ZERO_CHECKPOINT.clone())?;
+
+        // Create a relation on the timeline
+        tline.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?;
+
+        tline.advance_last_record_lsn(Lsn(0x30));
+
+        // Check that list_rels() lists it after LSN 2, but no before it
+        assert!(!tline.list_rels(0, TESTDB, Lsn(0x10))?.contains(&TESTREL_A));
+        assert!(tline.list_rels(0, TESTDB, Lsn(0x20))?.contains(&TESTREL_A));
+        assert!(tline.list_rels(0, TESTDB, Lsn(0x30))?.contains(&TESTREL_A));
+
+        // Create a branch, check that the relation is visible there
+        let newtimelineid = ZTimelineId::from_str("AA223344556677881122334455667788").unwrap();
+        repo.branch_timeline(timelineid, newtimelineid, Lsn(0x30))?;
+        let newtline = repo.get_timeline(newtimelineid)?;
+
+        assert!(newtline
+            .list_rels(0, TESTDB, Lsn(0x30))?
+            .contains(&TESTREL_A));
+
+        // Drop it on the branch
+        newtline.drop_relish(TESTREL_A, Lsn(0x40))?;
+        newtline.advance_last_record_lsn(Lsn(0x40));
+
+        // Check that it's no longer listed on the branch after the point where it was dropped
+        assert!(newtline
+            .list_rels(0, TESTDB, Lsn(0x30))?
+            .contains(&TESTREL_A));
+        assert!(!newtline
+            .list_rels(0, TESTDB, Lsn(0x40))?
+            .contains(&TESTREL_A));
+
+        // Run checkpoint and garbage collection and check that it's still not visible
+        newtline.checkpoint()?;
+        repo.gc_iteration(Some(newtimelineid), 0, true)?;
+
+        // FIXME: this is currently failing
+        assert!(!newtline
+            .list_rels(0, TESTDB, Lsn(0x40))?
+            .contains(&TESTREL_A));
+
+        Ok(())
    }

    ///
    /// Test branch creation
    ///
-    fn test_branch(repo: &dyn Repository) -> Result<()> {
+    #[test]
+    fn test_branch() -> Result<()> {
+        let repo = get_test_repo("test_branch")?;
        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
+        let tline = repo.create_empty_timeline(timelineid)?;

        // Import initial dummy checkpoint record, otherwise the get_timeline() call
        // after branching fails below
-        tline.put_page_image(RelishTag::Checkpoint, 0, Lsn(1), ZERO_PAGE.clone(), false)?;
+        tline.put_page_image(RelishTag::Checkpoint, 0, Lsn(0x10), ZERO_CHECKPOINT.clone())?;

        // Create a relation on the timeline
-        tline.init_valid_lsn(Lsn(1));
-        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("foo blk 0 at 4"), true)?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(0x30), TEST_IMG("foo blk 0 at 3"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(0x40), TEST_IMG("foo blk 0 at 4"))?;

        // Create another relation
-        tline.put_page_image(TESTREL_B, 0, Lsn(2), TEST_IMG("foobar blk 0 at 2"), true)?;
+        tline.put_page_image(TESTREL_B, 0, Lsn(0x20), TEST_IMG("foobar blk 0 at 2"))?;

-        tline.advance_last_valid_lsn(Lsn(4));
+        tline.advance_last_record_lsn(Lsn(0x40));
+        assert_current_logical_size(&tline, Lsn(0x40));

        // Branch the history, modify relation differently on the new timeline
        let newtimelineid = ZTimelineId::from_str("AA223344556677881122334455667788").unwrap();
-        repo.branch_timeline(timelineid, newtimelineid, Lsn(3))?;
+        repo.branch_timeline(timelineid, newtimelineid, Lsn(0x30))?;
        let newtline = repo.get_timeline(newtimelineid)?;

-        newtline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("bar blk 0 at 4"), true)?;
-        newtline.advance_last_valid_lsn(Lsn(4));
+        newtline.put_page_image(TESTREL_A, 0, Lsn(0x40), TEST_IMG("bar blk 0 at 4"))?;
+        newtline.advance_last_record_lsn(Lsn(0x40));

        // Check page contents on both branches
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(4))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x40))?,
            TEST_IMG("foo blk 0 at 4")
        );

        assert_eq!(
-            newtline.get_page_at_lsn(TESTREL_A, 0, Lsn(4))?,
+            newtline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x40))?,
            TEST_IMG("bar blk 0 at 4")
        );

        assert_eq!(
-            newtline.get_page_at_lsn(TESTREL_B, 0, Lsn(4))?,
+            newtline.get_page_at_lsn(TESTREL_B, 0, Lsn(0x40))?,
            TEST_IMG("foobar blk 0 at 2")
        );

-        assert_eq!(newtline.get_relish_size(TESTREL_B, Lsn(4))?.unwrap(), 1);
+        assert_eq!(newtline.get_relish_size(TESTREL_B, Lsn(0x40))?.unwrap(), 1);

-        Ok(())
-    }
-
-    #[test]
-    fn test_history_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_history_rocksdb", RepositoryFormat::RocksDb)?;
-        test_history(&*repo)
-    }
-    #[test]
-    // TODO: This doesn't work with the layered storage, the functions needed for push/pull
-    // functionality haven't been implemented yet.
-    #[ignore]
-    fn test_history_layered() -> Result<()> {
-        let repo = get_test_repo("test_history_layered", RepositoryFormat::Layered)?;
-        test_history(&*repo)
-    }
-    fn test_history(repo: &dyn Repository) -> Result<()> {
-        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
-
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(0));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        // add a page and advance the last valid LSN
-        let rel = TESTREL_A;
-        tline.put_page_image(rel, 1, Lsn(1), TEST_IMG("blk 1 @ lsn 1"), true)?;
-        tline.advance_last_valid_lsn(Lsn(1));
-
-        let expected_page = Modification {
-            tag: ObjectTag::Buffer(rel, 1),
-            lsn: Lsn(1),
-            data: ObjectValue::ser(&ObjectValue::Page(PageEntry::Page(TEST_IMG(
-                "blk 1 @ lsn 1",
-            ))))?,
-        };
-        let expected_init_size = Modification {
-            tag: ObjectTag::RelationMetadata(rel),
-            lsn: Lsn(1),
-            data: ObjectValue::ser(&ObjectValue::RelationSize(RelationSizeEntry::Size(2)))?,
-        };
-        let expected_trunc_size = Modification {
-            tag: ObjectTag::RelationMetadata(rel),
-            lsn: Lsn(2),
-            data: ObjectValue::ser(&ObjectValue::RelationSize(RelationSizeEntry::Size(0)))?,
-        };
-
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(1));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(
-            Some(&expected_init_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        // truncate to zero, but don't advance the last valid LSN
-        tline.put_truncation(rel, Lsn(2), 0)?;
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(1));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(
-            Some(&expected_init_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        // advance the last valid LSN and the truncation should be observable
-        tline.advance_last_valid_lsn(Lsn(2));
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(2));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(
-            Some(&expected_init_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(
-            Some(&expected_trunc_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
-        assert_eq!(None, snapshot.next().transpose()?);
+        assert_current_logical_size(&tline, Lsn(0x40));

        Ok(())
    }
--- a/pageserver/src/restore_local_repo.rs
+++ b/pageserver/src/restore_local_repo.rs
@@ -3,13 +3,13 @@
 //! zenith Timeline.
 //!
 use log::*;
+use postgres_ffi::nonrelfile_utils::clogpage_precedes;
+use postgres_ffi::nonrelfile_utils::slru_may_delete_clogsegment;
 use std::cmp::min;
 use std::fs;
 use std::fs::File;
 use std::io::Read;
-use std::io::Seek;
-use std::io::SeekFrom;
-use std::path::{Path, PathBuf};
+use std::path::Path;

 use anyhow::Result;
 use bytes::{Buf, Bytes};
@@ -17,16 +17,17 @@ use bytes::{Buf, Bytes};
 use crate::relish::*;
 use crate::repository::*;
 use crate::waldecoder::*;
+use postgres_ffi::nonrelfile_utils::mx_offset_to_member_segment;
 use postgres_ffi::relfile_utils::*;
 use postgres_ffi::xlog_utils::*;
+use postgres_ffi::Oid;
 use postgres_ffi::{pg_constants, CheckPoint, ControlFileData};
-use postgres_ffi::{Oid, TransactionId};
 use zenith_utils::lsn::Lsn;

 const MAX_MBR_BLKNO: u32 =
    pg_constants::MAX_MULTIXACT_ID / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;

-const ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
+static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);

 ///
 /// Import all relation data pages from local disk into the repository.
@@ -42,15 +43,8 @@ pub fn import_timeline_from_postgres_datadir(
        match direntry.file_name().to_str() {
            None => continue,

-            // These special files appear in the snapshot, but are not needed by the page server
            Some("pg_control") => {
-                import_nonrel_file(timeline, lsn, RelishTag::ControlFile, &direntry.path())?;
-                // Extract checkpoint record from pg_control and store is as separate object
-                let pg_control_bytes =
-                    timeline.get_page_at_lsn_nowait(RelishTag::ControlFile, 0, lsn)?;
-                let pg_control = ControlFileData::decode(&pg_control_bytes)?;
-                let checkpoint_bytes = pg_control.checkPointCopy.encode();
-                timeline.put_page_image(RelishTag::Checkpoint, 0, lsn, checkpoint_bytes, false)?;
+                import_control_file(timeline, lsn, &direntry.path())?;
            }
            Some("pg_filenode.map") => import_nonrel_file(
                timeline,
@@ -90,7 +84,6 @@ pub fn import_timeline_from_postgres_datadir(
            match direntry.file_name().to_str() {
                None => continue,

-                // These special files appear in the snapshot, but are not needed by the page server
                Some("PG_VERSION") => continue,
                Some("pg_filenode.map") => import_nonrel_file(
                    timeline,
@@ -127,13 +120,12 @@ pub fn import_timeline_from_postgres_datadir(
    }
    for entry in fs::read_dir(path.join("pg_twophase"))? {
        let entry = entry?;
-        let xid = u32::from_str_radix(&entry.path().to_str().unwrap(), 16)?;
+        let xid = u32::from_str_radix(entry.path().to_str().unwrap(), 16)?;
        import_nonrel_file(timeline, lsn, RelishTag::TwoPhase { xid }, &entry.path())?;
    }
    // TODO: Scan pg_tblspc

-    timeline.advance_last_valid_lsn(lsn);
-    timeline.checkpoint()?;
+    timeline.advance_last_record_lsn(lsn);

    Ok(())
 }
@@ -150,7 +142,7 @@ fn import_relfile(

    let p = parse_relfilename(path.file_name().unwrap().to_str().unwrap());
    if let Err(e) = p {
-        warn!("unrecognized file in snapshot: {:?} ({})", path, e);
+        warn!("unrecognized file in postgres datadir: {:?} ({})", path, e);
        return Err(e.into());
    }
    let (relnode, forknum, segno) = p.unwrap();
@@ -170,7 +162,7 @@ fn import_relfile(
                    forknum,
                };
                let tag = RelishTag::Relation(rel);
-                timeline.put_page_image(tag, blknum, lsn, Bytes::copy_from_slice(&buf), true)?;
+                timeline.put_page_image(tag, blknum, lsn, Bytes::copy_from_slice(&buf))?;
            }

            // TODO: UnexpectedEof is expected
@@ -211,7 +203,36 @@ fn import_nonrel_file(

    info!("importing non-rel file {}", path.display());

-    timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buffer[..]), false)?;
+    timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buffer[..]))?;
+    Ok(())
+}
+
+///
+/// Import pg_control file into the repository.
+///
+/// The control file is imported as is, but we also extract the checkpoint record
+/// from it and store it separated.
+fn import_control_file(timeline: &dyn Timeline, lsn: Lsn, path: &Path) -> Result<()> {
+    let mut file = File::open(path)?;
+    let mut buffer = Vec::new();
+    // read the whole file
+    file.read_to_end(&mut buffer)?;
+
+    info!("importing control file {}", path.display());
+
+    // Import it as ControlFile
+    timeline.put_page_image(
+        RelishTag::ControlFile,
+        0,
+        lsn,
+        Bytes::copy_from_slice(&buffer[..]),
+    )?;
+
+    // Extract the checkpoint record and import it separately.
+    let pg_control = ControlFileData::decode(&buffer)?;
+    let checkpoint_bytes = pg_control.checkPointCopy.encode();
+    timeline.put_page_image(RelishTag::Checkpoint, 0, lsn, checkpoint_bytes)?;
+
    Ok(())
 }

@@ -236,7 +257,6 @@ fn import_slru_file(timeline: &dyn Timeline, lsn: Lsn, slru: SlruKind, path: &Pa
                    rpageno,
                    lsn,
                    Bytes::copy_from_slice(&buf),
-                    true,
                )?;
            }

@@ -261,86 +281,6 @@ fn import_slru_file(timeline: &dyn Timeline, lsn: Lsn, slru: SlruKind, path: &Pa
    Ok(())
 }

-/// Scan PostgreSQL WAL files in given directory
-/// and load all records >= 'startpoint' into the repository.
-pub fn import_timeline_wal(walpath: &Path, timeline: &dyn Timeline, startpoint: Lsn) -> Result<()> {
-    let mut waldecoder = WalStreamDecoder::new(startpoint);
-
-    let mut segno = startpoint.segment_number(pg_constants::WAL_SEGMENT_SIZE);
-    let mut offset = startpoint.segment_offset(pg_constants::WAL_SEGMENT_SIZE);
-    let mut last_lsn = startpoint;
-
-    let checkpoint_bytes = timeline.get_page_at_lsn_nowait(RelishTag::Checkpoint, 0, startpoint)?;
-    let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
-
-    loop {
-        // FIXME: assume postgresql tli 1 for now
-        let filename = XLogFileName(1, segno, pg_constants::WAL_SEGMENT_SIZE);
-        let mut buf = Vec::new();
-
-        //Read local file
-        let mut path = walpath.join(&filename);
-
-        // It could be as .partial
-        if !PathBuf::from(&path).exists() {
-            path = walpath.join(filename + ".partial");
-        }
-
-        // Slurp the WAL file
-        let open_result = File::open(&path);
-        if let Err(e) = &open_result {
-            if e.kind() == std::io::ErrorKind::NotFound {
-                break;
-            }
-        }
-        let mut file = open_result?;
-
-        if offset > 0 {
-            file.seek(SeekFrom::Start(offset as u64))?;
-        }
-
-        let nread = file.read_to_end(&mut buf)?;
-        if nread != pg_constants::WAL_SEGMENT_SIZE - offset as usize {
-            // Maybe allow this for .partial files?
-            error!("read only {} bytes from WAL file", nread);
-        }
-
-        waldecoder.feed_bytes(&buf);
-
-        let mut nrecords = 0;
-        loop {
-            let rec = waldecoder.poll_decode();
-            if rec.is_err() {
-                // Assume that an error means we've reached the end of
-                // a partial WAL record. So that's ok.
-                trace!("WAL decoder error {:?}", rec);
-                break;
-            }
-            if let Some((lsn, recdata)) = rec.unwrap() {
-                let decoded = decode_wal_record(recdata.clone());
-                save_decoded_record(&mut checkpoint, timeline, &decoded, recdata, lsn)?;
-                last_lsn = lsn;
-            } else {
-                break;
-            }
-            nrecords += 1;
-        }
-
-        info!("imported {} records up to {}", nrecords, last_lsn);
-
-        segno += 1;
-        offset = 0;
-    }
-
-    info!("reached end of WAL at {}", last_lsn);
-    let checkpoint_bytes = checkpoint.encode();
-    timeline.put_page_image(RelishTag::Checkpoint, 0, last_lsn, checkpoint_bytes, false)?;
-
-    timeline.advance_last_valid_lsn(last_lsn);
-    timeline.checkpoint()?;
-    Ok(())
-}
-
 ///
 /// Helper function to parse a WAL record and call the Timeline's PUT functions for all the
 /// relations/pages that the record affects.
@@ -387,9 +327,34 @@ pub fn save_decoded_record(
        if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) == pg_constants::XLOG_DBASE_CREATE {
            let createdb = XlCreateDatabase::decode(&mut buf);
            save_xlog_dbase_create(timeline, lsn, &createdb)?;
-        } else {
-            // TODO
-            trace!("XLOG_DBASE_DROP is not handled yet");
+        } else if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK)
+            == pg_constants::XLOG_DBASE_DROP
+        {
+            let dropdb = XlDropDatabase::decode(&mut buf);
+
+            // To drop the database, we need to drop all the relations in it. Like in
+            // save_xlog_dbase_create(), use the previous record's LSN in the list_rels() call
+            let req_lsn = min(timeline.get_last_record_lsn(), lsn);
+
+            for tablespace_id in dropdb.tablespace_ids {
+                let rels = timeline.list_rels(tablespace_id, dropdb.db_id, req_lsn)?;
+                for rel in rels {
+                    timeline.drop_relish(rel, lsn)?;
+                }
+                trace!(
+                    "Drop FileNodeMap {}, {} at lsn {}",
+                    tablespace_id,
+                    dropdb.db_id,
+                    lsn
+                );
+                timeline.drop_relish(
+                    RelishTag::FileNodeMap {
+                        spcnode: tablespace_id,
+                        dbnode: dropdb.db_id,
+                    },
+                    lsn,
+                )?;
+            }
        }
    } else if decoded.xl_rmid == pg_constants::RM_TBLSPC_ID {
        trace!("XLOG_TBLSPC_CREATE/DROP is not handled yet");
@@ -406,8 +371,7 @@ pub fn save_decoded_record(
                },
                rpageno,
                lsn,
-                ZERO_PAGE,
-                true,
+                ZERO_PAGE.clone(),
            )?;
        } else {
            assert!(info == pg_constants::CLOG_TRUNCATE);
@@ -426,10 +390,12 @@ pub fn save_decoded_record(
            save_xact_record(timeline, lsn, &parsed_xact, decoded)?;
            // Remove twophase file. see RemoveTwoPhaseFile() in postgres code
            trace!(
-                "unlink twophaseFile for xid {} parsed_xact.xid {} here at {}",
-                decoded.xl_xid, parsed_xact.xid, lsn
+                "Drop twophaseFile for xid {} parsed_xact.xid {} here at {}",
+                decoded.xl_xid,
+                parsed_xact.xid,
+                lsn
            );
-            timeline.put_unlink(
+            timeline.drop_relish(
                RelishTag::TwoPhase {
                    xid: parsed_xact.xid,
                },
@@ -446,7 +412,6 @@ pub fn save_decoded_record(
                0,
                lsn,
                Bytes::copy_from_slice(&buf[..]),
-                true,
            )?;
        }
    } else if decoded.xl_rmid == pg_constants::RM_MULTIXACT_ID {
@@ -463,10 +428,9 @@ pub fn save_decoded_record(
                },
                rpageno,
                lsn,
-                ZERO_PAGE,
-                true,
+                ZERO_PAGE.clone(),
            )?;
-        } else if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE {
+        } else if info == pg_constants::XLOG_MULTIXACT_ZERO_MEM_PAGE {
            let pageno = buf.get_u32_le();
            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
@@ -477,8 +441,7 @@ pub fn save_decoded_record(
                },
                rpageno,
                lsn,
-                ZERO_PAGE,
-                true,
+                ZERO_PAGE.clone(),
            )?;
        } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
            let xlrec = XlMultiXactCreate::decode(&mut buf);
@@ -489,7 +452,7 @@ pub fn save_decoded_record(
        }
    } else if decoded.xl_rmid == pg_constants::RM_RELMAP_ID {
        let xlrec = XlRelmapUpdate::decode(&mut buf);
-        save_relmap_record(timeline, lsn, &xlrec, decoded)?;
+        save_relmap_page(timeline, lsn, &xlrec, decoded)?;
    } else if decoded.xl_rmid == pg_constants::RM_XLOG_ID {
        let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
        if info == pg_constants::XLOG_NEXTOID {
@@ -513,9 +476,6 @@ pub fn save_decoded_record(
            }
        }
    }
-    // Now that this record has been handled, let the repository know that
-    // it is up-to-date to this LSN
-    timeline.advance_last_record_lsn(lsn);
    Ok(())
 }

@@ -539,56 +499,52 @@ fn save_xlog_dbase_create(timeline: &dyn Timeline, lsn: Lsn, rec: &XlCreateDatab

    let mut num_rels_copied = 0;
    let mut num_blocks_copied = 0;
-    for src_rel in rels {
-        assert_eq!(src_rel.spcnode, src_tablespace_id);
-        assert_eq!(src_rel.dbnode, src_db_id);
+    for rel in rels {
+        if let RelishTag::Relation(src_rel) = rel {
+            assert_eq!(src_rel.spcnode, src_tablespace_id);
+            assert_eq!(src_rel.dbnode, src_db_id);

-        let nblocks = timeline
-            .get_relish_size(RelishTag::Relation(src_rel), req_lsn)?
-            .unwrap_or(0);
-        let dst_rel = RelTag {
-            spcnode: tablespace_id,
-            dbnode: db_id,
-            relnode: src_rel.relnode,
-            forknum: src_rel.forknum,
-        };
+            let nblocks = timeline.get_relish_size(rel, req_lsn)?.unwrap_or(0);
+            let dst_rel = RelTag {
+                spcnode: tablespace_id,
+                dbnode: db_id,
+                relnode: src_rel.relnode,
+                forknum: src_rel.forknum,
+            };

-        // Copy content
-        for blknum in 0..nblocks {
-            let content =
-                timeline.get_page_at_lsn_nowait(RelishTag::Relation(src_rel), blknum, req_lsn)?;
+            // Copy content
+            for blknum in 0..nblocks {
+                let content = timeline.get_page_at_lsn(rel, blknum, req_lsn)?;

-            debug!("copying block {} from {} to {}", blknum, src_rel, dst_rel);
+                debug!("copying block {} from {} to {}", blknum, src_rel, dst_rel);

-            timeline.put_page_image(RelishTag::Relation(dst_rel), blknum, lsn, content, true)?;
-            num_blocks_copied += 1;
+                timeline.put_page_image(RelishTag::Relation(dst_rel), blknum, lsn, content)?;
+                num_blocks_copied += 1;
+            }
+
+            if nblocks == 0 {
+                // make sure we have some trace of the relation, even if it's empty
+                timeline.put_truncation(RelishTag::Relation(dst_rel), lsn, 0)?;
+            }
+
+            num_rels_copied += 1;
        }
-
-        if nblocks == 0 {
-            // make sure we have some trace of the relation, even if it's empty
-            timeline.put_truncation(RelishTag::Relation(dst_rel), lsn, 0)?;
-        }
-
-        num_rels_copied += 1;
    }

    // Copy relfilemap
    // TODO This implementation is very inefficient -
    // it scans all non-rels only to find FileNodeMaps
    for tag in timeline.list_nonrels(req_lsn)? {
-        match tag {
-            RelishTag::FileNodeMap { spcnode, dbnode } => {
-                if spcnode == src_tablespace_id && dbnode == src_db_id {
-                    let img = timeline.get_page_at_lsn_nowait(tag, 0, req_lsn)?;
-                    let new_tag = RelishTag::FileNodeMap {
-                        spcnode: tablespace_id,
-                        dbnode: db_id,
-                    };
-                    timeline.put_page_image(new_tag, 0, lsn, img, false)?;
-                    break;
-                }
+        if let RelishTag::FileNodeMap { spcnode, dbnode } = tag {
+            if spcnode == src_tablespace_id && dbnode == src_db_id {
+                let img = timeline.get_page_at_lsn(tag, 0, req_lsn)?;
+                let new_tag = RelishTag::FileNodeMap {
+                    spcnode: tablespace_id,
+                    dbnode: db_id,
+                };
+                timeline.put_page_image(new_tag, 0, lsn, img)?;
+                break;
            }
-            _ => {} // do nothing
        }
    }
    info!(
@@ -662,7 +618,6 @@ fn save_xlog_smgr_truncate(timeline: &dyn Timeline, lsn: Lsn, rec: &XlSmgrTrunca

 /// Subroutine of save_decoded_record(), to handle an XLOG_XACT_* records.
 ///
-/// We are currently only interested in the dropped relations.
 fn save_xact_record(
    timeline: &dyn Timeline,
    lsn: Lsn,
@@ -713,50 +668,12 @@ fn save_xact_record(
                dbnode: xnode.dbnode,
                relnode: xnode.relnode,
            };
-            timeline.put_unlink(RelishTag::Relation(rel), lsn)?;
+            timeline.drop_relish(RelishTag::Relation(rel), lsn)?;
        }
    }
    Ok(())
 }

-// See TransactionIdPrecedes in transam.c
-fn transaction_id_precedes(id1: TransactionId, id2: TransactionId) -> bool {
-    /*
-     * If either ID is a permanent XID then we can just do unsigned
-     * comparison.  If both are normal, do a modulo-2^32 comparison.
-     */
-
-    if !(id1 >= pg_constants::FIRST_NORMAL_TRANSACTION_ID)
-        || !(id2 >= pg_constants::FIRST_NORMAL_TRANSACTION_ID)
-    {
-        return id1 < id2;
-    }
-
-    let diff = id1.wrapping_sub(id2) as i32;
-    return diff < 0;
-}
-
-// See CLOGPagePrecedes in clog.c
-fn clogpage_precedes(page1: u32, page2: u32) -> bool {
-    let mut xid1 = page1 * pg_constants::CLOG_XACTS_PER_PAGE;
-    xid1 += pg_constants::FIRST_NORMAL_TRANSACTION_ID + 1;
-    let mut xid2 = page2 * pg_constants::CLOG_XACTS_PER_PAGE;
-    xid2 += pg_constants::FIRST_NORMAL_TRANSACTION_ID + 1;
-
-    return transaction_id_precedes(xid1, xid2)
-        && transaction_id_precedes(xid1, xid2 + pg_constants::CLOG_XACTS_PER_PAGE - 1);
-}
-
-// See SlruMayDeleteSegment() in slru.c
-fn slru_may_delete_clogsegment(segpage: u32, cutoff_page: u32) -> bool {
-    let seg_last_page = segpage + pg_constants::SLRU_PAGES_PER_SEGMENT - 1;
-
-    assert_eq!(segpage % pg_constants::SLRU_PAGES_PER_SEGMENT, 0);
-
-    return clogpage_precedes(segpage, cutoff_page)
-        && clogpage_precedes(seg_last_page, cutoff_page);
-}
-
 fn save_clog_truncate_record(
    checkpoint: &mut CheckPoint,
    timeline: &dyn Timeline,
@@ -793,7 +710,7 @@ fn save_clog_truncate_record(
        return Ok(());
    }

-    // Iterate via SLRU CLOG segments and unlink segments that we're ready to truncate
+    // Iterate via SLRU CLOG segments and drop segments that we're ready to truncate
    // TODO This implementation is very inefficient -
    // it scans all non-rels only to find Clog
    //
@@ -803,17 +720,14 @@ fn save_clog_truncate_record(
    // instead.
    let req_lsn = min(timeline.get_last_record_lsn(), lsn);
    for obj in timeline.list_nonrels(req_lsn)? {
-        match obj {
-            RelishTag::Slru { slru, segno } => {
-                if slru == SlruKind::Clog {
-                    let segpage = segno * pg_constants::SLRU_PAGES_PER_SEGMENT;
-                    if slru_may_delete_clogsegment(segpage, xlrec.pageno) {
-                        timeline.put_unlink(RelishTag::Slru { slru, segno }, lsn)?;
-                        trace!("unlink CLOG segment {:>04X} at lsn {}", segno, lsn);
-                    }
+        if let RelishTag::Slru { slru, segno } = obj {
+            if slru == SlruKind::Clog {
+                let segpage = segno * pg_constants::SLRU_PAGES_PER_SEGMENT;
+                if slru_may_delete_clogsegment(segpage, xlrec.pageno) {
+                    timeline.drop_relish(RelishTag::Slru { slru, segno }, lsn)?;
+                    trace!("Drop CLOG segment {:>04X} at lsn {}", segno, lsn);
                }
            }
-            _ => {}
        }
    }

@@ -894,18 +808,6 @@ fn save_multixact_create_record(
    Ok(())
 }

-#[allow(non_upper_case_globals)]
-const MaxMultiXactOffset: u32 = 0xFFFFFFFF;
-
-#[allow(non_snake_case)]
-const fn MXOffsetToMemberPage(xid: u32) -> u32 {
-    xid / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32
-}
-#[allow(non_snake_case)]
-const fn MXOffsetToMemberSegment(xid: u32) -> i32 {
-    (MXOffsetToMemberPage(xid) / pg_constants::SLRU_PAGES_PER_SEGMENT) as i32
-}
-
 fn save_multixact_truncate_record(
    checkpoint: &mut CheckPoint,
    timeline: &dyn Timeline,
@@ -916,15 +818,15 @@ fn save_multixact_truncate_record(
    checkpoint.oldestMultiDB = xlrec.oldest_multi_db;

    // PerformMembersTruncation
-    let maxsegment: i32 = MXOffsetToMemberSegment(MaxMultiXactOffset);
-    let startsegment: i32 = MXOffsetToMemberSegment(xlrec.start_trunc_memb);
-    let endsegment: i32 = MXOffsetToMemberSegment(xlrec.end_trunc_memb);
+    let maxsegment: i32 = mx_offset_to_member_segment(pg_constants::MAX_MULTIXACT_OFFSET);
+    let startsegment: i32 = mx_offset_to_member_segment(xlrec.start_trunc_memb);
+    let endsegment: i32 = mx_offset_to_member_segment(xlrec.end_trunc_memb);
    let mut segment: i32 = startsegment;

    // Delete all the segments except the last one. The last segment can still
    // contain, possibly partially, valid data.
    while segment != endsegment {
-        timeline.put_unlink(
+        timeline.drop_relish(
            RelishTag::Slru {
                slru: SlruKind::MultiXactMembers,
                segno: segment as u32,
@@ -946,22 +848,23 @@ fn save_multixact_truncate_record(
    Ok(())
 }

-fn save_relmap_record(
+fn save_relmap_page(
    timeline: &dyn Timeline,
    lsn: Lsn,
    xlrec: &XlRelmapUpdate,
    decoded: &DecodedWALRecord,
 ) -> Result<()> {
-    let rec = WALRecord {
-        lsn,
-        will_init: true,
-        rec: decoded.record.clone(),
-        main_data_offset: decoded.main_data_offset as u32,
-    };
    let tag = RelishTag::FileNodeMap {
        spcnode: xlrec.tsid,
        dbnode: xlrec.dbid,
    };
-    timeline.put_wal_record(tag, 0, rec)?;
+
+    let mut buf = decoded.record.clone();
+    buf.advance(decoded.main_data_offset);
+    // skip xl_relmap_update
+    buf.advance(12);
+
+    timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buf[..]))?;
+
    Ok(())
 }
--- a/pageserver/src/rocksdb_storage.rs
+++ b/pageserver/src/rocksdb_storage.rs
@@ -1,475 +0,0 @@
-//!
-//! An implementation of the ObjectStore interface, backed by RocksDB
-//!
-use crate::object_key::*;
-use crate::object_store::ObjectStore;
-use crate::relish::*;
-use crate::PageServerConf;
-use anyhow::{bail, Result};
-use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
-use std::sync::{Arc, Mutex};
-use zenith_utils::bin_ser::BeSer;
-use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::ZTenantId;
-use zenith_utils::zid::ZTimelineId;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-struct StorageKey {
-    obj_key: ObjectKey,
-    lsn: Lsn,
-}
-
-impl StorageKey {
-    /// The first key for a given timeline
-    fn timeline_start(timeline: ZTimelineId) -> Self {
-        Self {
-            obj_key: ObjectKey {
-                timeline,
-                tag: ObjectTag::TimelineMetadataTag,
-            },
-            lsn: Lsn(0),
-        }
-    }
-}
-
-///
-/// RocksDB very inefficiently delete random record. Instead of it we have to use merge
-/// filter, which allows to throw away records at LSM merge phase.
-/// Unfortunately, it is hard (if ever possible)  to determine whether version can be removed
-/// at merge time. Version ca be removed if:
-/// 1. It is above PITR horizon (we need to get current LSN and gc_horizon from config)
-/// 2. Page is reconstructed at horizon (all WAL records above horizon are applied and can be removed)
-///
-/// So we have GC process which reconstructs pages at horizon and mark deteriorated WAL record
-/// for deletion. To mark object for deletion we can either set some flag in object itself.
-/// But it is complicated with new object value format, because RocksDB storage knows nothing about
-/// this format. Also updating whole record just to set one bit seems to be inefficient in any case.
-/// This is why we keep keys of marked for deletion versions in HashSet in memory.
-/// When LSM merge filter found key in this map, it removes it from the set preventing memory overflow.
-///
-struct GarbageCollector {
-    garbage: Mutex<HashSet<Vec<u8>>>,
-}
-
-impl GarbageCollector {
-    fn new() -> GarbageCollector {
-        GarbageCollector {
-            garbage: Mutex::new(HashSet::new()),
-        }
-    }
-
-    /// Called by GC to mark version as delete
-    fn mark_for_deletion(&self, key: &[u8]) {
-        let mut garbage = self.garbage.lock().unwrap();
-        garbage.insert(key.to_vec());
-    }
-
-    /// Called by LSM merge filter. If it finds key in the set, then
-    /// it doesn't merge it and removes from this set.
-    fn was_deleted(&self, key: &[u8]) -> bool {
-        let key = key.to_vec();
-        let mut garbage = self.garbage.lock().unwrap();
-        garbage.remove(&key)
-    }
-}
-
-pub struct RocksObjectStore {
-    _conf: &'static PageServerConf,
-
-    // RocksDB handle
-    db: rocksdb::DB,
-    gc: Arc<GarbageCollector>,
-}
-
-impl ObjectStore for RocksObjectStore {
-    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>> {
-        let val = self.db.get(StorageKey::ser(&StorageKey {
-            obj_key: key.clone(),
-            lsn,
-        })?)?;
-        if let Some(val) = val {
-            Ok(val)
-        } else {
-            bail!("could not find page {:?}", key);
-        }
-    }
-
-    fn get_next_key(&self, key: &ObjectKey) -> Result<Option<ObjectKey>> {
-        let mut iter = self.db.raw_iterator();
-        let search_key = StorageKey {
-            obj_key: key.clone(),
-            lsn: Lsn(0),
-        };
-        iter.seek(search_key.ser()?);
-        if !iter.valid() {
-            Ok(None)
-        } else {
-            let key = StorageKey::des(iter.key().unwrap())?;
-            Ok(Some(key.obj_key.clone()))
-        }
-    }
-
-    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()> {
-        self.db.put(
-            StorageKey::ser(&StorageKey {
-                obj_key: key.clone(),
-                lsn,
-            })?,
-            value,
-        )?;
-        Ok(())
-    }
-
-    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()> {
-        self.gc.mark_for_deletion(&StorageKey::ser(&StorageKey {
-            obj_key: key.clone(),
-            lsn,
-        })?);
-        Ok(())
-    }
-
-    /// Iterate through page versions of given page, starting from the given LSN.
-    /// The versions are walked in descending LSN order.
-    fn object_versions<'a>(
-        &'a self,
-        key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>> {
-        let iter = RocksObjectVersionIter::new(&self.db, key, lsn)?;
-        Ok(Box::new(iter))
-    }
-
-    /// Iterate through all timeline objects
-    fn list_objects<'a>(
-        &'a self,
-        timeline: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = ObjectTag> + 'a>> {
-        let iter = RocksObjectIter::new(&self.db, timeline, lsn)?;
-        Ok(Box::new(iter))
-    }
-
-    /// Get a list of all distinct relations in given tablespace and database.
-    ///
-    /// TODO: This implementation is very inefficient, it scans
-    /// through all entries in the given database. In practice, this
-    /// is used for CREATE DATABASE, and usually the template database is small.
-    /// But if it's not, this will be slow.
-    fn list_rels(
-        &self,
-        timelineid: ZTimelineId,
-        spcnode: u32,
-        dbnode: u32,
-        lsn: Lsn,
-    ) -> Result<HashSet<RelTag>> {
-        // FIXME: This scans everything. Very slow
-
-        let mut rels: HashSet<RelTag> = HashSet::new();
-
-        let mut search_rel_tag = RelTag {
-            spcnode,
-            dbnode,
-            relnode: 0,
-            forknum: 0u8,
-        };
-        let mut iter = self.db.raw_iterator();
-        loop {
-            let search_key = StorageKey {
-                obj_key: ObjectKey {
-                    timeline: timelineid,
-                    tag: ObjectTag::RelationMetadata(RelishTag::Relation(search_rel_tag)),
-                },
-                lsn: Lsn(0),
-            };
-            iter.seek(search_key.ser()?);
-            if !iter.valid() {
-                break;
-            }
-            let key = StorageKey::des(iter.key().unwrap())?;
-
-            if let ObjectTag::RelationMetadata(RelishTag::Relation(rel_tag)) = key.obj_key.tag {
-                if spcnode != 0 && rel_tag.spcnode != spcnode
-                    || dbnode != 0 && rel_tag.dbnode != dbnode
-                {
-                    break;
-                }
-                if key.lsn <= lsn {
-                    // visible in this snapshot
-                    rels.insert(rel_tag);
-                }
-                search_rel_tag = rel_tag;
-                // skip to next relation
-                // FIXME: What if relnode is u32::MAX ?
-                search_rel_tag.relnode += 1;
-            } else {
-                // no more relation metadata entries
-                break;
-            }
-        }
-
-        Ok(rels)
-    }
-
-    /// Get a list of all distinct NON-relations in timeline
-    /// that are visible at given lsn.
-    ///
-    /// TODO: This implementation is very inefficient, it scans
-    /// through all non-rel page versions in the system. In practice, this
-    /// is used when initializing a new compute node, and the non-rel files
-    /// are never very large nor change very frequently, so this will do for now.
-    fn list_nonrels(&self, timelineid: ZTimelineId, lsn: Lsn) -> Result<HashSet<RelishTag>> {
-        let mut rels: HashSet<RelishTag> = HashSet::new();
-
-        let search_key = StorageKey {
-            obj_key: ObjectKey {
-                timeline: timelineid,
-                tag: ObjectTag::Buffer(FIRST_NONREL_RELISH_TAG, 0),
-            },
-            lsn: Lsn(0),
-        };
-
-        let mut iter = self.db.raw_iterator();
-        iter.seek(search_key.ser()?);
-        while iter.valid() {
-            let key = StorageKey::des(iter.key().unwrap())?;
-
-            if key.obj_key.timeline != timelineid {
-                // reached end of this timeline in the store
-                break;
-            }
-
-            if let ObjectTag::Buffer(rel_tag, _blknum) = key.obj_key.tag {
-                if key.lsn <= lsn {
-                    // visible in this snapshot
-                    rels.insert(rel_tag);
-                }
-            }
-            // TODO: we could skip to next relation here like we do in list_rels(),
-            // but hopefully there are not that many SLRU segments or other non-rel
-            // entries for it to matter.
-            iter.next();
-        }
-
-        Ok(rels)
-    }
-
-    /// Iterate through versions of all objects in a timeline.
-    ///
-    /// Returns objects in increasing key-version order.
-    /// Returns all versions up to and including the specified LSN.
-    fn objects<'a>(
-        &'a self,
-        timeline: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = Result<(ObjectTag, Lsn, Vec<u8>)>> + 'a>> {
-        let start_key = StorageKey::timeline_start(timeline);
-        let start_key_bytes = StorageKey::ser(&start_key)?;
-        let iter = self.db.iterator(rocksdb::IteratorMode::From(
-            &start_key_bytes,
-            rocksdb::Direction::Forward,
-        ));
-
-        Ok(Box::new(RocksObjects {
-            iter,
-            timeline,
-            lsn,
-        }))
-    }
-
-    fn compact(&self) {
-        self.db.compact_range::<&[u8], &[u8]>(None, None);
-    }
-}
-
-impl RocksObjectStore {
-    /// Open a RocksDB database.
-    pub fn open(conf: &'static PageServerConf, tenantid: &ZTenantId) -> Result<RocksObjectStore> {
-        let opts = Self::get_rocksdb_opts();
-        let obj_store = Self::new(conf, opts, tenantid)?;
-        Ok(obj_store)
-    }
-
-    /// Create a new, empty RocksDB database.
-    pub fn create(conf: &'static PageServerConf, tenantid: &ZTenantId) -> Result<RocksObjectStore> {
-        let path = conf.tenant_path(&tenantid).join("rocksdb-storage");
-        std::fs::create_dir(&path)?;
-
-        let mut opts = Self::get_rocksdb_opts();
-        opts.create_if_missing(true);
-        opts.set_error_if_exists(true);
-        let obj_store = Self::new(conf, opts, tenantid)?;
-        Ok(obj_store)
-    }
-
-    fn new(
-        conf: &'static PageServerConf,
-        mut opts: rocksdb::Options,
-        tenantid: &ZTenantId,
-    ) -> Result<RocksObjectStore> {
-        let path = conf.tenant_path(&tenantid).join("rocksdb-storage");
-        let gc = Arc::new(GarbageCollector::new());
-        let gc_ref = gc.clone();
-        opts.set_compaction_filter("ttl", move |_level: u32, key: &[u8], _val: &[u8]| {
-            if gc_ref.was_deleted(key) {
-                rocksdb::compaction_filter::Decision::Remove
-            } else {
-                rocksdb::compaction_filter::Decision::Keep
-            }
-        });
-        let db = rocksdb::DB::open(&opts, &path)?;
-        let obj_store = RocksObjectStore {
-            _conf: conf,
-            db,
-            gc,
-        };
-        Ok(obj_store)
-    }
-
-    /// common options used by `open` and `create`
-    fn get_rocksdb_opts() -> rocksdb::Options {
-        let mut opts = rocksdb::Options::default();
-        opts.set_use_fsync(true);
-        opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
-        opts
-    }
-}
-
-///
-/// Iterator for `object_versions`. Returns all page versions of a given block, in
-/// reverse LSN order.
-///
-struct RocksObjectVersionIter<'a> {
-    obj_key: ObjectKey,
-    dbiter: rocksdb::DBRawIterator<'a>,
-    first_call: bool,
-}
-impl<'a> RocksObjectVersionIter<'a> {
-    fn new(
-        db: &'a rocksdb::DB,
-        obj_key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<RocksObjectVersionIter<'a>> {
-        let key = StorageKey {
-            obj_key: obj_key.clone(),
-            lsn,
-        };
-        let mut dbiter = db.raw_iterator();
-        dbiter.seek_for_prev(StorageKey::ser(&key)?); // locate last entry
-        Ok(RocksObjectVersionIter {
-            first_call: true,
-            obj_key: obj_key.clone(),
-            dbiter,
-        })
-    }
-}
-impl<'a> Iterator for RocksObjectVersionIter<'a> {
-    type Item = (Lsn, Vec<u8>);
-
-    fn next(&mut self) -> std::option::Option<Self::Item> {
-        if self.first_call {
-            self.first_call = false;
-        } else {
-            self.dbiter.prev(); // walk backwards
-        }
-
-        if !self.dbiter.valid() {
-            return None;
-        }
-        let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
-        if key.obj_key.tag != self.obj_key.tag {
-            return None;
-        }
-        let val = self.dbiter.value().unwrap();
-        let result = val.to_vec();
-
-        Some((key.lsn, result))
-    }
-}
-
-struct RocksObjects<'r> {
-    iter: rocksdb::DBIterator<'r>,
-    timeline: ZTimelineId,
-    lsn: Lsn,
-}
-
-impl<'r> Iterator for RocksObjects<'r> {
-    // TODO consider returning Box<[u8]>
-    type Item = Result<(ObjectTag, Lsn, Vec<u8>)>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.next_result().transpose()
-    }
-}
-
-impl<'r> RocksObjects<'r> {
-    fn next_result(&mut self) -> Result<Option<(ObjectTag, Lsn, Vec<u8>)>> {
-        for (key_bytes, v) in &mut self.iter {
-            let key = StorageKey::des(&key_bytes)?;
-
-            if key.obj_key.timeline != self.timeline {
-                return Ok(None);
-            }
-
-            if key.lsn > self.lsn {
-                // TODO can speed up by seeking iterator
-                continue;
-            }
-
-            return Ok(Some((key.obj_key.tag, key.lsn, v.to_vec())));
-        }
-
-        Ok(None)
-    }
-}
-
-///
-/// Iterator for `list_objects`. Returns all objects preceeding specified LSN
-///
-struct RocksObjectIter<'a> {
-    timeline: ZTimelineId,
-    key: StorageKey,
-    lsn: Lsn,
-    dbiter: rocksdb::DBRawIterator<'a>,
-}
-impl<'a> RocksObjectIter<'a> {
-    fn new(db: &'a rocksdb::DB, timeline: ZTimelineId, lsn: Lsn) -> Result<RocksObjectIter<'a>> {
-        let key = StorageKey {
-            obj_key: ObjectKey {
-                timeline,
-                tag: ObjectTag::FirstTag,
-            },
-            lsn: Lsn(0),
-        };
-        let dbiter = db.raw_iterator();
-        Ok(RocksObjectIter {
-            key,
-            timeline,
-            lsn,
-            dbiter,
-        })
-    }
-}
-impl<'a> Iterator for RocksObjectIter<'a> {
-    type Item = ObjectTag;
-
-    fn next(&mut self) -> std::option::Option<Self::Item> {
-        loop {
-            self.dbiter.seek(StorageKey::ser(&self.key).unwrap());
-            if !self.dbiter.valid() {
-                return None;
-            }
-            let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
-            if key.obj_key.timeline != self.timeline {
-                // End of this timeline
-                return None;
-            }
-            self.key = key.clone();
-            self.key.lsn = Lsn(u64::MAX); // next seek should skip all versions
-            if key.lsn <= self.lsn {
-                // visible in this snapshot
-                return Some(key.obj_key.tag);
-            }
-        }
-    }
-}
--- a/pageserver/src/tenant_mgr.rs
+++ b/pageserver/src/tenant_mgr.rs
@@ -3,19 +3,17 @@

 use crate::branches;
 use crate::layered_repository::LayeredRepository;
-use crate::object_repository::ObjectRepository;
-use crate::repository::Repository;
-use crate::rocksdb_storage::RocksObjectStore;
+use crate::repository::{Repository, Timeline};
 use crate::walredo::PostgresRedoManager;
-use crate::{PageServerConf, RepositoryFormat};
-use anyhow::{anyhow, bail, Result};
+use crate::PageServerConf;
+use anyhow::{anyhow, bail, Context, Result};
 use lazy_static::lazy_static;
 use log::info;
 use std::collections::HashMap;
 use std::fs;
 use std::str::FromStr;
 use std::sync::{Arc, Mutex};
-use zenith_utils::zid::ZTenantId;
+use zenith_utils::zid::{ZTenantId, ZTimelineId};

 lazy_static! {
    pub static ref REPOSITORY: Mutex<HashMap<ZTenantId, Arc<dyn Repository>>> =
@@ -33,27 +31,13 @@ pub fn init(conf: &'static PageServerConf) {
        let walredo_mgr = PostgresRedoManager::new(conf, tenantid);

        // Set up an object repository, for actual data storage.
-        let repo: Arc<dyn Repository + Sync + Send> = match conf.repository_format {
-            RepositoryFormat::Layered => {
-                let repo = Arc::new(LayeredRepository::new(
-                    conf,
-                    Arc::new(walredo_mgr),
-                    tenantid,
-                ));
-                LayeredRepository::launch_checkpointer_thread(conf, repo.clone());
-                repo
-            }
-            RepositoryFormat::RocksDb => {
-                let obj_store = RocksObjectStore::open(conf, &tenantid).unwrap();
-
-                Arc::new(ObjectRepository::new(
-                    conf,
-                    Arc::new(obj_store),
-                    Arc::new(walredo_mgr),
-                    tenantid,
-                ))
-            }
-        };
+        let repo = Arc::new(LayeredRepository::new(
+            conf,
+            Arc::new(walredo_mgr),
+            tenantid,
+        ));
+        LayeredRepository::launch_checkpointer_thread(conf, repo.clone());
+        LayeredRepository::launch_gc_thread(conf, repo.clone());

        info!("initialized storage for tenant: {}", &tenantid);
        m.insert(tenantid, repo);
@@ -83,9 +67,18 @@ pub fn insert_repository_for_tenant(tenantid: ZTenantId, repo: Arc<dyn Repositor
    o.insert(tenantid, repo);
 }

-pub fn get_repository_for_tenant(tenantid: &ZTenantId) -> Result<Arc<dyn Repository>> {
+pub fn get_repository_for_tenant(tenantid: ZTenantId) -> Result<Arc<dyn Repository>> {
    let o = &REPOSITORY.lock().unwrap();
-    o.get(tenantid)
+    o.get(&tenantid)
        .map(|repo| Arc::clone(repo))
        .ok_or_else(|| anyhow!("repository not found for tenant name {}", tenantid))
 }
+
+pub fn get_timeline_for_tenant(
+    tenantid: ZTenantId,
+    timelineid: ZTimelineId,
+) -> Result<Arc<dyn Timeline>> {
+    get_repository_for_tenant(tenantid)?
+        .get_timeline(timelineid)
+        .with_context(|| format!("cannot fetch timeline {}", timelineid))
+}
--- a/pageserver/src/waldecoder.rs
+++ b/pageserver/src/waldecoder.rs
@@ -1,6 +1,6 @@
 //!
 //! WAL decoder. For each WAL record, it decodes the record to figure out which data blocks
-//! the record affects, to add the records to the page cache.
+//! the record affects, so that they can be stored in repository.
 //!
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use crc32c::*;
@@ -10,17 +10,12 @@ use postgres_ffi::xlog_utils::*;
 use postgres_ffi::XLogLongPageHeaderData;
 use postgres_ffi::XLogPageHeaderData;
 use postgres_ffi::XLogRecord;
-use postgres_ffi::{Oid, TransactionId};
+use postgres_ffi::{BlockNumber, OffsetNumber};
+use postgres_ffi::{MultiXactId, MultiXactOffset, MultiXactStatus, Oid, TransactionId};
 use std::cmp::min;
 use thiserror::Error;
 use zenith_utils::lsn::Lsn;

-pub type BlockNumber = u32;
-pub type OffsetNumber = u16;
-pub type MultiXactId = TransactionId;
-pub type MultiXactOffset = u32;
-pub type MultiXactStatus = u32;
-
 #[allow(dead_code)]
 pub struct WalStreamDecoder {
    lsn: Lsn,
@@ -59,6 +54,11 @@ impl WalStreamDecoder {
        }
    }

+    // The latest LSN position fed to the decoder.
+    pub fn available(&self) -> Lsn {
+        self.lsn + self.inputbuf.remaining() as u64
+    }
+
    pub fn feed_bytes(&mut self, buf: &[u8]) {
        self.inputbuf.extend_from_slice(buf);
    }
@@ -164,17 +164,10 @@ impl WalStreamDecoder {
                    let recordbuf = recordbuf.freeze();
                    let mut buf = recordbuf.clone();

+                    let xlogrec = XLogRecord::from_bytes(&mut buf);
+
                    // XLOG_SWITCH records are special. If we see one, we need to skip
                    // to the next WAL segment.
-                    let xlogrec = XLogRecord::from_bytes(&mut buf);
-                    let mut crc = crc32c_append(0, &recordbuf[XLOG_RECORD_CRC_OFFS + 4..]);
-                    crc = crc32c_append(crc, &recordbuf[0..XLOG_RECORD_CRC_OFFS]);
-                    if crc != xlogrec.xl_crc {
-                        return Err(WalDecodeError {
-                            msg: "WAL record crc mismatch".into(),
-                            lsn: self.lsn,
-                        });
-                    }
                    if xlogrec.is_xlog_switch_record() {
                        trace!("saw xlog switch record at {}", self.lsn);
                        self.padlen =
@@ -184,7 +177,19 @@ impl WalStreamDecoder {
                        self.padlen = self.lsn.calc_padding(8u32) as u32;
                    }

-                    let result = (self.lsn, recordbuf);
+                    let mut crc = crc32c_append(0, &recordbuf[XLOG_RECORD_CRC_OFFS + 4..]);
+                    crc = crc32c_append(crc, &recordbuf[0..XLOG_RECORD_CRC_OFFS]);
+                    if crc != xlogrec.xl_crc {
+                        return Err(WalDecodeError {
+                            msg: "WAL record crc mismatch".into(),
+                            lsn: self.lsn,
+                        });
+                    }
+
+                    // Always align resulting LSN on 0x8 boundary -- that is important for getPage()
+                    // and WalReceiver integration. Since this code is used both for WalReceiver and
+                    // initial WAL import let's force alignment right here.
+                    let result = (self.lsn.align(), recordbuf);
                    return Ok(Some(result));
                }
                continue;
@@ -199,6 +204,7 @@ impl WalStreamDecoder {
 }

 #[allow(dead_code)]
+#[derive(Default)]
 pub struct DecodedBkpBlock {
    /* Is this block ref in use? */
    //in_use: bool,
@@ -231,25 +237,7 @@ pub struct DecodedBkpBlock {

 impl DecodedBkpBlock {
    pub fn new() -> DecodedBkpBlock {
-        DecodedBkpBlock {
-            rnode_spcnode: 0,
-            rnode_dbnode: 0,
-            rnode_relnode: 0,
-            forknum: 0,
-            blkno: 0,
-
-            flags: 0,
-            has_image: false,
-            apply_image: false,
-            will_init: false,
-            hole_offset: 0,
-            hole_length: 0,
-            bimg_len: 0,
-            bimg_info: 0,
-
-            has_data: false,
-            data_len: 0,
-        }
+        Default::default()
    }
 }

@@ -331,6 +319,31 @@ impl XlCreateDatabase {
    }
 }

+#[repr(C)]
+#[derive(Debug)]
+pub struct XlDropDatabase {
+    pub db_id: Oid,
+    pub n_tablespaces: Oid, /* number of tablespace IDs */
+    pub tablespace_ids: Vec<Oid>,
+}
+
+impl XlDropDatabase {
+    pub fn decode(buf: &mut Bytes) -> XlDropDatabase {
+        let mut rec = XlDropDatabase {
+            db_id: buf.get_u32_le(),
+            n_tablespaces: buf.get_u32_le(),
+            tablespace_ids: Vec::<Oid>::new(),
+        };
+
+        for _i in 0..rec.n_tablespaces {
+            let id = buf.get_u32_le();
+            rec.tablespace_ids.push(id);
+        }
+
+        rec
+    }
+}
+
 #[repr(C)]
 #[derive(Debug)]
 pub struct XlHeapInsert {
@@ -351,6 +364,7 @@ impl XlHeapInsert {
 #[derive(Debug)]
 pub struct XlHeapMultiInsert {
    pub flags: u8,
+    pub _padding: u8,
    pub ntuples: u16,
 }

@@ -358,6 +372,7 @@ impl XlHeapMultiInsert {
    pub fn decode(buf: &mut Bytes) -> XlHeapMultiInsert {
        XlHeapMultiInsert {
            flags: buf.get_u8(),
+            _padding: buf.get_u8(),
            ntuples: buf.get_u16_le(),
        }
    }
@@ -368,6 +383,8 @@ impl XlHeapMultiInsert {
 pub struct XlHeapDelete {
    pub xmax: TransactionId,
    pub offnum: OffsetNumber,
+    pub _padding: u16,
+    pub t_cid: u32,
    pub infobits_set: u8,
    pub flags: u8,
 }
@@ -377,6 +394,8 @@ impl XlHeapDelete {
        XlHeapDelete {
            xmax: buf.get_u32_le(),
            offnum: buf.get_u16_le(),
+            _padding: buf.get_u16_le(),
+            t_cid: buf.get_u32_le(),
            infobits_set: buf.get_u8(),
            flags: buf.get_u8(),
        }
@@ -390,6 +409,7 @@ pub struct XlHeapUpdate {
    pub old_offnum: OffsetNumber,
    pub old_infobits_set: u8,
    pub flags: u8,
+    pub t_cid: u32,
    pub new_xmax: TransactionId,
    pub new_offnum: OffsetNumber,
 }
@@ -401,6 +421,7 @@ impl XlHeapUpdate {
            old_offnum: buf.get_u16_le(),
            old_infobits_set: buf.get_u8(),
            flags: buf.get_u8(),
+            t_cid: buf.get_u32(),
            new_xmax: buf.get_u32_le(),
            new_offnum: buf.get_u16_le(),
        }
@@ -844,6 +865,7 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
        let blkno = blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32;
        if info == pg_constants::XLOG_HEAP_INSERT {
            let xlrec = XlHeapInsert::decode(&mut buf);
+            assert_eq!(0, buf.remaining());
            if (xlrec.flags
                & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
                    | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
@@ -859,6 +881,7 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
            }
        } else if info == pg_constants::XLOG_HEAP_DELETE {
            let xlrec = XlHeapDelete::decode(&mut buf);
+            assert_eq!(0, buf.remaining());
            if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 {
                let mut blk = DecodedBkpBlock::new();
                blk.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
@@ -872,6 +895,9 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
            || info == pg_constants::XLOG_HEAP_HOT_UPDATE
        {
            let xlrec = XlHeapUpdate::decode(&mut buf);
+            // the size of tuple data is inferred from the size of the record.
+            // we can't validate the remaining number of bytes without parsing
+            // the tuple data.
            if (xlrec.flags & pg_constants::XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0 {
                let mut blk = DecodedBkpBlock::new();
                blk.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
@@ -897,6 +923,15 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
        let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
        if info == pg_constants::XLOG_HEAP2_MULTI_INSERT {
            let xlrec = XlHeapMultiInsert::decode(&mut buf);
+
+            let offset_array_len = if xlogrec.xl_info & pg_constants::XLOG_HEAP_INIT_PAGE > 0 {
+                // the offsets array is omitted if XLOG_HEAP_INIT_PAGE is set
+                0
+            } else {
+                std::mem::size_of::<u16>() * xlrec.ntuples as usize
+            };
+            assert_eq!(offset_array_len, buf.remaining());
+
            if (xlrec.flags
                & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
                    | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
@@ -923,3 +958,71 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
        main_data_offset,
    }
 }
+
+///
+/// Build a human-readable string to describe a WAL record
+///
+/// For debugging purposes
+pub fn describe_wal_record(record: &Bytes) -> String {
+    // TODO: It would be nice to use the PostgreSQL rmgrdesc infrastructure for this.
+    // Maybe use the postgres wal redo process, the same used for replaying WAL records?
+    // Or could we compile the rmgrdesc routines into the dump_layer_file() binary directly,
+    // without worrying about security?
+    //
+    // But for now, we have a hand-written code for a few common WAL record types here.
+
+    let mut buf = record.clone();
+
+    // 1. Parse XLogRecord struct
+
+    // FIXME: assume little-endian here
+    let xlogrec = XLogRecord::from_bytes(&mut buf);
+
+    let unknown_str: String;
+
+    let result: &str = match xlogrec.xl_rmid {
+        pg_constants::RM_HEAP2_ID => {
+            let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
+            match info {
+                pg_constants::XLOG_HEAP2_MULTI_INSERT => "HEAP2 MULTI_INSERT",
+                pg_constants::XLOG_HEAP2_VISIBLE => "HEAP2 VISIBLE",
+                _ => {
+                    unknown_str = format!("HEAP2 UNKNOWN_0x{:02x}", info);
+                    &unknown_str
+                }
+            }
+        }
+        pg_constants::RM_HEAP_ID => {
+            let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
+            match info {
+                pg_constants::XLOG_HEAP_INSERT => "HEAP INSERT",
+                pg_constants::XLOG_HEAP_DELETE => "HEAP DELETE",
+                pg_constants::XLOG_HEAP_UPDATE => "HEAP UPDATE",
+                pg_constants::XLOG_HEAP_HOT_UPDATE => "HEAP HOT_UPDATE",
+                _ => {
+                    unknown_str = format!("HEAP2 UNKNOWN_0x{:02x}", info);
+                    &unknown_str
+                }
+            }
+        }
+        pg_constants::RM_XLOG_ID => {
+            let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
+            match info {
+                pg_constants::XLOG_FPI => "XLOG FPI",
+                pg_constants::XLOG_FPI_FOR_HINT => "XLOG FPI_FOR_HINT",
+                _ => {
+                    unknown_str = format!("XLOG UNKNOWN_0x{:02x}", info);
+                    &unknown_str
+                }
+            }
+        }
+        rmid => {
+            let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
+
+            unknown_str = format!("UNKNOWN_RM_{} INFO_0x{:02x}", rmid, info);
+            &unknown_str
+        }
+    };
+
+    String::from(result)
+}
--- a/pageserver/src/walreceiver.rs
+++ b/pageserver/src/walreceiver.rs
@@ -1,14 +1,15 @@
 //!
-//! WAL receiver connects to the WAL safekeeper service,
-//! streams WAL, decodes records and saves them in page cache.
+//! WAL receiver connects to the WAL safekeeper service, streams WAL,
+//! decodes records and saves them in the repository for the correct
+//! timeline.
 //!
 //! We keep one WAL receiver active per timeline.

-use crate::page_cache;
 use crate::relish::*;
 use crate::restore_local_repo;
+use crate::tenant_mgr;
 use crate::waldecoder::*;
-use crate::{PageServerConf, RepositoryFormat};
+use crate::PageServerConf;
 use anyhow::{Error, Result};
 use lazy_static::lazy_static;
 use log::*;
@@ -22,8 +23,6 @@ use postgres_types::PgLsn;
 use std::cmp::{max, min};
 use std::collections::HashMap;
 use std::fs;
-use std::fs::{File, OpenOptions};
-use std::io::{Seek, SeekFrom, Write};
 use std::str::FromStr;
 use std::sync::Mutex;
 use std::thread;
@@ -65,10 +64,13 @@ pub fn launch_wal_receiver(
            receivers.insert(timelineid, receiver);

            // Also launch a new thread to handle this connection
+            //
+            // NOTE: This thread name is checked in the assertion in wait_lsn. If you change
+            // this, make sure you update the assertion too.
            let _walreceiver_thread = thread::Builder::new()
                .name("WAL receiver thread".into())
                .spawn(move || {
-                    thread_main(conf, timelineid, &tenantid);
+                    thread_main(conf, timelineid, tenantid);
                })
                .unwrap();
        }
@@ -89,7 +91,7 @@ fn get_wal_producer_connstr(timelineid: ZTimelineId) -> String {
 //
 // This is the entry point for the WAL receiver thread.
 //
-fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId, tenantid: &ZTenantId) {
+fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId, tenantid: ZTenantId) {
    info!(
        "WAL receiver thread started for timeline : '{}'",
        timelineid
@@ -119,7 +121,7 @@ fn walreceiver_main(
    conf: &PageServerConf,
    timelineid: ZTimelineId,
    wal_producer_connstr: &str,
-    tenantid: &ZTenantId,
+    tenantid: ZTenantId,
 ) -> Result<(), Error> {
    // Connect to the database in replication mode.
    info!("connecting to {:?}", wal_producer_connstr);
@@ -145,15 +147,13 @@ fn walreceiver_main(
    let end_of_wal = Lsn::from(u64::from(identify.xlogpos));
    let mut caught_up = false;

-    let repository = page_cache::get_repository_for_tenant(tenantid)?;
-    let timeline = repository.get_timeline(timelineid).unwrap();
+    let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)?;

    //
    // Start streaming the WAL, from where we left off previously.
    //
    // If we had previously received WAL up to some point in the middle of a WAL record, we
-    // better start from the end of last full WAL record, not in the middle of one. Hence,
-    // use 'last_record_lsn' rather than 'last_valid_lsn' here.
+    // better start from the end of last full WAL record, not in the middle of one.
    let mut last_rec_lsn = timeline.get_last_record_lsn();
    let mut startpoint = last_rec_lsn;

@@ -162,13 +162,9 @@ fn walreceiver_main(
    }

    // There might be some padding after the last full record, skip it.
-    //
-    // FIXME: It probably would be better to always start streaming from the beginning
-    // of the page, or the segment, so that we could check the page/segment headers
-    // too. Just for the sake of paranoia.
    startpoint += startpoint.calc_padding(8u32);

-    debug!(
+    info!(
        "last_record_lsn {} starting replication from {} for timeline {}, server is at {}...",
        last_rec_lsn, startpoint, timelineid, end_of_wal
    );
@@ -180,7 +176,7 @@ fn walreceiver_main(

    let mut waldecoder = WalStreamDecoder::new(startpoint);

-    let checkpoint_bytes = timeline.get_page_at_lsn_nowait(RelishTag::Checkpoint, 0, startpoint)?;
+    let checkpoint_bytes = timeline.get_page_at_lsn(RelishTag::Checkpoint, 0, startpoint)?;
    let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
    trace!("CheckPoint.nextXid = {}", checkpoint.nextXid.value);

@@ -194,15 +190,6 @@ fn walreceiver_main(
                let endlsn = startlsn + data.len() as u64;
                let prev_last_rec_lsn = last_rec_lsn;

-                write_wal_file(
-                    conf,
-                    startlsn,
-                    &timelineid,
-                    pg_constants::WAL_SEGMENT_SIZE,
-                    data,
-                    tenantid,
-                )?;
-
                trace!("received XLogData between {} and {}", startlsn, endlsn);

                waldecoder.feed_bytes(data);
@@ -211,6 +198,12 @@ fn walreceiver_main(
                    // Save old checkpoint value to compare with it after decoding WAL record
                    let old_checkpoint_bytes = checkpoint.encode();
                    let decoded = decode_wal_record(recdata.clone());
+
+                    // It is important to deal with the aligned records as lsn in getPage@LSN is
+                    // aligned and can be several bytes bigger. Without this alignment we are
+                    // at risk of hittind a deadlock.
+                    assert!(lsn.is_aligned());
+
                    restore_local_repo::save_decoded_record(
                        &mut checkpoint,
                        &*timeline,
@@ -218,7 +211,6 @@ fn walreceiver_main(
                        recdata,
                        lsn,
                    )?;
-                    last_rec_lsn = lsn;

                    let new_checkpoint_bytes = checkpoint.encode();
                    // Check if checkpoint data was updated by save_decoded_record
@@ -228,18 +220,14 @@ fn walreceiver_main(
                            0,
                            lsn,
                            new_checkpoint_bytes,
-                            false,
                        )?;
                    }
-                }

-                // Update the last_valid LSN value in the page cache one more time. We updated
-                // it in the loop above, between each WAL record, but we might have received
-                // a partial record after the last completed record. Our page cache's value
-                // better reflect that, because GetPage@LSN requests might also point in the
-                // middle of a record, if the request LSN was taken from the server's current
-                // flush ptr.
-                timeline.advance_last_valid_lsn(endlsn);
+                    // Now that this record has been fully handled, including updating the
+                    // checkpoint data, let the repository know that it is up-to-date to this LSN
+                    timeline.advance_last_record_lsn(lsn);
+                    last_rec_lsn = lsn;
+                }

                // Somewhat arbitrarily, if we have at least 10 complete wal segments (16 MB each),
                // "checkpoint" the repository to flush all the changes from WAL we've processed
@@ -260,16 +248,10 @@ fn walreceiver_main(
                        &timelineid,
                        pg_constants::WAL_SEGMENT_SIZE,
                        last_rec_lsn,
-                        tenantid,
+                        &tenantid,
                    )?;

                    if newest_segno - oldest_segno >= 10 {
-                        // FIXME: The layered repository performs checkpointing in a separate thread, so this
-                        // isn't needed anymore. Remove 'checkpoint' from the Timeline trait altogether?
-                        if conf.repository_format == RepositoryFormat::RocksDb {
-                            timeline.checkpoint()?;
-                        }
-
                        // TODO: This is where we could remove WAL older than last_rec_lsn.
                        //remove_wal_files(timelineid, pg_constants::WAL_SEGMENT_SIZE, last_rec_lsn)?;
                    }
@@ -296,7 +278,7 @@ fn walreceiver_main(
                );

                if reply_requested {
-                    Some(timeline.get_last_valid_lsn())
+                    Some(timeline.get_last_record_lsn())
                } else {
                    None
                }
@@ -367,6 +349,9 @@ fn find_wal_file_range(
 ///
 /// [postgres docs]: https://www.postgresql.org/docs/current/protocol-replication.html
 #[derive(Debug)]
+// As of nightly 2021-09-11, fields that are only read by the type's `Debug` impl still count as
+// unused. Relevant issue: https://github.com/rust-lang/rust/issues/88900
+#[allow(dead_code)]
 pub struct IdentifySystem {
    systemid: u64,
    timeline: u32,
@@ -407,98 +392,3 @@ pub fn identify_system(client: &mut Client) -> Result<IdentifySystem, Error> {
        Err(IdentifyError.into())
    }
 }
-
-fn write_wal_file(
-    conf: &PageServerConf,
-    startpos: Lsn,
-    timelineid: &ZTimelineId,
-    wal_seg_size: usize,
-    buf: &[u8],
-    tenantid: &ZTenantId,
-) -> anyhow::Result<()> {
-    let mut bytes_left: usize = buf.len();
-    let mut bytes_written: usize = 0;
-    let mut partial;
-    let mut start_pos = startpos;
-    const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ];
-
-    let wal_dir = conf.wal_dir_path(timelineid, tenantid);
-
-    /* Extract WAL location for this block */
-    let mut xlogoff = start_pos.segment_offset(wal_seg_size);
-
-    while bytes_left != 0 {
-        let bytes_to_write;
-
-        /*
-         * If crossing a WAL boundary, only write up until we reach wal
-         * segment size.
-         */
-        if xlogoff + bytes_left > wal_seg_size {
-            bytes_to_write = wal_seg_size - xlogoff;
-        } else {
-            bytes_to_write = bytes_left;
-        }
-
-        /* Open file */
-        let segno = start_pos.segment_number(wal_seg_size);
-        let wal_file_name = XLogFileName(
-            1, // FIXME: always use Postgres timeline 1
-            segno,
-            wal_seg_size,
-        );
-        let wal_file_path = wal_dir.join(wal_file_name.clone());
-        let wal_file_partial_path = wal_dir.join(wal_file_name.clone() + ".partial");
-
-        {
-            let mut wal_file: File;
-            /* Try to open already completed segment */
-            if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_path) {
-                wal_file = file;
-                partial = false;
-            } else if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_partial_path) {
-                /* Try to open existed partial file */
-                wal_file = file;
-                partial = true;
-            } else {
-                /* Create and fill new partial file */
-                partial = true;
-                match OpenOptions::new()
-                    .create(true)
-                    .write(true)
-                    .open(&wal_file_partial_path)
-                {
-                    Ok(mut file) => {
-                        for _ in 0..(wal_seg_size / XLOG_BLCKSZ) {
-                            file.write_all(&ZERO_BLOCK)?;
-                        }
-                        wal_file = file;
-                    }
-                    Err(e) => {
-                        error!("Failed to open log file {:?}: {}", &wal_file_path, e);
-                        return Err(e.into());
-                    }
-                }
-            }
-            wal_file.seek(SeekFrom::Start(xlogoff as u64))?;
-            wal_file.write_all(&buf[bytes_written..(bytes_written + bytes_to_write)])?;
-
-            // FIXME: Flush the file
-            //wal_file.sync_all()?;
-        }
-        /* Write was successful, advance our position */
-        bytes_written += bytes_to_write;
-        bytes_left -= bytes_to_write;
-        start_pos += bytes_to_write as u64;
-        xlogoff += bytes_to_write;
-
-        /* Did we reach the end of a WAL segment? */
-        if start_pos.segment_offset(wal_seg_size) == 0 {
-            xlogoff = 0;
-            if partial {
-                fs::rename(&wal_file_partial_path, &wal_file_path)?;
-            }
-        }
-    }
-    Ok(())
-}
--- a/pageserver/src/walredo.rs
+++ b/pageserver/src/walredo.rs
@@ -44,9 +44,12 @@ use zenith_utils::zid::ZTenantId;

 use crate::relish::*;
 use crate::repository::WALRecord;
+use crate::waldecoder::XlMultiXactCreate;
 use crate::waldecoder::XlXactParsedRecord;
-use crate::waldecoder::{MultiXactId, XlMultiXactCreate};
 use crate::PageServerConf;
+use postgres_ffi::nonrelfile_utils::mx_offset_to_flags_bitshift;
+use postgres_ffi::nonrelfile_utils::mx_offset_to_flags_offset;
+use postgres_ffi::nonrelfile_utils::mx_offset_to_member_offset;
 use postgres_ffi::nonrelfile_utils::transaction_id_set_status;
 use postgres_ffi::pg_constants;
 use postgres_ffi::XLogRecord;
@@ -206,7 +209,7 @@ impl WalRedoManager for PostgresRedoManager {
            let process = (*process_guard).as_ref().unwrap();

            self.runtime
-                .block_on(self.handle_apply_request(&process, &request))
+                .block_on(self.handle_apply_request(process, &request))
        };
        end_time = Instant::now();

@@ -217,24 +220,6 @@ impl WalRedoManager for PostgresRedoManager {
    }
 }

-fn mx_offset_to_flags_offset(xid: MultiXactId) -> usize {
-    ((xid / pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP as u32) as u16
-        % pg_constants::MULTIXACT_MEMBERGROUPS_PER_PAGE
-        * pg_constants::MULTIXACT_MEMBERGROUP_SIZE) as usize
-}
-
-fn mx_offset_to_flags_bitshift(xid: MultiXactId) -> u16 {
-    (xid as u16) % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP
-        * pg_constants::MXACT_MEMBER_BITS_PER_XACT
-}
-
-/* Location (byte offset within page) of TransactionId of given member */
-fn mx_offset_to_member_offset(xid: MultiXactId) -> usize {
-    mx_offset_to_flags_offset(xid)
-        + (pg_constants::MULTIXACT_FLAGBYTES_PER_GROUP
-            + (xid as u16 % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP) * 4) as usize
-}
-
 impl PostgresRedoManager {
    ///
    /// Create a new PostgresRedoManager.
@@ -339,7 +324,7 @@ impl PostgresRedoManager {
                            if rec_segno == segno && blknum == rpageno {
                                transaction_id_set_status(
                                    *subxact,
-                                    pg_constants::TRANSACTION_STATUS_SUB_COMMITTED,
+                                    pg_constants::TRANSACTION_STATUS_COMMITTED,
                                    &mut page,
                                );
                            }
@@ -418,11 +403,6 @@ impl PostgresRedoManager {
                    } else {
                        panic!();
                    }
-                } else if xlogrec.xl_rmid == pg_constants::RM_RELMAP_ID {
-                    // Relation map file has size 512 bytes
-                    page.clear();
-                    page.extend_from_slice(&buf[12..]); // skip xl_relmap_update
-                    assert!(page.len() == 512); // size of pg_filenode.map
                }
            }

@@ -473,7 +453,7 @@ impl PostgresRedoProcess {
        // FIXME: We need a dummy Postgres cluster to run the process in. Currently, we
        // just create one with constant name. That fails if you try to launch more than
        // one WAL redo manager concurrently.
-        let datadir = conf.tenant_path(&tenantid).join("wal-redo-datadir");
+        let datadir = conf.tenant_path(tenantid).join("wal-redo-datadir");

        // Create empty data directory for wal-redo postgres, deleting old one first.
        if datadir.exists() {
--- a/postgres_ffi/Cargo.toml
+++ b/postgres_ffi/Cargo.toml
@@ -19,8 +19,9 @@ lazy_static = "1.4"
 log = "0.4.14"
 memoffset = "0.6.2"
 thiserror = "1.0"
+serde = { version = "1.0", features = ["derive"] }
 workspace_hack = { path = "../workspace_hack" }
 zenith_utils = { path = "../zenith_utils" }

 [build-dependencies]
-bindgen = "0.57"
+bindgen = "0.59.1"
--- a/postgres_ffi/README
+++ b/postgres_ffi/README
@@ -13,11 +13,6 @@ in each major PostgreSQL version. Currently, this module is based on
 PostgreSQL v14, but in the future we will probably need a separate
 copy for each PostgreSQL version.

-To interact with the C structs, there is some unsafe code in this
-module. Do not copy-paste that to the rest of the codebase! Keep the
-amount of unsafe code to a minimum, and limited to this module only,
-and only where it's truly needed.
-
 TODO: Currently, there is also some code that deals with WAL records
 in pageserver/src/waldecoder.rs.  That should be moved into this
 module. The rest of the codebase should not have intimate knowledge of
--- a/postgres_ffi/build.rs
+++ b/postgres_ffi/build.rs
@@ -3,6 +3,44 @@ extern crate bindgen;
 use std::env;
 use std::path::PathBuf;

+use bindgen::callbacks::ParseCallbacks;
+
+#[derive(Debug)]
+struct PostgresFfiCallbacks;
+
+impl ParseCallbacks for PostgresFfiCallbacks {
+    fn include_file(&self, filename: &str) {
+        // This does the equivalent of passing bindgen::CargoCallbacks
+        // to the builder .parse_callbacks() method.
+        let cargo_callbacks = bindgen::CargoCallbacks;
+        cargo_callbacks.include_file(filename)
+    }
+
+    // Add any custom #[derive] attributes to the data structures that bindgen
+    // creates.
+    fn add_derives(&self, name: &str) -> Vec<String> {
+        // This is the list of data structures that we want to serialize/deserialize.
+        let serde_list = [
+            "XLogRecord",
+            "XLogPageHeaderData",
+            "XLogLongPageHeaderData",
+            "CheckPoint",
+            "FullTransactionId",
+            "ControlFileData",
+        ];
+
+        if serde_list.contains(&name) {
+            vec![
+                "Default".into(), // Default allows us to easily fill the padding fields with 0.
+                "Serialize".into(),
+                "Deserialize".into(),
+            ]
+        } else {
+            vec![]
+        }
+    }
+}
+
 fn main() {
    // Tell cargo to invalidate the built crate whenever the wrapper changes
    println!("cargo:rerun-if-changed=pg_control_ffi.h");
@@ -19,20 +57,28 @@ fn main() {
        // Tell cargo to invalidate the built crate whenever any of the
        // included header files changed.
        //
-        .parse_callbacks(Box::new(bindgen::CargoCallbacks))
+        .parse_callbacks(Box::new(PostgresFfiCallbacks))
        //
        // These are the types and constants that we want to generate bindings for
        //
-        .whitelist_type("ControlFileData")
-        .whitelist_type("CheckPoint")
-        .whitelist_type("FullTransactionId")
-        .whitelist_type("XLogRecord")
-        .whitelist_type("XLogPageHeaderData")
-        .whitelist_type("XLogLongPageHeaderData")
-        .whitelist_var("XLOG_PAGE_MAGIC")
-        .whitelist_var("PG_CONTROL_FILE_SIZE")
-        .whitelist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
-        .whitelist_type("DBState")
+        .allowlist_type("BlockNumber")
+        .allowlist_type("OffsetNumber")
+        .allowlist_type("MultiXactId")
+        .allowlist_type("MultiXactOffset")
+        .allowlist_type("MultiXactStatus")
+        .allowlist_type("ControlFileData")
+        .allowlist_type("CheckPoint")
+        .allowlist_type("FullTransactionId")
+        .allowlist_type("XLogRecord")
+        .allowlist_type("XLogPageHeaderData")
+        .allowlist_type("XLogLongPageHeaderData")
+        .allowlist_var("XLOG_PAGE_MAGIC")
+        .allowlist_var("PG_CONTROL_FILE_SIZE")
+        .allowlist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
+        .allowlist_type("DBState")
+        // Because structs are used for serialization, tell bindgen to emit
+        // explicit padding fields.
+        .explicit_padding(true)
        //
        // Path the server include dir. It is in tmp_install/include/server, if you did
        // "configure --prefix=<path to tmp_install>". But if you used "configure --prefix=/",
--- a/postgres_ffi/pg_control_ffi.h
+++ b/postgres_ffi/pg_control_ffi.h
@@ -8,3 +8,6 @@
 #include "catalog/pg_control.h"
 #include "access/xlog_internal.h"

+#include "storage/block.h"
+#include "storage/off.h"
+#include "access/multixact.h"
--- a/postgres_ffi/src/controlfile_utils.rs
+++ b/postgres_ffi/src/controlfile_utils.rs
@@ -11,7 +11,7 @@
 //! data directory is compatible with a postgres binary. That includes
 //! a version number, configuration options that can be set at
 //! compilation time like the block size, and the platform's alignment
-//! and endianess information. (The PostgreSQL on-disk file format is
+//! and endianness information. (The PostgreSQL on-disk file format is
 //! not portable across platforms.)
 //!
 //! The control file is stored in the PostgreSQL data directory, as
@@ -43,6 +43,8 @@ impl ControlFileData {
    /// Interpret a slice of bytes as a Postgres control file.
    ///
    pub fn decode(buf: &[u8]) -> Result<ControlFileData> {
+        use zenith_utils::bin_ser::LeSer;
+
        // Check that the slice has the expected size. The control file is
        // padded with zeros up to a 512 byte sector size, so accept a
        // larger size too, so that the caller can just the whole file
@@ -55,26 +57,8 @@ impl ControlFileData {
        let OFFSETOF_CRC = Self::pg_control_crc_offset();
        let expectedcrc = crc32c::crc32c(&buf[0..OFFSETOF_CRC]);

-        // Convert the slice into an array of the right size, and use `transmute` to
-        // reinterpret the raw bytes as a ControlFileData struct.
-        //
-        // NB: Ideally we would use 'zerocopy::FromBytes' for this, but bindgen doesn't
-        // derive FromBytes for us. The safety of this depends on the same constraints
-        // as for FromBytes, namely, all of its fields must implement FromBytes. That
-        // includes the primitive integer types, like `u8`, `u16`, `u32`, `u64` and their
-        // signed variants. But `bool` is not safe, because the contents of the high bits
-        // in a rust bool are undefined. In practice, PostgreSQL uses 1 to represent
-        // true and 0 for false, which is compatible with Rust bool, but let's try not to
-        // depend on it.
-        //
-        // FIXME: ControlFileData does contain 'bool's at the moment.
-        //
-        // See https://github.com/zenithdb/zenith/issues/207 for discussion on the safety
-        // of this.
-        let mut b: [u8; SIZEOF_CONTROLDATA] = [0u8; SIZEOF_CONTROLDATA];
-        b.copy_from_slice(&buf[0..SIZEOF_CONTROLDATA]);
-        let controlfile: ControlFileData =
-            unsafe { std::mem::transmute::<[u8; SIZEOF_CONTROLDATA], ControlFileData>(b) };
+        // Use serde to deserialize the input as a ControlFileData struct.
+        let controlfile = ControlFileData::des_prefix(buf)?;

        // Check the CRC
        if expectedcrc != controlfile.crc {
@@ -93,21 +77,10 @@ impl ControlFileData {
    ///
    /// The CRC is recomputed to match the contents of the fields.
    pub fn encode(&self) -> Bytes {
-        //
-        // Use `transmute` to reinterpret struct as raw bytes.
-        //
-        // FIXME: This triggers undefined behavior, because the contents
-        // of the padding bytes are undefined, and this leaks those
-        // undefined bytes into the resulting array. The Rust code won't
-        // care what's in those bytes, and PostgreSQL doesn't care
-        // either. HOWEVER, it is a potential security issue, because the
-        // bytes can contain arbitrary pieces of memory from the page
-        // server. In the worst case, that could be private keys or
-        // another tenant's data.
-        //
-        // See https://github.com/zenithdb/zenith/issues/207 for discussion.
-        let b: [u8; SIZEOF_CONTROLDATA] =
-            unsafe { std::mem::transmute::<ControlFileData, [u8; SIZEOF_CONTROLDATA]>(*self) };
+        use zenith_utils::bin_ser::LeSer;
+
+        // Serialize into a new buffer.
+        let b = self.ser().unwrap();

        // Recompute the CRC
        let OFFSETOF_CRC = Self::pg_control_crc_offset();
--- a/postgres_ffi/src/lib.rs
+++ b/postgres_ffi/src/lib.rs
@@ -4,6 +4,9 @@
 // suppress warnings on rust 1.53 due to bindgen unit tests.
 // https://github.com/rust-lang/rust-bindgen/issues/1651
 #![allow(deref_nullptr)]
+
+use serde::{Deserialize, Serialize};
+
 include!(concat!(env!("OUT_DIR"), "/bindings.rs"));

 pub mod controlfile_utils;
@@ -11,3 +14,23 @@ pub mod nonrelfile_utils;
 pub mod pg_constants;
 pub mod relfile_utils;
 pub mod xlog_utils;
+
+//  See TransactionIdIsNormal in transam.h
+pub const fn transaction_id_is_normal(id: TransactionId) -> bool {
+    id > pg_constants::FIRST_NORMAL_TRANSACTION_ID
+}
+
+// See TransactionIdPrecedes in transam.c
+pub const fn transaction_id_precedes(id1: TransactionId, id2: TransactionId) -> bool {
+    /*
+     * If either ID is a permanent XID then we can just do unsigned
+     * comparison.  If both are normal, do a modulo-2^32 comparison.
+     */
+
+    if !(transaction_id_is_normal(id1)) || !transaction_id_is_normal(id2) {
+        return id1 < id2;
+    }
+
+    let diff = id1.wrapping_sub(id2) as i32;
+    diff < 0
+}
--- a/postgres_ffi/src/nonrelfile_utils.rs
+++ b/postgres_ffi/src/nonrelfile_utils.rs
@@ -1,10 +1,12 @@
 //!
 //! Common utilities for dealing with PostgreSQL non-relation files.
 //!
-use crate::pg_constants;
+use crate::{pg_constants, transaction_id_precedes};
 use bytes::BytesMut;
 use log::*;

+use crate::MultiXactId;
+
 pub fn transaction_id_set_status(xid: u32, status: u8, page: &mut BytesMut) {
    trace!(
        "handle_apply_request for RM_XACT_ID-{} (1-commit, 2-abort, 3-sub_commit)",
@@ -30,3 +32,51 @@ pub fn transaction_id_get_status(xid: u32, page: &[u8]) -> u8 {

    ((page[byteno] >> bshift) & pg_constants::CLOG_XACT_BITMASK) as u8
 }
+
+// See CLOGPagePrecedes in clog.c
+pub const fn clogpage_precedes(page1: u32, page2: u32) -> bool {
+    let mut xid1 = page1 * pg_constants::CLOG_XACTS_PER_PAGE;
+    xid1 += pg_constants::FIRST_NORMAL_TRANSACTION_ID + 1;
+    let mut xid2 = page2 * pg_constants::CLOG_XACTS_PER_PAGE;
+    xid2 += pg_constants::FIRST_NORMAL_TRANSACTION_ID + 1;
+
+    transaction_id_precedes(xid1, xid2)
+        && transaction_id_precedes(xid1, xid2 + pg_constants::CLOG_XACTS_PER_PAGE - 1)
+}
+
+// See SlruMayDeleteSegment() in slru.c
+pub fn slru_may_delete_clogsegment(segpage: u32, cutoff_page: u32) -> bool {
+    let seg_last_page = segpage + pg_constants::SLRU_PAGES_PER_SEGMENT - 1;
+
+    assert_eq!(segpage % pg_constants::SLRU_PAGES_PER_SEGMENT, 0);
+
+    clogpage_precedes(segpage, cutoff_page) && clogpage_precedes(seg_last_page, cutoff_page)
+}
+
+// Multixact utils
+
+pub fn mx_offset_to_flags_offset(xid: MultiXactId) -> usize {
+    ((xid / pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP as u32) as u16
+        % pg_constants::MULTIXACT_MEMBERGROUPS_PER_PAGE
+        * pg_constants::MULTIXACT_MEMBERGROUP_SIZE) as usize
+}
+
+pub fn mx_offset_to_flags_bitshift(xid: MultiXactId) -> u16 {
+    (xid as u16) % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP
+        * pg_constants::MXACT_MEMBER_BITS_PER_XACT
+}
+
+/* Location (byte offset within page) of TransactionId of given member */
+pub fn mx_offset_to_member_offset(xid: MultiXactId) -> usize {
+    mx_offset_to_flags_offset(xid)
+        + (pg_constants::MULTIXACT_FLAGBYTES_PER_GROUP
+            + (xid as u16 % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP) * 4) as usize
+}
+
+fn mx_offset_to_member_page(xid: u32) -> u32 {
+    xid / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32
+}
+
+pub fn mx_offset_to_member_segment(xid: u32) -> i32 {
+    (mx_offset_to_member_page(xid) / pg_constants::SLRU_PAGES_PER_SEGMENT) as i32
+}
--- a/postgres_ffi/src/pg_constants.rs
+++ b/postgres_ffi/src/pg_constants.rs
@@ -87,11 +87,14 @@ pub const XACT_XINFO_HAS_TWOPHASE: u32 = 1u32 << 4;
 pub const XLOG_NEXTOID: u8 = 0x30;
 pub const XLOG_SWITCH: u8 = 0x40;
 pub const XLOG_SMGR_TRUNCATE: u8 = 0x20;
+pub const XLOG_FPI_FOR_HINT: u8 = 0xA0;
+pub const XLOG_FPI: u8 = 0xB0;
 pub const DB_SHUTDOWNED: u32 = 1;

 // From multixact.h
 pub const FIRST_MULTIXACT_ID: u32 = 1;
 pub const MAX_MULTIXACT_ID: u32 = 0xFFFFFFFF;
+pub const MAX_MULTIXACT_OFFSET: u32 = 0xFFFFFFFF;

 pub const XLOG_MULTIXACT_ZERO_OFF_PAGE: u8 = 0x00;
 pub const XLOG_MULTIXACT_ZERO_MEM_PAGE: u8 = 0x10;
@@ -116,6 +119,7 @@ pub const XLOG_HEAP_INSERT: u8 = 0x00;
 pub const XLOG_HEAP_DELETE: u8 = 0x10;
 pub const XLOG_HEAP_UPDATE: u8 = 0x20;
 pub const XLOG_HEAP_HOT_UPDATE: u8 = 0x40;
+pub const XLOG_HEAP_INIT_PAGE: u8 = 0x80;
 pub const XLOG_HEAP2_VISIBLE: u8 = 0x40;
 pub const XLOG_HEAP2_MULTI_INSERT: u8 = 0x50;
 pub const XLH_INSERT_ALL_FROZEN_SET: u8 = (1 << 5) as u8;
@@ -185,11 +189,11 @@ pub const XLOG_CHECKPOINT_SHUTDOWN: u8 = 0x00;
 pub const XLOG_CHECKPOINT_ONLINE: u8 = 0x10;
 pub const XLP_LONG_HEADER: u16 = 0x0002;

-pub const PG_MAJORVERSION: &'static str = "14";
+pub const PG_MAJORVERSION: &str = "14";

 // List of subdirectories inside pgdata.
 // Copied from src/bin/initdb/initdb.c
-pub const PGDATA_SUBDIRS: [&'static str; 22] = [
+pub const PGDATA_SUBDIRS: [&str; 22] = [
    "global",
    "pg_wal/archive_status",
    "pg_commit_ts",
@@ -214,11 +218,11 @@ pub const PGDATA_SUBDIRS: [&'static str; 22] = [
    "pg_logical/mappings",
 ];

-pub const PGDATA_SPECIAL_FILES: [&'static str; 4] = [
-    "pg_hba.conf",
-    "pg_ident.conf",
-    "postgresql.conf",
-    "postgresql.auto.conf",
-];
+// Don't include postgresql.conf as it is inconvenient on node start:
+// we need postgresql.conf before basebackup to synchronize safekeepers
+// so no point in overwriting it during backup restore. Rest of the files
+// here are not needed before backup so it is okay to edit them after.
+pub const PGDATA_SPECIAL_FILES: [&str; 3] =
+    ["pg_hba.conf", "pg_ident.conf", "postgresql.auto.conf"];

-pub static PG_HBA: &'static str = include_str!("../samples/pg_hba.conf");
+pub static PG_HBA: &str = include_str!("../samples/pg_hba.conf");
--- a/postgres_ffi/src/xlog_utils.rs
+++ b/postgres_ffi/src/xlog_utils.rs
@@ -16,16 +16,20 @@ use crate::XLogPageHeaderData;
 use crate::XLogRecord;
 use crate::XLOG_PAGE_MAGIC;

+use anyhow::{bail, Result};
 use byteorder::{ByteOrder, LittleEndian};
 use bytes::{Buf, Bytes};
 use bytes::{BufMut, BytesMut};
 use crc32c::*;
 use log::*;
+use std::cmp::max;
 use std::cmp::min;
 use std::fs::{self, File};
 use std::io::prelude::*;
+use std::io::SeekFrom;
 use std::path::{Path, PathBuf};
 use std::time::SystemTime;
+use zenith_utils::lsn::Lsn;

 pub const XLOG_FNAME_LEN: usize = 24;
 pub const XLOG_BLCKSZ: usize = 8192;
@@ -37,6 +41,7 @@ pub const MAX_SEND_SIZE: usize = XLOG_BLCKSZ * 16;
 pub const XLOG_SIZE_OF_XLOG_SHORT_PHD: usize = std::mem::size_of::<XLogPageHeaderData>();
 pub const XLOG_SIZE_OF_XLOG_LONG_PHD: usize = std::mem::size_of::<XLogLongPageHeaderData>();
 pub const XLOG_SIZE_OF_XLOG_RECORD: usize = std::mem::size_of::<XLogRecord>();
+#[allow(clippy::identity_op)]
 pub const SIZE_OF_XLOG_RECORD_DATA_HEADER_SHORT: usize = 1 * 2;

 pub type XLogRecPtr = u64;
@@ -88,6 +93,21 @@ pub fn IsPartialXLogFileName(fname: &str) -> bool {
    fname.ends_with(".partial") && IsXLogFileName(&fname[0..fname.len() - 8])
 }

+/// If LSN points to the beginning of the page, then shift it to first record,
+/// otherwise align on 8-bytes boundary (required for WAL records)
+pub fn normalize_lsn(lsn: Lsn, seg_sz: usize) -> Lsn {
+    if lsn.0 % XLOG_BLCKSZ as u64 == 0 {
+        let hdr_size = if lsn.0 % seg_sz as u64 == 0 {
+            XLOG_SIZE_OF_XLOG_LONG_PHD
+        } else {
+            XLOG_SIZE_OF_XLOG_SHORT_PHD
+        };
+        lsn + hdr_size as u64
+    } else {
+        lsn.align()
+    }
+}
+
 pub fn get_current_timestamp() -> TimestampTz {
    const UNIX_EPOCH_JDATE: u64 = 2440588; /* == date2j(1970, 1, 1) */
    const POSTGRES_EPOCH_JDATE: u64 = 2451545; /* == date2j(2000, 1, 1) */
@@ -108,8 +128,10 @@ fn find_end_of_wal_segment(
    segno: XLogSegNo,
    tli: TimeLineID,
    wal_seg_size: usize,
-) -> u32 {
-    let mut offs: usize = 0;
+    start_offset: usize, // start reading at this point
+) -> Result<u32> {
+    // step back to the beginning of the page to read it in...
+    let mut offs: usize = start_offset - start_offset % XLOG_BLCKSZ;
    let mut contlen: usize = 0;
    let mut wal_crc: u32 = 0;
    let mut crc: u32 = 0;
@@ -118,23 +140,33 @@ fn find_end_of_wal_segment(
    let file_name = XLogFileName(tli, segno, wal_seg_size);
    let mut last_valid_rec_pos: usize = 0;
    let mut file = File::open(data_dir.join(file_name.clone() + ".partial")).unwrap();
+    file.seek(SeekFrom::Start(offs as u64))?;
    let mut rec_hdr = [0u8; XLOG_RECORD_CRC_OFFS];

    while offs < wal_seg_size {
+        // we are at the beginning of the page; read it in
        if offs % XLOG_BLCKSZ == 0 {
-            if let Ok(bytes_read) = file.read(&mut buf) {
-                if bytes_read != buf.len() {
-                    break;
-                }
-            } else {
-                break;
+            let bytes_read = file.read(&mut buf)?;
+            if bytes_read != buf.len() {
+                bail!(
+                    "failed to read {} bytes from {} at {}",
+                    XLOG_BLCKSZ,
+                    file_name,
+                    offs
+                );
            }
+
            let xlp_magic = LittleEndian::read_u16(&buf[0..2]);
            let xlp_info = LittleEndian::read_u16(&buf[2..4]);
            let xlp_rem_len = LittleEndian::read_u32(&buf[XLP_REM_LEN_OFFS..XLP_REM_LEN_OFFS + 4]);
+            // this is expected in current usage when valid WAL starts after page header
            if xlp_magic != XLOG_PAGE_MAGIC as u16 {
-                info!("Invalid WAL file {}.partial magic {}", file_name, xlp_magic);
-                break;
+                trace!(
+                    "invalid WAL file {}.partial magic {} at {:?}",
+                    file_name,
+                    xlp_magic,
+                    Lsn(XLogSegNoOffsetToRecPtr(segno, offs as u32, wal_seg_size)),
+                );
            }
            if offs == 0 {
                offs = XLOG_SIZE_OF_XLOG_LONG_PHD;
@@ -144,11 +176,19 @@ fn find_end_of_wal_segment(
            } else {
                offs += XLOG_SIZE_OF_XLOG_SHORT_PHD;
            }
+            // ... and step forward again if asked
+            offs = max(offs, start_offset);
+
+        // beginning of the next record
        } else if contlen == 0 {
            let page_offs = offs % XLOG_BLCKSZ;
            let xl_tot_len = LittleEndian::read_u32(&buf[page_offs..page_offs + 4]) as usize;
            if xl_tot_len == 0 {
-                break;
+                info!(
+                    "find_end_of_wal_segment reached zeros at {:?}",
+                    Lsn(XLogSegNoOffsetToRecPtr(segno, offs as u32, wal_seg_size))
+                );
+                break; // zeros, reached the end
            }
            last_valid_rec_pos = offs;
            offs += 4;
@@ -156,12 +196,13 @@ fn find_end_of_wal_segment(
            contlen = xl_tot_len - 4;
            rec_hdr[0..4].copy_from_slice(&buf[page_offs..page_offs + 4]);
        } else {
-            let page_offs = offs % XLOG_BLCKSZ;
            // we're continuing a record, possibly from previous page.
+            let page_offs = offs % XLOG_BLCKSZ;
            let pageleft = XLOG_BLCKSZ - page_offs;

            // read the rest of the record, or as much as fits on this page.
            let n = min(contlen, pageleft);
+            // fill rec_hdr (header up to (but not including) xl_crc field)
            if rec_offs < XLOG_RECORD_CRC_OFFS {
                let len = min(XLOG_RECORD_CRC_OFFS - rec_offs, n);
                rec_hdr[rec_offs..rec_offs + len].copy_from_slice(&buf[page_offs..page_offs + len]);
@@ -170,12 +211,11 @@ fn find_end_of_wal_segment(
                let crc_offs = page_offs - rec_offs + XLOG_RECORD_CRC_OFFS;
                wal_crc = LittleEndian::read_u32(&buf[crc_offs..crc_offs + 4]);
                crc = crc32c_append(0, &buf[crc_offs + 4..page_offs + n]);
-                crc = !crc;
            } else {
                crc ^= 0xFFFFFFFFu32;
                crc = crc32c_append(crc, &buf[page_offs..page_offs + n]);
-                crc = !crc;
            }
+            crc = !crc;
            rec_offs += n;
            offs += n;
            contlen -= n;
@@ -185,6 +225,8 @@ fn find_end_of_wal_segment(
                crc = crc32c_append(crc, &rec_hdr);
                offs = (offs + 7) & !7; // pad on 8 bytes boundary */
                if crc == wal_crc {
+                    // record is valid, advance the result to its end (with
+                    // alignment to the next record taken into account)
                    last_valid_rec_pos = offs;
                } else {
                    info!(
@@ -196,17 +238,21 @@ fn find_end_of_wal_segment(
            }
        }
    }
-    last_valid_rec_pos as u32
+    Ok(last_valid_rec_pos as u32)
 }

 ///
 /// Scan a directory that contains PostgreSQL WAL files, for the end of WAL.
+/// If precise, returns end LSN (next insertion point, basically);
+/// otherwise, start of the last segment.
+/// Returns (0, 0) if there is no WAL.
 ///
 pub fn find_end_of_wal(
    data_dir: &Path,
    wal_seg_size: usize,
    precise: bool,
-) -> (XLogRecPtr, TimeLineID) {
+    start_lsn: Lsn, // start reading WAL at this point or later
+) -> Result<(XLogRecPtr, TimeLineID)> {
    let mut high_segno: XLogSegNo = 0;
    let mut high_tli: TimeLineID = 0;
    let mut high_ispartial = false;
@@ -248,19 +294,32 @@ pub fn find_end_of_wal(
            high_segno += 1;
        } else if precise {
            /* otherwise locate last record in last partial segment */
-            high_offs = find_end_of_wal_segment(data_dir, high_segno, high_tli, wal_seg_size);
+            if start_lsn.segment_number(wal_seg_size) > high_segno {
+                bail!(
+                    "provided start_lsn {:?} is beyond highest segno {:?} available",
+                    start_lsn,
+                    high_segno,
+                );
+            }
+            high_offs = find_end_of_wal_segment(
+                data_dir,
+                high_segno,
+                high_tli,
+                wal_seg_size,
+                start_lsn.segment_offset(wal_seg_size),
+            )?;
        }
        let high_ptr = XLogSegNoOffsetToRecPtr(high_segno, high_offs, wal_seg_size);
-        return (high_ptr, high_tli);
+        return Ok((high_ptr, high_tli));
    }
-    (0, 0)
+    Ok((0, 0))
 }

 pub fn main() {
    let mut data_dir = PathBuf::new();
    data_dir.push(".");
    let wal_seg_size = 16 * 1024 * 1024;
-    let (wal_end, tli) = find_end_of_wal(&data_dir, wal_seg_size, true);
+    let (wal_end, tli) = find_end_of_wal(&data_dir, wal_seg_size, true, Lsn(0)).unwrap();
    println!(
        "wal_end={:>08X}{:>08X}, tli={}",
        (wal_end >> 32) as u32,
@@ -271,23 +330,13 @@ pub fn main() {

 impl XLogRecord {
    pub fn from_bytes(buf: &mut Bytes) -> XLogRecord {
-        XLogRecord {
-            xl_tot_len: buf.get_u32_le(),
-            xl_xid: buf.get_u32_le(),
-            xl_prev: buf.get_u64_le(),
-            xl_info: buf.get_u8(),
-            xl_rmid: buf.get_u8(),
-            xl_crc: {
-                buf.advance(2);
-                buf.get_u32_le()
-            },
-        }
+        use zenith_utils::bin_ser::LeSer;
+        XLogRecord::des_from(&mut buf.reader()).unwrap()
    }

    pub fn encode(&self) -> Bytes {
-        let b: [u8; XLOG_SIZE_OF_XLOG_RECORD];
-        b = unsafe { std::mem::transmute::<XLogRecord, [u8; XLOG_SIZE_OF_XLOG_RECORD]>(*self) };
-        Bytes::copy_from_slice(&b[..])
+        use zenith_utils::bin_ser::LeSer;
+        self.ser().unwrap().into()
    }

    // Is this record an XLOG_SWITCH record? They need some special processing,
@@ -298,34 +347,20 @@ impl XLogRecord {

 impl XLogPageHeaderData {
    pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogPageHeaderData {
-        let hdr: XLogPageHeaderData = XLogPageHeaderData {
-            xlp_magic: buf.get_u16_le(),
-            xlp_info: buf.get_u16_le(),
-            xlp_tli: buf.get_u32_le(),
-            xlp_pageaddr: buf.get_u64_le(),
-            xlp_rem_len: buf.get_u32_le(),
-        };
-        buf.get_u32_le(); //padding
-        hdr
+        use zenith_utils::bin_ser::LeSer;
+        XLogPageHeaderData::des_from(&mut buf.reader()).unwrap()
    }
 }

 impl XLogLongPageHeaderData {
    pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogLongPageHeaderData {
-        XLogLongPageHeaderData {
-            std: XLogPageHeaderData::from_bytes(buf),
-            xlp_sysid: buf.get_u64_le(),
-            xlp_seg_size: buf.get_u32_le(),
-            xlp_xlog_blcksz: buf.get_u32_le(),
-        }
+        use zenith_utils::bin_ser::LeSer;
+        XLogLongPageHeaderData::des_from(&mut buf.reader()).unwrap()
    }

    pub fn encode(&self) -> Bytes {
-        let b: [u8; XLOG_SIZE_OF_XLOG_LONG_PHD];
-        b = unsafe {
-            std::mem::transmute::<XLogLongPageHeaderData, [u8; XLOG_SIZE_OF_XLOG_LONG_PHD]>(*self)
-        };
-        Bytes::copy_from_slice(&b[..])
+        use zenith_utils::bin_ser::LeSer;
+        self.ser().unwrap().into()
    }
 }

@@ -333,17 +368,13 @@ pub const SIZEOF_CHECKPOINT: usize = std::mem::size_of::<CheckPoint>();

 impl CheckPoint {
    pub fn encode(&self) -> Bytes {
-        let b: [u8; SIZEOF_CHECKPOINT];
-        b = unsafe { std::mem::transmute::<CheckPoint, [u8; SIZEOF_CHECKPOINT]>(*self) };
-        Bytes::copy_from_slice(&b[..])
+        use zenith_utils::bin_ser::LeSer;
+        self.ser().unwrap().into()
    }

    pub fn decode(buf: &[u8]) -> Result<CheckPoint, anyhow::Error> {
-        let mut b = [0u8; SIZEOF_CHECKPOINT];
-        b.copy_from_slice(&buf[0..SIZEOF_CHECKPOINT]);
-        let checkpoint: CheckPoint;
-        checkpoint = unsafe { std::mem::transmute::<[u8; SIZEOF_CHECKPOINT], CheckPoint>(b) };
-        Ok(checkpoint)
+        use zenith_utils::bin_ser::LeSer;
+        Ok(CheckPoint::des(buf)?)
    }

    // Update next XID based on provided new_xid and stored epoch.
@@ -385,6 +416,7 @@ pub fn generate_wal_segment(pg_control: &ControlFileData) -> Bytes {
                xlp_tli: 1, // FIXME: always use Postgres timeline 1
                xlp_pageaddr: pg_control.checkPoint - XLOG_SIZE_OF_XLOG_LONG_PHD as u64,
                xlp_rem_len: 0,
+                ..Default::default() // Put 0 in padding fields.
            }
        },
        xlp_sysid: pg_control.system_identifier,
@@ -404,6 +436,7 @@ pub fn generate_wal_segment(pg_control: &ControlFileData) -> Bytes {
        xl_info: pg_constants::XLOG_CHECKPOINT_SHUTDOWN,
        xl_rmid: pg_constants::RM_XLOG_ID,
        xl_crc: 0,
+        ..Default::default() // Put 0 in padding fields.
    };

    let mut rec_shord_hdr_bytes = BytesMut::new();
@@ -434,7 +467,6 @@ mod tests {
    use super::*;
    use regex::Regex;
    use std::{env, process::Command, str::FromStr};
-    use zenith_utils::lsn::Lsn;

    // Run find_end_of_wal against file in test_wal dir
    // Ensure that it finds last record correctly
@@ -466,7 +498,7 @@ mod tests {
        let wal_seg_size = 16 * 1024 * 1024;

        // 3. Check end_of_wal on non-partial WAL segment (we treat it as fully populated)
-        let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true);
+        let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true, Lsn(0)).unwrap();
        let wal_end = Lsn(wal_end);
        println!("wal_end={}, tli={}", wal_end, tli);
        assert_eq!(wal_end, "0/2000000".parse::<Lsn>().unwrap());
@@ -483,7 +515,7 @@ mod tests {
        let waldump_output = std::str::from_utf8(&waldump_output.stderr).unwrap();
        println!("waldump_output = '{}'", &waldump_output);
        let re = Regex::new(r"invalid record length at (.+):").unwrap();
-        let caps = re.captures(&waldump_output).unwrap();
+        let caps = re.captures(waldump_output).unwrap();
        let waldump_wal_end = Lsn::from_str(caps.get(1).unwrap().as_str()).unwrap();

        // 5. Rename file to partial to actually find last valid lsn
@@ -492,7 +524,7 @@ mod tests {
            wal_dir.join("000000010000000000000001.partial"),
        )
        .unwrap();
-        let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true);
+        let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true, Lsn(0)).unwrap();
        let wal_end = Lsn(wal_end);
        println!("wal_end={}, tli={}", wal_end, tli);
        assert_eq!(wal_end, waldump_wal_end);
--- a/pre-commit.py
+++ b/pre-commit.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+from typing import List
+import subprocess
+import sys
+import enum
+import argparse
+import os
+
+
+@enum.unique
+class Color(enum.Enum):
+    RED = "\033[0;31m"
+    GREEN = "\033[0;33m"
+    CYAN = "\033[0;36m"
+
+
+NC = "\033[0m"  # No Color
+
+
+def colorify(
+    s: str,
+    color: Color,
+    no_color: bool = False,
+):
+    if no_color:
+        return s
+    return f"{color.value}{s}{NC}"
+
+
+def rustfmt(fix_inplace: bool = False, no_color: bool = False) -> str:
+    cmd = "rustfmt --edition=2018"
+    if not fix_inplace:
+        cmd += " --check"
+    if no_color:
+        cmd += " --color=never"
+    return cmd
+
+
+def get_commit_files() -> List[str]:
+    files = subprocess.check_output(
+        "git diff --cached --name-only --diff-filter=ACM".split()
+    )
+    return files.decode().splitlines()
+
+
+def check(
+    name: str, suffix: str, cmd: str, changed_files: List[str], no_color: bool = False
+):
+    print(f"Checking: {name} ", end="")
+    applicable_files = list(
+        filter(lambda fname: fname.strip().endswith(suffix), changed_files)
+    )
+    if not applicable_files:
+        print(colorify("[NOT APPLICABLE]", Color.CYAN, no_color))
+        return
+
+    cmd = f'{cmd} {" ".join(applicable_files)}'
+    res = subprocess.run(cmd.split(), capture_output=True)
+    if res.returncode != 0:
+        print(colorify("[FAILED]", Color.RED, no_color))
+        print("Please inspect the output below and run make fmt to fix automatically\n")
+        print(res.stdout.decode())
+        exit(1)
+
+    print(colorify("[OK]", Color.GREEN, no_color))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--fix-inplace", action="store_true", help="apply fixes inplace"
+    )
+    parser.add_argument(
+        "--no-color", action="store_true", help="disable colored output", default=not sys.stdout.isatty()
+    )
+    args = parser.parse_args()
+
+    files = get_commit_files()
+    # we use rustfmt here because cargo fmt does not accept list of files
+    # it internally gathers project files and feeds them to rustfmt
+    # so because we want to check only files included in the commit we use rustfmt directly
+    check(
+        name="rustfmt",
+        suffix=".rs",
+        cmd=rustfmt(fix_inplace=args.fix_inplace, no_color=args.no_color),
+        changed_files=files,
+        no_color=args.no_color,
+    )
--- a/proxy/Cargo.toml
+++ b/proxy/Cargo.toml
@@ -14,8 +14,9 @@ rand = "0.8.3"
 hex = "0.4.3"
 serde = "1"
 serde_json = "1"
-tokio = { version = "1.7.1", features = ["full"] }
-tokio-postgres = "0.7.2"
+tokio = "1.11"
+tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 clap = "2.33.0"
+rustls = "0.19.1"

 zenith_utils = { path = "../zenith_utils" }
--- a/proxy/src/cplane_api.rs
+++ b/proxy/src/cplane_api.rs
@@ -56,7 +56,7 @@ impl CPlaneApi {
            md5::compute([stored_hash.as_bytes(), salt].concat())
        );

-        let received_hash = std::str::from_utf8(&md5_response)?;
+        let received_hash = std::str::from_utf8(md5_response)?;

        println!(
            "auth: {} rh={} sh={} ssh={} {:?}",
--- a/proxy/src/main.rs
+++ b/proxy/src/main.rs
@@ -8,13 +8,15 @@
 use std::{
    collections::HashMap,
    net::{SocketAddr, TcpListener},
-    sync::{mpsc, Mutex},
+    sync::{mpsc, Arc, Mutex},
    thread,
 };

-use clap::{App, Arg};
+use anyhow::{anyhow, bail, ensure, Context};
+use clap::{App, Arg, ArgMatches};

 use cplane_api::DatabaseInfo;
+use rustls::{internal::pemfile, NoClientAuth, ProtocolVersion, ServerConfig};

 mod cplane_api;
 mod mgmt;
@@ -33,6 +35,8 @@ pub struct ProxyConf {

    /// control plane address where we would check auth.
    pub cplane_address: SocketAddr,
+
+    pub ssl_config: Option<Arc<ServerConfig>>,
 }

 pub struct ProxyState {
@@ -40,6 +44,38 @@ pub struct ProxyState {
    pub waiters: Mutex<HashMap<String, mpsc::Sender<anyhow::Result<DatabaseInfo>>>>,
 }

+fn configure_ssl(arg_matches: &ArgMatches) -> anyhow::Result<Option<Arc<ServerConfig>>> {
+    let (key_path, cert_path) = match (
+        arg_matches.value_of("ssl-key"),
+        arg_matches.value_of("ssl-cert"),
+    ) {
+        (Some(key_path), Some(cert_path)) => (key_path, cert_path),
+        (None, None) => return Ok(None),
+        _ => bail!("either both or neither ssl-key and ssl-cert must be specified"),
+    };
+
+    let key = {
+        let key_bytes = std::fs::read(key_path).context("SSL key file")?;
+        let mut keys = pemfile::rsa_private_keys(&mut &key_bytes[..])
+            .or_else(|_| pemfile::pkcs8_private_keys(&mut &key_bytes[..]))
+            .map_err(|_| anyhow!("couldn't read TLS keys"))?;
+        ensure!(keys.len() == 1, "keys.len() = {} (should be 1)", keys.len());
+        keys.pop().unwrap()
+    };
+
+    let cert_chain = {
+        let cert_chain_bytes = std::fs::read(cert_path).context("SSL cert file")?;
+        pemfile::certs(&mut &cert_chain_bytes[..])
+            .map_err(|_| anyhow!("couldn't read TLS certificates"))?
+    };
+
+    let mut config = ServerConfig::new(NoClientAuth::new());
+    config.set_single_cert(cert_chain, key)?;
+    config.versions = vec![ProtocolVersion::TLSv1_3];
+
+    Ok(Some(Arc::new(config)))
+}
+
 fn main() -> anyhow::Result<()> {
    let arg_matches = App::new("Zenith proxy/router")
        .arg(
@@ -66,6 +102,20 @@ fn main() -> anyhow::Result<()> {
                .help("redirect unauthenticated users to given uri")
                .default_value("http://localhost:3000/psql_session/"),
        )
+        .arg(
+            Arg::with_name("ssl-key")
+                .short("k")
+                .long("ssl-key")
+                .takes_value(true)
+                .help("path to SSL key for client postgres connections"),
+        )
+        .arg(
+            Arg::with_name("ssl-cert")
+                .short("c")
+                .long("ssl-cert")
+                .takes_value(true)
+                .help("path to SSL cert for client postgres connections"),
+        )
        .get_matches();

    let conf = ProxyConf {
@@ -73,6 +123,7 @@ fn main() -> anyhow::Result<()> {
        mgmt_address: arg_matches.value_of("mgmt").unwrap().parse()?,
        redirect_uri: arg_matches.value_of("uri").unwrap().parse()?,
        cplane_address: "127.0.0.1:3000".parse()?,
+        ssl_config: configure_ssl(&arg_matches)?,
    };
    let state = ProxyState {
        conf,
@@ -92,10 +143,10 @@ fn main() -> anyhow::Result<()> {
        // for each connection.
        thread::Builder::new()
            .name("Proxy thread".into())
-            .spawn(move || proxy::thread_main(&state, pageserver_listener))?,
+            .spawn(move || proxy::thread_main(state, pageserver_listener))?,
        thread::Builder::new()
            .name("Mgmt thread".into())
-            .spawn(move || mgmt::thread_main(&state, mgmt_listener))?,
+            .spawn(move || mgmt::thread_main(state, mgmt_listener))?,
    ];

    for t in threads.into_iter() {
--- a/proxy/src/mgmt.rs
+++ b/proxy/src/mgmt.rs
@@ -34,7 +34,7 @@ pub fn thread_main(state: &'static ProxyState, listener: TcpListener) -> anyhow:

 pub fn mgmt_conn_main(state: &'static ProxyState, socket: TcpStream) -> anyhow::Result<()> {
    let mut conn_handler = MgmtHandler { state };
-    let pgbackend = PostgresBackend::new(socket, AuthType::Trust)?;
+    let pgbackend = PostgresBackend::new(socket, AuthType::Trust, None)?;
    pgbackend.run(&mut conn_handler)
 }

--- a/proxy/src/proxy.rs
+++ b/proxy/src/proxy.rs
@@ -6,11 +6,12 @@ use anyhow::bail;
 use tokio_postgres::NoTls;

 use rand::Rng;
-use std::sync::mpsc::channel;
-use std::thread;
-use tokio::io::AsyncWriteExt;
+use std::io::Write;
+use std::{io, sync::mpsc::channel, thread};
+use zenith_utils::postgres_backend::Stream;
 use zenith_utils::postgres_backend::{PostgresBackend, ProtoState};
 use zenith_utils::pq_proto::*;
+use zenith_utils::sock_split::{ReadStream, WriteStream};
 use zenith_utils::{postgres_backend, pq_proto::BeMessage};

 ///
@@ -59,7 +60,11 @@ pub fn proxy_conn_main(
        cplane: CPlaneApi::new(&state.conf.cplane_address),
        user: "".into(),
        database: "".into(),
-        pgb: PostgresBackend::new(socket, postgres_backend::AuthType::MD5)?,
+        pgb: PostgresBackend::new(
+            socket,
+            postgres_backend::AuthType::MD5,
+            state.conf.ssl_config.clone(),
+        )?,
        md5_salt: [0u8; 4],
        psql_session_id: "".into(),
    };
@@ -75,17 +80,7 @@ pub fn proxy_conn_main(
        conn.handle_new_user()?
    };

-    // ok, proxy pass user connection to database_uri
-    let runtime = tokio::runtime::Builder::new_current_thread()
-        .enable_all()
-        .build()
-        .unwrap();
-
-    let _ = runtime.block_on(proxy_pass(conn.pgb, db_info))?;
-
-    println!("proxy_conn_main done;");
-
-    Ok(())
+    proxy_pass(conn.pgb, db_info)
 }

 impl ProxyConnection {
@@ -94,6 +89,7 @@ impl ProxyConnection {
    }

    fn handle_startup(&mut self) -> anyhow::Result<()> {
+        let mut encrypted = false;
        loop {
            let msg = self.pgb.read_message()?;
            println!("got message {:?}", msg);
@@ -102,11 +98,29 @@ impl ProxyConnection {
                    println!("got startup message {:?}", m);

                    match m.kind {
-                        StartupRequestCode::NegotiateGss | StartupRequestCode::NegotiateSsl => {
+                        StartupRequestCode::NegotiateGss => {
+                            self.pgb
+                                .write_message(&BeMessage::EncryptionResponse(false))?;
+                        }
+                        StartupRequestCode::NegotiateSsl => {
                            println!("SSL requested");
-                            self.pgb.write_message(&BeMessage::Negotiate)?;
+                            if self.pgb.tls_config.is_some() {
+                                self.pgb
+                                    .write_message(&BeMessage::EncryptionResponse(true))?;
+                                self.pgb.start_tls()?;
+                                encrypted = true;
+                            } else {
+                                self.pgb
+                                    .write_message(&BeMessage::EncryptionResponse(false))?;
+                            }
                        }
                        StartupRequestCode::Normal => {
+                            if self.state.conf.ssl_config.is_some() && !encrypted {
+                                self.pgb.write_message(&BeMessage::ErrorResponse(
+                                    "must connect with TLS".to_string(),
+                                ))?;
+                                bail!("client did not connect with TLS");
+                            }
                            self.user = m
                                .params
                                .get("user")
@@ -182,14 +196,14 @@ impl ProxyConnection {

        let hello_message = format!("☀️  Welcome to Zenith!

-To proceed with database creation open following link:
+To proceed with database creation, open the following link:

-    {}{}
+    {redirect_uri}{sess_id}

-It needed to be done once and we will send you '.pgpass' file which will allow you to access or create
+It needs to be done once and we will send you '.pgpass' file, which will allow you to access or create
 databases without opening the browser.

-", self.state.conf.redirect_uri,self.psql_session_id);
+", redirect_uri = self.state.conf.redirect_uri, sess_id = self.psql_session_id);

        self.pgb
            .write_message_noflush(&BeMessage::AuthenticationOk)?;
@@ -226,31 +240,52 @@ databases without opening the browser.
    }
 }

-async fn proxy_pass(pgb: PostgresBackend, db_info: DatabaseInfo) -> anyhow::Result<()> {
+/// Create a TCP connection to a postgres database, authenticate with it, and receive the ReadyForQuery message
+async fn connect_to_db(db_info: DatabaseInfo) -> anyhow::Result<tokio::net::TcpStream> {
    let mut socket = tokio::net::TcpStream::connect(db_info.socket_addr()).await?;
    let config = db_info.conn_string().parse::<tokio_postgres::Config>()?;
    let _ = config.connect_raw(&mut socket, NoTls).await?;
+    Ok(socket)
+}

-    println!("Connected to pg, proxying");
+/// Concurrently proxy both directions of the client and server connections
+fn proxy(
+    client_read: ReadStream,
+    client_write: WriteStream,
+    server_read: ReadStream,
+    server_write: WriteStream,
+) -> anyhow::Result<()> {
+    fn do_proxy(mut reader: ReadStream, mut writer: WriteStream) -> io::Result<()> {
+        std::io::copy(&mut reader, &mut writer)?;
+        writer.flush()?;
+        writer.shutdown(std::net::Shutdown::Both)
+    }

-    let incoming_std = pgb.into_stream();
-    incoming_std.set_nonblocking(true)?;
-    let mut incoming_conn = tokio::net::TcpStream::from_std(incoming_std)?;
+    let client_to_server_jh = thread::spawn(move || do_proxy(client_read, server_write));

-    let (mut ri, mut wi) = incoming_conn.split();
-    let (mut ro, mut wo) = socket.split();
-
-    let client_to_server = async {
-        tokio::io::copy(&mut ri, &mut wo).await?;
-        wo.shutdown().await
-    };
-
-    let server_to_client = async {
-        tokio::io::copy(&mut ro, &mut wi).await?;
-        wi.shutdown().await
-    };
-
-    tokio::try_join!(client_to_server, server_to_client)?;
+    let res1 = do_proxy(server_read, client_write);
+    let res2 = client_to_server_jh.join().unwrap();
+    res1?;
+    res2?;

    Ok(())
 }
+
+/// Proxy a client connection to a postgres database
+fn proxy_pass(pgb: PostgresBackend, db_info: DatabaseInfo) -> anyhow::Result<()> {
+    let runtime = tokio::runtime::Builder::new_current_thread().build()?;
+    let db_stream = runtime.block_on(connect_to_db(db_info))?;
+    let db_stream = db_stream.into_std()?;
+    db_stream.set_nonblocking(false)?;
+
+    let db_stream = zenith_utils::sock_split::BidiStream::from_tcp(db_stream);
+    let (db_read, db_write) = db_stream.split();
+
+    let stream = match pgb.into_stream() {
+        Stream::Bidirectional(bidi_stream) => bidi_stream,
+        _ => bail!("invalid stream"),
+    };
+
+    let (client_read, client_write) = stream.split();
+    proxy(client_read, client_write, db_read, db_write)
+}
--- a/run_clippy.sh
+++ b/run_clippy.sh
@@ -8,4 +8,8 @@
 # warnings and errors right in the editor.
 # In vscode, this setting is Rust-analyzer>Check On Save:Command

-cargo clippy "${@:2}" -- -A clippy::new_without_default -A clippy::manual_range_contains -A clippy::comparison_chain
+
+# * `-A unknown_lints` – do not warn about unknown lint suppressions
+#                        that people with newer toolchains might use
+# * `-D warnings`      - fail on any warnings (`cargo` returns non-zero exit status)
+cargo clippy "${@:2}" --all-targets --all-features --all --tests -- -A unknown_lints -D warnings
--- a/test_runner/Pipfile
+++ b/test_runner/Pipfile
@@ -8,6 +8,9 @@ pytest = ">=6.0.0"
 psycopg2 = "*"
 typing-extensions = "*"
 pyjwt = {extras = ["crypto"], version = "*"}
+requests = "*"
+pytest-xdist = "*"
+asyncpg = "*"

 [dev-packages]
 yapf = "*"
--- a/test_runner/Pipfile.lock
+++ b/test_runner/Pipfile.lock
@@ -1,7 +1,7 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "f60a966726bcc19670402ad3fa57396b5dacf0a027544418ceb7cc0d42d94a52"
+            "sha256": "3cdc048691824d0b93912b6b78a0aa01dc98f278212c1badb0cc2edbd2103c3a"
        },
        "pipfile-spec": 6,
        "requires": {
@@ -16,6 +16,25 @@
        ]
    },
    "default": {
+        "asyncpg": {
+            "hashes": [
+                "sha256:129d501f3d30616afd51eb8d3142ef51ba05374256bd5834cec3ef4956a9b317",
+                "sha256:29ef6ae0a617fc13cc2ac5dc8e9b367bb83cba220614b437af9b67766f4b6b20",
+                "sha256:41704c561d354bef01353835a7846e5606faabbeb846214dfcf666cf53319f18",
+                "sha256:556b0e92e2b75dc028b3c4bc9bd5162ddf0053b856437cf1f04c97f9c6837d03",
+                "sha256:8ff5073d4b654e34bd5eaadc01dc4d68b8a9609084d835acd364cd934190a08d",
+                "sha256:a458fc69051fbb67d995fdda46d75a012b5d6200f91e17d23d4751482640ed4c",
+                "sha256:a7095890c96ba36f9f668eb552bb020dddb44f8e73e932f8573efc613ee83843",
+                "sha256:a738f4807c853623d3f93f0fea11f61be6b0e5ca16ea8aeb42c2c7ee742aa853",
+                "sha256:c4fc0205fe4ddd5aeb3dfdc0f7bafd43411181e1f5650189608e5971cceacff1",
+                "sha256:dd2fa063c3344823487d9ddccb40802f02622ddf8bf8a6cc53885ee7a2c1c0c6",
+                "sha256:ddffcb85227bf39cd1bedd4603e0082b243cf3b14ced64dce506a15b05232b83",
+                "sha256:e36c6806883786b19551bb70a4882561f31135dc8105a59662e0376cf5b2cbc5",
+                "sha256:eed43abc6ccf1dc02e0d0efc06ce46a411362f3358847c6b0ec9a43426f91ece"
+            ],
+            "index": "pypi",
+            "version": "==0.24.0"
+        },
        "attrs": {
            "hashes": [
                "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1",
@@ -24,6 +43,13 @@
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
            "version": "==21.2.0"
        },
+        "certifi": {
+            "hashes": [
+                "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee",
+                "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"
+            ],
+            "version": "==2021.5.30"
+        },
        "cffi": {
            "hashes": [
                "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d",
@@ -74,22 +100,53 @@
            ],
            "version": "==1.14.6"
        },
+        "charset-normalizer": {
+            "hashes": [
+                "sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6",
+                "sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==2.0.6"
+        },
        "cryptography": {
            "hashes": [
-                "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d",
-                "sha256:1e056c28420c072c5e3cb36e2b23ee55e260cb04eee08f702e0edfec3fb51959",
-                "sha256:240f5c21aef0b73f40bb9f78d2caff73186700bf1bc6b94285699aff98cc16c6",
-                "sha256:26965837447f9c82f1855e0bc8bc4fb910240b6e0d16a664bb722df3b5b06873",
-                "sha256:37340614f8a5d2fb9aeea67fd159bfe4f5f4ed535b1090ce8ec428b2f15a11f2",
-                "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713",
-                "sha256:3d8427734c781ea5f1b41d6589c293089704d4759e34597dce91014ac125aad1",
-                "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177",
-                "sha256:8e56e16617872b0957d1c9742a3f94b43533447fd78321514abbe7db216aa250",
-                "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca",
-                "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d",
-                "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9"
+                "sha256:0a7dcbcd3f1913f664aca35d47c1331fce738d44ec34b7be8b9d332151b0b01e",
+                "sha256:1eb7bb0df6f6f583dd8e054689def236255161ebbcf62b226454ab9ec663746b",
+                "sha256:21ca464b3a4b8d8e86ba0ee5045e103a1fcfac3b39319727bc0fc58c09c6aff7",
+                "sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085",
+                "sha256:3520667fda779eb788ea00080124875be18f2d8f0848ec00733c0ec3bb8219fc",
+                "sha256:3c4129fc3fdc0fa8e40861b5ac0c673315b3c902bbdc05fc176764815b43dd1d",
+                "sha256:3fa3a7ccf96e826affdf1a0a9432be74dc73423125c8f96a909e3835a5ef194a",
+                "sha256:5b0fbfae7ff7febdb74b574055c7466da334a5371f253732d7e2e7525d570498",
+                "sha256:695104a9223a7239d155d7627ad912953b540929ef97ae0c34c7b8bf30857e89",
+                "sha256:8695456444f277af73a4877db9fc979849cd3ee74c198d04fc0776ebc3db52b9",
+                "sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c",
+                "sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7",
+                "sha256:9965c46c674ba8cc572bc09a03f4c649292ee73e1b683adb1ce81e82e9a6a0fb",
+                "sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14",
+                "sha256:a305600e7a6b7b855cd798e00278161b681ad6e9b7eca94c721d5f588ab212af",
+                "sha256:cd65b60cfe004790c795cc35f272e41a3df4631e2fb6b35aa7ac6ef2859d554e",
+                "sha256:d2a6e5ef66503da51d2110edf6c403dc6b494cc0082f85db12f54e9c5d4c3ec5",
+                "sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06",
+                "sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7"
            ],
-            "version": "==3.4.7"
+            "version": "==3.4.8"
+        },
+        "execnet": {
+            "hashes": [
+                "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5",
+                "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==1.9.0"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a",
+                "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==3.2"
        },
        "iniconfig": {
            "hashes": [
@@ -108,11 +165,11 @@
        },
        "pluggy": {
            "hashes": [
-                "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
-                "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
+                "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159",
+                "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"
            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==0.13.1"
+            "markers": "python_version >= '3.6'",
+            "version": "==1.0.0"
        },
        "psycopg2": {
            "hashes": [
@@ -166,11 +223,35 @@
        },
        "pytest": {
            "hashes": [
-                "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b",
-                "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890"
+                "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89",
+                "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"
            ],
            "index": "pypi",
-            "version": "==6.2.4"
+            "version": "==6.2.5"
+        },
+        "pytest-forked": {
+            "hashes": [
+                "sha256:6aa9ac7e00ad1a539c41bec6d21011332de671e938c7637378ec9710204e37ca",
+                "sha256:dc4147784048e70ef5d437951728825a131b81714b398d5d52f17c7c144d8815"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==1.3.0"
+        },
+        "pytest-xdist": {
+            "hashes": [
+                "sha256:7b61ebb46997a0820a263553179d6d1e25a8c50d8a8620cd1aa1e20e3be99168",
+                "sha256:89b330316f7fc475f999c81b577c2b926c9569f3d397ae432c0c2e2496d61ff9"
+            ],
+            "index": "pypi",
+            "version": "==2.4.0"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24",
+                "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"
+            ],
+            "index": "pypi",
+            "version": "==2.26.0"
        },
        "toml": {
            "hashes": [
@@ -182,12 +263,20 @@
        },
        "typing-extensions": {
            "hashes": [
-                "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497",
-                "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342",
-                "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"
+                "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e",
+                "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7",
+                "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"
            ],
            "index": "pypi",
-            "version": "==3.10.0.0"
+            "version": "==3.10.0.2"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4",
+                "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.26.6"
        }
    },
    "develop": {
@@ -268,12 +357,12 @@
        },
        "typing-extensions": {
            "hashes": [
-                "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497",
-                "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342",
-                "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"
+                "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e",
+                "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7",
+                "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"
            ],
            "index": "pypi",
-            "version": "==3.10.0.0"
+            "version": "==3.10.0.2"
        },
        "yapf": {
            "hashes": [
--- a/test_runner/README.md
+++ b/test_runner/README.md
@@ -7,7 +7,8 @@ Prerequisites:
 - Dependencies: install them via `pipenv install`. Note that Debian/Ubuntu
  packages are stale, as it commonly happens, so manual installation is not
  recommended.
-  Run `pipenv shell` to activate the venv.
+  Run `pipenv shell` to activate the venv or use `pipenv run` to run a single
+  command in the venv, e.g. `pipenv run pytest`.
 - Zenith and Postgres binaries
    - See the root README.md for build directions
    - Tests can be run from the git tree; or see the environment variables
--- a/test_runner/batch_others/test_auth.py
+++ b/test_runner/batch_others/test_auth.py
@@ -1,61 +1,21 @@

 from contextlib import closing
-from pathlib import Path
+from typing import Iterator
 from uuid import uuid4
-from dataclasses import dataclass
-import jwt
 import psycopg2
-from fixtures.zenith_fixtures import Postgres, ZenithCli, ZenithPageserver
+from fixtures.zenith_fixtures import PortDistributor, Postgres, ZenithCli, ZenithPageserver, PgBin
 import pytest


-@pytest.fixture
-def pageserver_auth_enabled(zenith_cli: ZenithCli):
-    with ZenithPageserver(zenith_cli).init(enable_auth=True).start() as ps:
-        # For convenience in tests, create a branch from the freshly-initialized cluster.
-        zenith_cli.run(["branch", "empty", "main"])
-        yield ps
+pytest_plugins = ("fixtures.zenith_fixtures")


-@dataclass
-class AuthKeys:
-    pub: bytes
-    priv: bytes
-
-    def generate_management_token(self):
-        token = jwt.encode({"scope": "pageserverapi"}, self.priv, algorithm="RS256")
-
-        # jwt.encode can return 'bytes' or 'str', depending on Python version or type
-        # hinting or something (not sure what). If it returned 'bytes', convert it to 'str'
-        # explicitly.
-        if isinstance(token, bytes):
-            token = token.decode()
-
-        return token
-
-    def generate_tenant_token(self, tenant_id):
-        token = jwt.encode({"scope": "tenant", "tenant_id": tenant_id}, self.priv, algorithm="RS256")
-
-        if isinstance(token, bytes):
-            token = token.decode()
-
-        return token
-
-
-@pytest.fixture
-def auth_keys(repo_dir: str) -> AuthKeys:
-    # TODO probably this should be specified in cli config and used in tests for single source of truth
-    pub = (Path(repo_dir) / 'auth_public_key.pem').read_bytes()
-    priv = (Path(repo_dir) / 'auth_private_key.pem').read_bytes()
-    return AuthKeys(pub=pub, priv=priv)
-
-
-def test_pageserver_auth(pageserver_auth_enabled: ZenithPageserver, auth_keys: AuthKeys):
+def test_pageserver_auth(pageserver_auth_enabled: ZenithPageserver):
    ps = pageserver_auth_enabled

-    tenant_token = auth_keys.generate_tenant_token(ps.initial_tenant)
-    invalid_tenant_token = auth_keys.generate_tenant_token(uuid4().hex)
-    management_token = auth_keys.generate_management_token()
+    tenant_token = ps.auth_keys.generate_tenant_token(ps.initial_tenant)
+    invalid_tenant_token = ps.auth_keys.generate_tenant_token(uuid4().hex)
+    management_token = ps.auth_keys.generate_management_token()

    # this does not invoke auth check and only decodes jwt and checks it for validity
    # check both tokens
@@ -86,12 +46,13 @@ def test_compute_auth_to_pageserver(
    pageserver_auth_enabled: ZenithPageserver,
    repo_dir: str,
    with_wal_acceptors: bool,
-    auth_keys: AuthKeys,
+    pg_bin: PgBin,
+    port_distributor: PortDistributor,
 ):
    ps = pageserver_auth_enabled
    # since we are in progress of refactoring protocols between compute safekeeper and page server
    # use hardcoded management token in safekeeper
-    management_token = auth_keys.generate_management_token()
+    management_token = ps.auth_keys.generate_management_token()

    branch = f"test_compute_auth_to_pageserver{with_wal_acceptors}"
    zenith_cli.run(["branch", branch, "empty"])
@@ -101,8 +62,9 @@ def test_compute_auth_to_pageserver(
    with Postgres(
        zenith_cli=zenith_cli,
        repo_dir=repo_dir,
+        pg_bin=pg_bin,
        tenant_id=ps.initial_tenant,
-        port=55432, # FIXME port distribution is hardcoded in tests and in cli
+        port=port_distributor.get_port(),
    ).create_start(
        branch,
        wal_acceptors=wa_factory.get_connstrs() if with_wal_acceptors else None,
--- a/test_runner/batch_others/test_branch_behind.py
+++ b/test_runner/batch_others/test_branch_behind.py
@@ -1,3 +1,4 @@
+import subprocess
 from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver


@@ -74,3 +75,18 @@ def test_branch_behind(zenith_cli, pageserver: ZenithPageserver, postgres: Postg
    # All the rows are visible on the main branch
    main_cur.execute('SELECT count(*) FROM foo')
    assert main_cur.fetchone() == (200100, )
+
+    # Check bad lsn's for branching
+
+    # branch at segment boundary
+    zenith_cli.run(["branch", "test_branch_segment_boundary", "test_branch_behind@0/3000000"])
+    pg = postgres.create_start("test_branch_segment_boundary")
+    cur = pg.connect().cursor()
+    cur.execute('SELECT 1')
+    assert cur.fetchone() == (1, )
+
+    # branch at pre-initdb lsn
+    try:
+        zenith_cli.run(["branch", "test_branch_preinitdb", "test_branch_behind@0/42"])
+    except subprocess.CalledProcessError:
+        print("Branch creation with pre-initdb LSN failed (as expected)")
--- a/test_runner/batch_others/test_createdb.py
+++ b/test_runner/batch_others/test_createdb.py
@@ -1,38 +0,0 @@
-from contextlib import closing
-from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory, ZenithCli
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-
-#
-# Test CREATE DATABASE when there have been relmapper changes
-#
-def test_createdb(
-    zenith_cli: ZenithCli,
-    pageserver: ZenithPageserver,
-    postgres: PostgresFactory,
-    pg_bin,
-):
-    zenith_cli.run(["branch", "test_createdb", "empty"])
-
-    pg = postgres.create_start('test_createdb')
-    print("postgres is running on 'test_createdb' branch")
-
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            # Cause a 'relmapper' change in the original branch
-            cur.execute('VACUUM FULL pg_class')
-
-            cur.execute('CREATE DATABASE foodb')
-
-            cur.execute('SELECT pg_current_wal_insert_lsn()')
-            lsn = cur.fetchone()[0]
-
-    # Create a branch
-    zenith_cli.run(["branch", "test_createdb2", "test_createdb@" + lsn])
-
-    pg2 = postgres.create_start('test_createdb2')
-
-    # Test that you can connect to the new database on both branches
-    for db in (pg, pg2):
-        db.connect(dbname='foodb').close()
--- a/test_runner/batch_others/test_createdropdb.py
+++ b/test_runner/batch_others/test_createdropdb.py
@@ -0,0 +1,102 @@
+import os
+import pathlib
+
+from contextlib import closing
+from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory, ZenithCli, check_restored_datadir_content
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+#
+# Test CREATE DATABASE when there have been relmapper changes
+#
+def test_createdb(
+    zenith_cli: ZenithCli,
+    pageserver: ZenithPageserver,
+    postgres: PostgresFactory,
+    pg_bin,
+):
+    zenith_cli.run(["branch", "test_createdb", "empty"])
+
+    pg = postgres.create_start('test_createdb')
+    print("postgres is running on 'test_createdb' branch")
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            # Cause a 'relmapper' change in the original branch
+            cur.execute('VACUUM FULL pg_class')
+
+            cur.execute('CREATE DATABASE foodb')
+
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn = cur.fetchone()[0]
+
+    # Create a branch
+    zenith_cli.run(["branch", "test_createdb2", "test_createdb@" + lsn])
+
+    pg2 = postgres.create_start('test_createdb2')
+
+    # Test that you can connect to the new database on both branches
+    for db in (pg, pg2):
+        db.connect(dbname='foodb').close()
+
+#
+# Test DROP DATABASE
+#
+def test_dropdb(
+    zenith_cli: ZenithCli,
+    pageserver: ZenithPageserver,
+    postgres: PostgresFactory,
+    pg_bin,
+    test_output_dir
+):
+    zenith_cli.run(["branch", "test_dropdb", "empty"])
+
+    pg = postgres.create_start('test_dropdb')
+    print("postgres is running on 'test_dropdb' branch")
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('CREATE DATABASE foodb')
+
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn_before_drop = cur.fetchone()[0]
+
+            cur.execute("SELECT oid FROM pg_database WHERE datname='foodb';")
+            dboid = cur.fetchone()[0]
+
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('DROP DATABASE foodb')
+
+            cur.execute('CHECKPOINT')
+
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn_after_drop = cur.fetchone()[0]
+
+
+    # Create two branches before and after database drop.
+    zenith_cli.run(["branch", "test_before_dropdb", "test_dropdb@" + lsn_before_drop])
+    pg_before = postgres.create_start('test_before_dropdb')
+
+    zenith_cli.run(["branch", "test_after_dropdb", "test_dropdb@" + lsn_after_drop])
+    pg_after = postgres.create_start('test_after_dropdb')
+
+    # Test that database exists on the branch before drop
+    pg_before.connect(dbname='foodb').close()
+
+    # Test that database subdir exists on the branch before drop
+    dbpath = pathlib.Path(pg_before.pgdata_dir) / 'base' / str(dboid)
+    print(dbpath)
+
+    assert os.path.isdir(dbpath) == True
+
+    # Test that database subdir doesn't exist on the branch after drop
+    dbpath = pathlib.Path(pg_after.pgdata_dir) / 'base' / str(dboid)
+    print(dbpath)
+
+    assert os.path.isdir(dbpath) == False
+
+    # Check that we restore the content of the datadir correctly
+    check_restored_datadir_content(zenith_cli, test_output_dir, pg, pageserver.service_port.pg)
--- a/test_runner/batch_others/test_gc.py
+++ b/test_runner/batch_others/test_gc.py
@@ -1,105 +0,0 @@
-import pytest
-
-from contextlib import closing
-from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
-import psycopg2.extras
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-#
-# Test Garbage Collection of old page versions.
-#
-# This test is pretty tightly coupled with the current implementation of page version storage
-# and garbage collection in object_repository.rs.
-#
-@pytest.mark.skip(reason=""""
-    Current GC test is flaky and overly strict. Since we are migrating to the layered repo format
-    with different GC implementation let's just silence this test for now. This test only
-    works with the RocksDB implementation.
-""")
-def test_gc(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
-    zenith_cli.run(["branch", "test_gc", "empty"])
-    pg = postgres.create_start('test_gc')
-
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            with closing(pageserver.connect()) as psconn:
-                with psconn.cursor(cursor_factory = psycopg2.extras.DictCursor) as pscur:
-
-                    # Get the timeline ID of our branch. We need it for the 'do_gc' command
-                    cur.execute("SHOW zenith.zenith_timeline")
-                    timeline = cur.fetchone()[0]
-
-                    # Create a test table
-                    cur.execute("CREATE TABLE foo(x integer)")
-
-                    # Run GC, to clear out any old page versions left behind in the catalogs by
-                    # the CREATE TABLE command. We want to have a clean slate with no garbage
-                    # before running the actual tests below, otherwise the counts won't match
-                    # what we expect.
-                    print("Running GC before test")
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    # remember the number of relations
-                    n_relations = row['n_relations']
-                    assert n_relations > 0
-
-                    # Insert a row. The first insert will also create a metadata entry for the
-                    # relation, with size == 1 block. Hence, bump up the expected relation count.
-                    n_relations += 1
-                    print("Inserting one row and running GC")
-                    cur.execute("INSERT INTO foo VALUES (1)")
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    assert row['n_relations'] == n_relations
-                    assert row['dropped'] == 0
-                    assert row['truncated'] == 31
-                    assert row['deleted'] == 4
-
-                    # Insert two more rows and run GC.
-                    print("Inserting two more rows and running GC")
-                    cur.execute("INSERT INTO foo VALUES (2)")
-                    cur.execute("INSERT INTO foo VALUES (3)")
-
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    assert row['n_relations'] == n_relations
-                    assert row['dropped'] == 0
-                    assert row['truncated'] == 31
-                    assert row['deleted'] == 4
-
-                    # Insert one more row. It creates one more page version, but doesn't affect the
-                    # relation size.
-                    print("Inserting one more row")
-                    cur.execute("INSERT INTO foo VALUES (3)")
-
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    assert row['n_relations'] == n_relations
-                    assert row['dropped'] == 0
-                    assert row['truncated'] == 31
-                    assert row['deleted'] == 2
-
-                    # Run GC again, with no changes in the database. Should not remove anything.
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    assert row['n_relations'] == n_relations
-                    assert row['dropped'] == 0
-                    assert row['truncated'] == 31
-                    assert row['deleted'] == 0
-
-                    #
-                    # Test DROP TABLE checks that relation data and metadata was deleted by GC from object storage
-                    #
-                    cur.execute("DROP TABLE foo")
-
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    # Each relation fork is counted separately, hence 3.
-                    assert row['dropped'] == 3
--- a/test_runner/batch_others/test_multixact.py
+++ b/test_runner/batch_others/test_multixact.py
@@ -1,4 +1,4 @@
-from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver, check_restored_datadir_content

 pytest_plugins = ("fixtures.zenith_fixtures")

@@ -9,7 +9,8 @@ pytest_plugins = ("fixtures.zenith_fixtures")
 # it only checks next_multixact_id field in restored pg_control,
 # since we don't have functions to check multixact internals.
 #
-def test_multixact(pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin, zenith_cli, base_dir):
+def test_multixact(pageserver: ZenithPageserver, postgres: PostgresFactory,
+                    pg_bin, zenith_cli, base_dir, test_output_dir):
    # Create a branch for us
    zenith_cli.run(["branch", "test_multixact", "empty"])
    pg = postgres.create_start('test_multixact')
@@ -63,3 +64,6 @@ def test_multixact(pageserver: ZenithPageserver, postgres: PostgresFactory, pg_b

    # Check that we restored pg_controlfile correctly
    assert next_multixact_id_new == next_multixact_id
+
+    # Check that we restore the content of the datadir correctly
+    check_restored_datadir_content(zenith_cli, test_output_dir, pg_new, pageserver.service_port.pg)
--- a/test_runner/batch_others/test_old_request_lsn.py
+++ b/test_runner/batch_others/test_old_request_lsn.py
@@ -0,0 +1,65 @@
+from contextlib import closing
+
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+#
+# Test where Postgres generates a lot of WAL, and it's garbage collected away, but
+# no pages are evicted so that Postgres uses an old LSN in a GetPage request.
+# We had a bug where the page server failed to find the page version because it
+# thought it was already garbage collected away, because the LSN in the GetPage
+# request was very old and the WAL from that time was indeed already removed.
+# In reality, the LSN on a GetPage request coming from a primary server is
+# just a hint that the page hasn't been modified since that LSN, and the page
+# server should return the latest page version regardless of the LSN.
+#
+def test_old_request_lsn(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_old_request_lsn", "empty"])
+    pg = postgres.create_start('test_old_request_lsn')
+    print('postgres is running on test_old_request_lsn branch')
+
+    pg_conn = pg.connect()
+    cur = pg_conn.cursor()
+
+    # Get the timeline ID of our branch. We need it for the 'do_gc' command
+    cur.execute("SHOW zenith.zenith_timeline")
+    timeline = cur.fetchone()[0]
+
+    psconn = pageserver.connect()
+    pscur = psconn.cursor()
+
+    # Create table, and insert some rows. Make it big enough that it doesn't fit in
+    # shared_buffers.
+    cur.execute('CREATE TABLE foo (id int4 PRIMARY KEY, val int, t text)')
+    cur.execute('''
+        INSERT INTO foo
+            SELECT g, 1, 'long string to consume some space' || g
+            FROM generate_series(1, 100000) g
+    ''')
+
+    # Verify that the table is larger than shared_buffers, so that the SELECT below
+    # will cause GetPage requests.
+    cur.execute('''
+        select setting::int * pg_size_bytes(unit) as shared_buffers, pg_relation_size('foo') as tbl_ize
+        from pg_settings where name = 'shared_buffers'
+    ''')
+    row = cur.fetchone()
+    print("shared_buffers is {}, table size {}", row[0], row[1]);
+    assert int(row[0]) < int(row[1])
+
+    cur.execute('VACUUM foo');
+
+    # Make a lot of updates on a single row, generating a lot of WAL. Trigger
+    # garbage collections so that the page server will remove old page versions.
+    for i in range(10):
+        pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+        for j in range(100):
+            cur.execute('UPDATE foo SET val = val + 1 WHERE id = 1;');
+
+    # All (or at least most of) the updates should've been on the same page, so
+    # that we haven't had to evict any dirty pages for a long time. Now run
+    # a query that sends GetPage@LSN requests with the old LSN.
+    cur.execute("SELECT COUNT(*), SUM(val) FROM foo");
+    assert cur.fetchone() == (100000, 101000)
--- a/test_runner/batch_others/test_pageserver_api.py
+++ b/test_runner/batch_others/test_pageserver_api.py
@@ -1,19 +1,20 @@
 import json
-import uuid
+from uuid import uuid4
 import pytest
 import psycopg2
-from fixtures.zenith_fixtures import ZenithPageserver
+import requests
+from fixtures.zenith_fixtures import ZenithPageserver, ZenithPageserverHttpClient

 pytest_plugins = ("fixtures.zenith_fixtures")


-def test_status(pageserver):
+def test_status_psql(pageserver):
    assert pageserver.safe_psql('status') == [
        ('hello world', ),
    ]


-def test_branch_list(pageserver: ZenithPageserver, zenith_cli):
+def test_branch_list_psql(pageserver: ZenithPageserver, zenith_cli):
    # Create a branch for us
    zenith_cli.run(["branch", "test_branch_list_main", "empty"])

@@ -52,7 +53,7 @@ def test_branch_list(pageserver: ZenithPageserver, zenith_cli):
    conn.close()


-def test_tenant_list(pageserver: ZenithPageserver, zenith_cli):
+def test_tenant_list_psql(pageserver: ZenithPageserver, zenith_cli):
    res = zenith_cli.run(["tenant", "list"])
    res.check_returncode()
    tenants = res.stdout.splitlines()
@@ -66,7 +67,7 @@ def test_tenant_list(pageserver: ZenithPageserver, zenith_cli):
        cur.execute(f'tenant_create {pageserver.initial_tenant}')

    # create one more tenant
-    tenant1 = uuid.uuid4().hex
+    tenant1 = uuid4().hex
    cur.execute(f'tenant_create {tenant1}')

    cur.execute('tenant_list')
@@ -74,3 +75,32 @@ def test_tenant_list(pageserver: ZenithPageserver, zenith_cli):
    # compare tenants list
    new_tenants = sorted(json.loads(cur.fetchone()[0]))
    assert sorted([pageserver.initial_tenant, tenant1]) == new_tenants
+
+
+def check_client(client: ZenithPageserverHttpClient, initial_tenant: str):
+    client.check_status()
+
+    # check initial tenant is there
+    assert initial_tenant in set(client.tenant_list())
+
+    # create new tenant and check it is also there
+    tenant_id = uuid4()
+    client.tenant_create(tenant_id)
+    assert tenant_id.hex in set(client.tenant_list())
+
+    # create branch
+    branch_name = uuid4().hex
+    client.branch_create(tenant_id, branch_name, "main")
+
+    # check it is there
+    assert branch_name in {b['name'] for b in client.branch_list(tenant_id)}
+
+
+def test_pageserver_http_api_client(pageserver: ZenithPageserver):
+    client = pageserver.http_client()
+    check_client(client, pageserver.initial_tenant)
+
+
+def test_pageserver_http_api_client_auth_enabled(pageserver_auth_enabled: ZenithPageserver):
+    client = pageserver_auth_enabled.http_client(auth_token=pageserver_auth_enabled.auth_keys.generate_management_token())
+    check_client(client, pageserver_auth_enabled.initial_tenant)
--- a/test_runner/batch_others/test_pgbench.py
+++ b/test_runner/batch_others/test_pgbench.py
@@ -12,5 +12,5 @@ def test_pgbench(postgres: PostgresFactory, pg_bin, zenith_cli):

    connstr = pg.connstr()

-    pg_bin.run_capture(['pgbench', '-i', connstr])
-    pg_bin.run_capture(['pgbench'] + '-c 10 -T 5 -P 1 -M prepared'.split() + [connstr])
+    pg_bin.run_capture(['pgbench', '-i', '-s', '100', connstr])
+    pg_bin.run_capture(['pgbench'] + '-c 1 -N -T 100 -P 1 -M prepared'.split() + [connstr])
--- a/test_runner/batch_others/test_restart_compute.py
+++ b/test_runner/batch_others/test_restart_compute.py
@@ -9,7 +9,10 @@ pytest_plugins = ("fixtures.zenith_fixtures")
 #
 # Test restarting and recreating a postgres instance
 #
-@pytest.mark.parametrize('with_wal_acceptors', [False, True])
+# XXX: with_wal_acceptors=True fails now, would be fixed with
+# `postgres --sync-walkeepers` patches.
+#
+@pytest.mark.parametrize('with_wal_acceptors', [False])
 def test_restart_compute(
        zenith_cli,
        pageserver: ZenithPageserver,
@@ -31,31 +34,58 @@ def test_restart_compute(

    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
-            # Create table, and insert a row
-            cur.execute('CREATE TABLE foo (t text)')
-            cur.execute("INSERT INTO foo VALUES ('bar')")
+            cur.execute('CREATE TABLE t(key int primary key, value text)')
+            cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
+            cur.execute('SELECT sum(key) FROM t')
+            r = cur.fetchone()
+            assert r == (5000050000, )
+            print("res = ", r)

-    # Stop and restart the Postgres instance
+    # Remove data directory and restart
    pg.stop_and_destroy().create_start('test_restart_compute',
                                       wal_acceptors=wal_acceptor_connstrs)

+
    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
            # We can still see the row
-            cur.execute('SELECT count(*) FROM foo')
-            assert cur.fetchone() == (1, )
+            cur.execute('SELECT sum(key) FROM t')
+            r = cur.fetchone()
+            assert r == (5000050000, )
+            print("res = ", r)

            # Insert another row
-            cur.execute("INSERT INTO foo VALUES ('bar2')")
-            cur.execute('SELECT count(*) FROM foo')
-            assert cur.fetchone() == (2, )
+            cur.execute("INSERT INTO t VALUES (100001, 'payload2')")
+            cur.execute('SELECT count(*) FROM t')

-    # Stop, and destroy the Postgres instance. Then recreate and restart it.
+            r = cur.fetchone()
+            assert r == (100001, )
+            print("res = ", r)
+
+    # Again remove data directory and restart
+    pg.stop_and_destroy().create_start('test_restart_compute',
+                                       wal_acceptors=wal_acceptor_connstrs)
+
+    # That select causes lots of FPI's and increases probability of wakeepers
+    # lagging behind after query completion
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            # We can still see the rows
+            cur.execute('SELECT count(*) FROM t')
+
+            r = cur.fetchone()
+            assert r == (100001, )
+            print("res = ", r)
+
+    # And again remove data directory and restart
    pg.stop_and_destroy().create_start('test_restart_compute',
                                       wal_acceptors=wal_acceptor_connstrs)

    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
            # We can still see the rows
-            cur.execute('SELECT count(*) FROM foo')
-            assert cur.fetchone() == (2, )
+            cur.execute('SELECT count(*) FROM t')
+
+            r = cur.fetchone()
+            assert r == (100001, )
+            print("res = ", r)
--- a/test_runner/batch_others/test_snapfiles_gc.py
+++ b/test_runner/batch_others/test_snapfiles_gc.py
@@ -6,19 +6,19 @@ pytest_plugins = ("fixtures.zenith_fixtures")

 def print_gc_result(row):
    print("GC duration {elapsed} ms".format_map(row));
-    print("  REL    total: {snapshot_relfiles_total}, needed_by_cutoff {snapshot_relfiles_needed_by_cutoff}, needed_by_branches: {snapshot_relfiles_needed_by_branches}, not_updated: {snapshot_relfiles_not_updated}, removed: {snapshot_relfiles_removed}, dropped: {snapshot_relfiles_dropped}".format_map(row))
-    print("  NONREL total: {snapshot_nonrelfiles_total}, needed_by_cutoff {snapshot_nonrelfiles_needed_by_cutoff}, needed_by_branches: {snapshot_nonrelfiles_needed_by_branches}, not_updated: {snapshot_nonrelfiles_not_updated}, removed: {snapshot_nonrelfiles_removed}, dropped: {snapshot_nonrelfiles_dropped}".format_map(row))
+    print("  REL    total: {layer_relfiles_total}, needed_by_cutoff {layer_relfiles_needed_by_cutoff}, needed_by_branches: {layer_relfiles_needed_by_branches}, not_updated: {layer_relfiles_not_updated}, removed: {layer_relfiles_removed}, dropped: {layer_relfiles_dropped}".format_map(row))
+    print("  NONREL total: {layer_nonrelfiles_total}, needed_by_cutoff {layer_nonrelfiles_needed_by_cutoff}, needed_by_branches: {layer_nonrelfiles_needed_by_branches}, not_updated: {layer_nonrelfiles_not_updated}, removed: {layer_nonrelfiles_removed}, dropped: {layer_nonrelfiles_dropped}".format_map(row))


 #
-# Test Garbage Collection of old snapshot files
+# Test Garbage Collection of old layer files
 #
 # This test is pretty tightly coupled with the current implementation of layered
 # storage, in layered_repository.rs.
 #
-def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin):
-    zenith_cli.run(["branch", "test_snapfiles_gc", "empty"])
-    pg = postgres.create_start('test_snapfiles_gc')
+def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin):
+    zenith_cli.run(["branch", "test_layerfiles_gc", "empty"])
+    pg = postgres.create_start('test_layerfiles_gc')

    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
@@ -55,8 +55,8 @@ def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin):
                    row = pscur.fetchone()
                    print_gc_result(row);
                    # remember the number of files
-                    snapshot_relfiles_remain = row['snapshot_relfiles_total'] - row['snapshot_relfiles_removed']
-                    assert snapshot_relfiles_remain > 0
+                    layer_relfiles_remain = row['layer_relfiles_total'] - row['layer_relfiles_removed']
+                    assert layer_relfiles_remain > 0

                    # Insert a row.
                    print("Inserting one row and running GC")
@@ -64,12 +64,12 @@ def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin):
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print_gc_result(row);
-                    assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain + 1
-                    assert row['snapshot_relfiles_removed'] == 1
-                    assert row['snapshot_relfiles_dropped'] == 0
+                    assert row['layer_relfiles_total'] == layer_relfiles_remain + 1
+                    assert row['layer_relfiles_removed'] == 1
+                    assert row['layer_relfiles_dropped'] == 0

                    # Insert two more rows and run GC.
-                    # This should create a new snapshot file with the new contents, and
+                    # This should create a new layer file with the new contents, and
                    # remove the old one.
                    print("Inserting two more rows and running GC")
                    cur.execute("INSERT INTO foo VALUES (2)")
@@ -78,11 +78,11 @@ def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin):
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print_gc_result(row);
-                    assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain + 1
-                    assert row['snapshot_relfiles_removed'] == 1
-                    assert row['snapshot_relfiles_dropped'] == 0
+                    assert row['layer_relfiles_total'] == layer_relfiles_remain + 1
+                    assert row['layer_relfiles_removed'] == 1
+                    assert row['layer_relfiles_dropped'] == 0

-                    # Do it again. Should again create a new snapshot file and remove old one.
+                    # Do it again. Should again create a new layer file and remove old one.
                    print("Inserting two more rows and running GC")
                    cur.execute("INSERT INTO foo VALUES (2)")
                    cur.execute("INSERT INTO foo VALUES (3)")
@@ -90,18 +90,18 @@ def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin):
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print_gc_result(row);
-                    assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain + 1
-                    assert row['snapshot_relfiles_removed'] == 1
-                    assert row['snapshot_relfiles_dropped'] == 0
+                    assert row['layer_relfiles_total'] == layer_relfiles_remain + 1
+                    assert row['layer_relfiles_removed'] == 1
+                    assert row['layer_relfiles_dropped'] == 0

                    # Run GC again, with no changes in the database. Should not remove anything.
                    print("Run GC again, with nothing to do")
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print_gc_result(row);
-                    assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain
-                    assert row['snapshot_relfiles_removed'] == 0
-                    assert row['snapshot_relfiles_dropped'] == 0
+                    assert row['layer_relfiles_total'] == layer_relfiles_remain
+                    assert row['layer_relfiles_removed'] == 0
+                    assert row['layer_relfiles_dropped'] == 0

                    #
                    # Test DROP TABLE checks that relation data and metadata was deleted by GC from object storage
@@ -114,11 +114,11 @@ def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin):
                    print_gc_result(row);

                    # Each relation fork is counted separately, hence 3.
-                    assert row['snapshot_relfiles_dropped'] == 3
+                    assert row['layer_relfiles_dropped'] == 3

-                    # The catalog updates also create new snapshot files of the catalogs, which
+                    # The catalog updates also create new layer files of the catalogs, which
                    # are counted as 'removed'
-                    assert row['snapshot_relfiles_removed'] > 0
+                    assert row['layer_relfiles_removed'] > 0

                    # TODO: perhaps we should count catalog and user relations separately,
                    # to make this kind of testing more robust
--- a/test_runner/batch_others/test_timeline_size.py
+++ b/test_runner/batch_others/test_timeline_size.py
@@ -0,0 +1,39 @@
+from contextlib import closing
+from uuid import UUID
+import psycopg2.extras
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+
+def test_timeline_size(
+    zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin
+):
+    # Branch at the point where only 100 rows were inserted
+    zenith_cli.run(["branch", "test_timeline_size", "empty"])
+
+    client = pageserver.http_client()
+    res = client.branch_detail(UUID(pageserver.initial_tenant), "test_timeline_size")
+    assert res["current_logical_size"] == res["current_logical_size_non_incremental"]
+
+    pgmain = postgres.create_start("test_timeline_size")
+    print("postgres is running on 'test_timeline_size' branch")
+
+    with closing(pgmain.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute("SHOW zenith.zenith_timeline")
+
+            # Create table, and insert the first 100 rows
+            cur.execute("CREATE TABLE foo (t text)")
+            cur.execute(
+                """
+                INSERT INTO foo
+                    SELECT 'long string to consume some space' || g
+                    FROM generate_series(1, 10) g
+            """
+            )
+
+            res = client.branch_detail(UUID(pageserver.initial_tenant), "test_timeline_size")
+            assert res["current_logical_size"] == res["current_logical_size_non_incremental"]
+            cur.execute("TRUNCATE foo")
+
+            res = client.branch_detail(UUID(pageserver.initial_tenant), "test_timeline_size")
+            assert res["current_logical_size"] == res["current_logical_size_non_incremental"]
--- a/test_runner/batch_others/test_twophase.py
+++ b/test_runner/batch_others/test_twophase.py
@@ -1,6 +1,6 @@
 import os

-from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver, PgBin


 pytest_plugins = ("fixtures.zenith_fixtures")
@@ -9,7 +9,7 @@ pytest_plugins = ("fixtures.zenith_fixtures")
 #
 # Test branching, when a transaction is in prepared state
 #
-def test_twophase(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+def test_twophase(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin: PgBin):
    zenith_cli.run(["branch", "test_twophase", "empty"])

    pg = postgres.create_start('test_twophase', config_lines=['max_prepared_transactions=5'])
@@ -59,9 +59,8 @@ def test_twophase(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFa
    # Create a branch with the transaction in prepared state
    zenith_cli.run(["branch", "test_twophase_prepared", "test_twophase"])

-    # Create compute node, but don't start.
-    # We want to observe pgdata before postgres starts
-    pg2 = postgres.create(
+    # Start compute on the new branch
+    pg2 = postgres.create_start(
        'test_twophase_prepared',
        config_lines=['max_prepared_transactions=5'],
    )
@@ -71,7 +70,6 @@ def test_twophase(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFa
    print(twophase_files2)
    assert twophase_files2.sort() == twophase_files.sort()

-    pg2 = pg2.start()
    conn2 = pg2.connect()
    cur2 = conn2.cursor()

--- a/test_runner/batch_others/test_vm_bits.py
+++ b/test_runner/batch_others/test_vm_bits.py
@@ -0,0 +1,79 @@
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+#
+# Test that the VM bit is cleared correctly at a HEAP_DELETE and
+# HEAP_UPDATE record.
+#
+def test_vm_bit_clear(pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin, zenith_cli, base_dir):
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_vm_bit_clear", "empty"])
+    pg = postgres.create_start('test_vm_bit_clear')
+
+    print("postgres is running on 'test_vm_bit_clear' branch")
+    pg_conn = pg.connect()
+    cur = pg_conn.cursor()
+
+    # Install extension containing function needed for test
+    cur.execute('CREATE EXTENSION zenith_test_utils')
+
+    # Create a test table and freeze it to set the VM bit.
+    cur.execute('CREATE TABLE vmtest_delete (id integer PRIMARY KEY)')
+    cur.execute('INSERT INTO vmtest_delete VALUES (1)')
+    cur.execute('VACUUM FREEZE vmtest_delete')
+
+    cur.execute('CREATE TABLE vmtest_update (id integer PRIMARY KEY)')
+    cur.execute('INSERT INTO vmtest_update SELECT g FROM generate_series(1, 1000) g')
+    cur.execute('VACUUM FREEZE vmtest_update')
+
+    # DELETE and UDPATE the rows.
+    cur.execute('DELETE FROM vmtest_delete WHERE id = 1')
+    cur.execute('UPDATE vmtest_update SET id = 5000 WHERE id = 1')
+
+    # Branch at this point, to test that later
+    zenith_cli.run(["branch", "test_vm_bit_clear_new", "test_vm_bit_clear"])
+
+    # Clear the buffer cache, to force the VM page to be re-fetched from
+    # the page server
+    cur.execute('SELECT clear_buffer_cache()')
+
+    # Check that an index-only scan doesn't see the deleted row. If the
+    # clearing of the VM bit was not replayed correctly, this would incorrectly
+    # return deleted row.
+    cur.execute('''
+    set enable_seqscan=off;
+    set enable_indexscan=on;
+    set enable_bitmapscan=off;
+    ''')
+
+    cur.execute('SELECT * FROM vmtest_delete WHERE id = 1')
+    assert(cur.fetchall() == []);
+    cur.execute('SELECT * FROM vmtest_update WHERE id = 1')
+    assert(cur.fetchall() == []);
+
+    cur.close()
+
+
+    # Check the same thing on the branch that we created right after the DELETE
+    #
+    # As of this writing, the code in smgrwrite() creates a full-page image whenever
+    # a dirty VM page is evicted. If the VM bit was not correctly cleared by the
+    # earlier WAL record, the full-page image hides the problem. Starting a new
+    # server at the right point-in-time avoids that full-page image.
+    pg_new = postgres.create_start('test_vm_bit_clear_new')
+
+    print("postgres is running on 'test_vm_bit_clear_new' branch")
+    pg_new_conn = pg_new.connect()
+    cur_new = pg_new_conn.cursor()
+
+    cur_new.execute('''
+    set enable_seqscan=off;
+    set enable_indexscan=on;
+    set enable_bitmapscan=off;
+    ''')
+
+    cur_new.execute('SELECT * FROM vmtest_delete WHERE id = 1')
+    assert(cur_new.fetchall() == []);
+    cur_new.execute('SELECT * FROM vmtest_update WHERE id = 1')
+    assert(cur_new.fetchall() == []);
--- a/test_runner/batch_others/test_wal_acceptor_async.py
+++ b/test_runner/batch_others/test_wal_acceptor_async.py
@@ -0,0 +1,154 @@
+import asyncio
+import asyncpg
+import random
+
+from fixtures.zenith_fixtures import WalAcceptor, WalAcceptorFactory, ZenithPageserver, PostgresFactory, Postgres
+from typing import List
+from fixtures.utils import debug_print
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+class BankClient(object):
+    def __init__(self, conn: asyncpg.Connection, n_accounts, init_amount):
+        self.conn: asyncpg.Connection = conn
+        self.n_accounts = n_accounts
+        self.init_amount = init_amount
+
+    async def initdb(self):
+        await self.conn.execute('DROP TABLE IF EXISTS bank_accs')
+        await self.conn.execute('CREATE TABLE bank_accs(uid int primary key, amount int)')
+        await self.conn.execute('''
+            INSERT INTO bank_accs
+            SELECT *, $1 FROM generate_series(0, $2)
+        ''', self.init_amount, self.n_accounts - 1)
+        await self.conn.execute('DROP TABLE IF EXISTS bank_log')
+        await self.conn.execute('CREATE TABLE bank_log(from_uid int, to_uid int, amount int)')
+        
+        # TODO: Remove when https://github.com/zenithdb/zenith/issues/644 is fixed
+        await self.conn.execute('ALTER TABLE bank_accs SET (autovacuum_enabled = false)')
+        await self.conn.execute('ALTER TABLE bank_log SET (autovacuum_enabled = false)')
+
+    async def check_invariant(self):
+        row = await self.conn.fetchrow('SELECT sum(amount) AS sum FROM bank_accs')
+        assert row['sum'] == self.n_accounts * self.init_amount
+
+async def bank_transfer(conn: asyncpg.Connection, from_uid, to_uid, amount):
+    # avoid deadlocks by sorting uids
+    if from_uid > to_uid:
+        from_uid, to_uid, amount = to_uid, from_uid, -amount
+
+    async with conn.transaction():
+        await conn.execute(
+            'UPDATE bank_accs SET amount = amount + ($1) WHERE uid = $2',
+            amount, to_uid,
+        )
+        await conn.execute(
+            'UPDATE bank_accs SET amount = amount - ($1) WHERE uid = $2',
+            amount, from_uid,
+        )
+        await conn.execute('INSERT INTO bank_log VALUES ($1, $2, $3)',
+            from_uid, to_uid, amount,
+        )
+
+class WorkerStats(object):
+    def __init__(self, n_workers):
+        self.counters = [0] * n_workers
+        self.running = True
+
+    def reset(self):
+        self.counters = [0] * len(self.counters)
+
+    def inc_progress(self, worker_id):
+        self.counters[worker_id] += 1
+
+    def check_progress(self):
+        debug_print("Workers progress: {}".format(self.counters))
+
+        # every worker should finish at least one tx
+        assert all(cnt > 0 for cnt in self.counters)
+
+        progress = sum(self.counters)
+        print('All workers made {} transactions'.format(progress))
+
+
+async def run_random_worker(stats: WorkerStats, pg: Postgres, worker_id, n_accounts, max_transfer):
+    pg_conn = await pg.connect_async()
+    debug_print('Started worker {}'.format(worker_id))
+
+    while stats.running:
+        from_uid = random.randint(0, n_accounts - 1)
+        to_uid = (from_uid + random.randint(1, n_accounts - 1)) % n_accounts
+        amount = random.randint(1, max_transfer)
+
+        await bank_transfer(pg_conn, from_uid, to_uid, amount)
+        stats.inc_progress(worker_id)
+
+        debug_print('Executed transfer({}) {} => {}'.format(amount, from_uid, to_uid))
+
+    debug_print('Finished worker {}'.format(worker_id))
+
+    await pg_conn.close()
+
+
+# This test will run several iterations and check progress in each of them.
+# On each iteration 1 acceptor is stopped, and 2 others should allow
+# background workers execute transactions. In the end, state should remain
+# consistent.
+async def run_restarts_under_load(pg: Postgres, acceptors: List[WalAcceptor], n_workers=10):
+    n_accounts = 100
+    init_amount = 100000
+    max_transfer = 100
+    period_time = 10
+    iterations = 6
+
+    pg_conn = await pg.connect_async()
+    bank = BankClient(pg_conn, n_accounts=n_accounts, init_amount=init_amount)
+    # create tables and initial balances
+    await bank.initdb()
+
+    stats = WorkerStats(n_workers)
+    workers = []
+    for worker_id in range(n_workers):
+        worker = run_random_worker(stats, pg, worker_id, bank.n_accounts, max_transfer)
+        workers.append(asyncio.create_task(worker))
+
+
+    for it in range(iterations):
+        victim = acceptors[it % len(acceptors)]
+        victim.stop()
+
+        # wait for transactions that could have started and finished before
+        # victim acceptor was stopped
+        await asyncio.sleep(1)
+
+        stats.reset()
+        await asyncio.sleep(period_time)
+        # assert that at least one transaction has completed in every worker
+        stats.check_progress()
+
+        victim.start()
+
+    print('Iterations are finished, exiting coroutines...')
+    stats.running = False
+    # await all workers
+    await asyncio.gather(*workers)
+    # assert balances sum hasn't changed
+    await bank.check_invariant()
+    await pg_conn.close()
+
+
+# restart acceptors one by one, while executing and validating bank transactions
+def test_restarts_under_load(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory,
+                             wa_factory: WalAcceptorFactory):
+
+    wa_factory.start_n_new(3)
+
+    zenith_cli.run(["branch", "test_wal_acceptors_restarts_under_load", "empty"])
+    pg = postgres.create_start('test_wal_acceptors_restarts_under_load',
+                               wal_acceptors=wa_factory.get_connstrs())
+
+    asyncio.run(run_restarts_under_load(pg, wa_factory.instances))
+
+    # TODO: Remove when https://github.com/zenithdb/zenith/issues/644 is fixed
+    pg.stop()
--- a/test_runner/batch_pg_regress/test_pg_regress.py
+++ b/test_runner/batch_pg_regress/test_pg_regress.py
@@ -1,7 +1,7 @@
 import os

 from fixtures.utils import mkdir_if_needed
-from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver, check_restored_datadir_content

 pytest_plugins = ("fixtures.zenith_fixtures")

@@ -49,3 +49,10 @@ def test_pg_regress(pageserver: ZenithPageserver, postgres: PostgresFactory, pg_
    # logs the exact same data to `regression.out` anyway.
    with capsys.disabled():
        pg_bin.run(pg_regress_command, env=env, cwd=runpath)
+
+        # checkpoint one more time to ensure that the lsn we get is the latest one
+        pg.safe_psql('CHECKPOINT')
+        lsn = pg.safe_psql('select pg_current_wal_insert_lsn()')[0][0]
+
+        # Check that we restore the content of the datadir correctly
+        check_restored_datadir_content(zenith_cli, test_output_dir, pg, pageserver.service_port.pg)
--- a/test_runner/batch_pg_regress/test_zenith_regress.py
+++ b/test_runner/batch_pg_regress/test_zenith_regress.py
@@ -1,13 +1,13 @@
 import os

 from fixtures.utils import mkdir_if_needed
-from fixtures.zenith_fixtures import PostgresFactory
+from fixtures.zenith_fixtures import PageserverPort, PostgresFactory, check_restored_datadir_content

 pytest_plugins = ("fixtures.zenith_fixtures")


 def test_zenith_regress(postgres: PostgresFactory, pg_bin, zenith_cli, test_output_dir, pg_distrib_dir,
-                        base_dir, capsys):
+                        base_dir, capsys, pageserver_port: PageserverPort):

    # Create a branch for us
    zenith_cli.run(["branch", "test_zenith_regress", "empty"])
@@ -50,3 +50,10 @@ def test_zenith_regress(postgres: PostgresFactory, pg_bin, zenith_cli, test_outp
    # logs the exact same data to `regression.out` anyway.
    with capsys.disabled():
        pg_bin.run(pg_regress_command, env=env, cwd=runpath)
+
+        # checkpoint one more time to ensure that the lsn we get is the latest one
+        pg.safe_psql('CHECKPOINT')
+        lsn = pg.safe_psql('select pg_current_wal_insert_lsn()')[0][0]
+
+        # Check that we restore the content of the datadir correctly
+        check_restored_datadir_content(zenith_cli, test_output_dir, pg, pageserver_port.pg)
--- a/test_runner/fixtures/benchmark_fixture.py
+++ b/test_runner/fixtures/benchmark_fixture.py
@@ -0,0 +1,192 @@
+from pprint import pprint
+
+import os
+import re
+import timeit
+import pathlib
+import uuid
+import psycopg2
+import pytest
+from _pytest.config import Config
+from _pytest.runner import CallInfo
+from _pytest.terminal import TerminalReporter
+import shutil
+import signal
+import subprocess
+import time
+
+from contextlib import contextmanager
+from contextlib import closing
+from pathlib import Path
+from dataclasses import dataclass
+
+# Type-related stuff
+from psycopg2.extensions import connection as PgConnection
+from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar, cast
+from typing_extensions import Literal
+
+from .utils import (get_self_dir, mkdir_if_needed, subprocess_capture)
+
+"""
+This file contains fixtures for micro-benchmarks.
+
+To use, declare the 'zenbenchmark' fixture in the test function. Run the
+bencmark, and then record the result by calling zenbenchmark.record. For example:
+
+import timeit
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
+
+def test_mybench(postgres: PostgresFactory, pageserver: ZenithPageserver, zenbenchmark):
+
+    # Initialize the test
+    ...
+    
+    # Run the test, timing how long it takes
+    with zenbenchmark.record_duration('test_query'):
+        cur.execute('SELECT test_query(...)')
+
+    # Record another measurement
+    zenbenchmark.record('speed_of_light', 300000, 'km/s')
+
+
+You can measure multiple things in one test, and record each one with a separate
+call to zenbenchmark. For example, you could time the bulk loading that happens
+in the test initialization, or measure disk usage after the test query.
+
+"""
+
+
+# All the results are collected in this list, as a tuple:
+# (test_name: str, metric_name: str, metric_value: float, unit: str)
+#
+# TODO: It would perhaps be better to store the results as additional
+# properties in the pytest TestReport objects, to make them visible to
+# other pytest tools.
+global zenbenchmark_results
+zenbenchmark_results = []
+
+class ZenithBenchmarkResults:
+    """ An object for recording benchmark results. """
+    def __init__(self):
+        self.results = []
+
+    def record(self, test_name: str, metric_name: str, metric_value: float, unit: str):
+        """
+        Record a benchmark result.
+        """
+
+        self.results.append((test_name, metric_name, metric_value, unit))
+
+# Session scope fixture that initializes the results object
+@pytest.fixture(autouse=True, scope='session')
+def zenbenchmark_global(request) -> Iterator[ZenithBenchmarkResults]:
+    """
+    This is a python decorator for benchmark fixtures
+    """
+    global zenbenchmark_results
+    zenbenchmark_results = ZenithBenchmarkResults()
+
+    yield zenbenchmark_results
+
+class ZenithBenchmarker:
+    """
+    An object for recording benchmark results. This is created for each test
+    function by the zenbenchmark fixture
+    """
+    def __init__(self, results, request):
+        self.results = results
+        self.request = request
+
+    def record(self, metric_name: str, metric_value: float, unit: str):
+        """
+        Record a benchmark result.
+        """
+        self.results.record(self.request.node.name, metric_name, metric_value, unit)
+
+
+    @contextmanager
+    def record_duration(self, metric_name):
+        """
+        Record a duration. Usage:
+        
+        with zenbenchmark.record_duration('foobar_runtime'):
+            foobar()   # measure this
+        
+        """
+        start = timeit.default_timer()
+        yield
+        end = timeit.default_timer()
+
+        self.results.record(self.request.node.name, metric_name, end - start, 's')
+
+    def get_io_writes(self, pageserver) -> int:
+        """
+        Fetch the "cumulative # of bytes written" metric from the pageserver
+        """
+        # Fetch all the exposed prometheus metrics from page server
+        all_metrics = pageserver.http_client().get_metrics()
+        # Use a regular expression to extract the one we're interested in
+        #
+        # TODO: If we start to collect more of the prometheus metrics in the
+        # performance test suite like this, we should refactor this to load and
+        # parse all the metrics into a more convenient structure in one go.
+        #
+        # The metric should be an integer, as it's a number of bytes. But in general
+        # all prometheus metrics are floats. So to be pedantic, read it as a float
+        # and round to integer.
+        matches = re.search(r'pageserver_disk_io_bytes{io_operation="write"} (\S+)', all_metrics)
+        return int(round(float(matches.group(1))))
+
+    @contextmanager
+    def record_pageserver_writes(self, pageserver, metric_name):
+        """
+        Record bytes written by the pageserver during a test.
+        """
+        before = self.get_io_writes(pageserver)
+        yield
+        after = self.get_io_writes(pageserver)
+
+        self.results.record(self.request.node.name, metric_name, round((after - before) / (1024 * 1024)), 'MB')
+
+@pytest.fixture(scope='function')
+def zenbenchmark(zenbenchmark_global, request) -> Iterator[ZenithBenchmarker]:
+    """
+    This is a python decorator for benchmark fixtures. It contains functions for
+    recording measurements, and prints them out at the end.
+    """
+    benchmarker = ZenithBenchmarker(zenbenchmark_global, request)
+    yield benchmarker
+
+
+# Hook to print the results at the end
+@pytest.hookimpl(hookwrapper=True)
+def pytest_terminal_summary(
+    terminalreporter: TerminalReporter, exitstatus: int, config: Config
+):
+    yield
+
+    global zenbenchmark_results
+
+    if not zenbenchmark_results:
+        return
+
+    terminalreporter.section('Benchmark results', "-")
+
+    for result in zenbenchmark_results.results:
+        func = result[0]
+        metric_name = result[1]
+        metric_value = result[2]
+        unit = result[3]
+
+        terminalreporter.write("{}.{}: ".format(func, metric_name))
+
+        if unit == 'MB':
+            terminalreporter.write("{0:,.0f}".format(metric_value), green=True)
+        elif unit == 's':
+            terminalreporter.write("{0:,.3f}".format(metric_value), green=True)
+        else:
+            terminalreporter.write("{0:,.4f}".format(metric_value), green=True)
+
+        terminalreporter.line(" {}".format(unit))
--- a/test_runner/fixtures/utils.py
+++ b/test_runner/fixtures/utils.py
@@ -14,10 +14,11 @@ def mkdir_if_needed(path: str) -> None:

    Note this won't try to create intermediate directories.
    """
-    if os.path.exists(path):
-        assert os.path.isdir(path)
-        return
-    os.mkdir(path)
+    try:
+        os.mkdir(path)
+    except FileExistsError:
+        pass
+    assert os.path.isdir(path)


 def subprocess_capture(capture_dir: str, cmd: List[str], **kwargs: Any) -> None:
@@ -53,3 +54,11 @@ def global_counter() -> int:
    global _global_counter
    _global_counter += 1
    return _global_counter
+
+def debug_print(*args, **kwargs) -> None:
+    """ Print to the console if TEST_DEBUG_PRINT is set in env.
+    
+    All parameters are passed to print().
+    """
+    if os.environ.get('TEST_DEBUG_PRINT') is not None:
+        print(*args, **kwargs)
--- a/test_runner/fixtures/zenith_fixtures.py
+++ b/test_runner/fixtures/zenith_fixtures.py
@@ -1,12 +1,18 @@
+from dataclasses import dataclass
+from functools import cached_property
+import asyncpg
 import os
 import pathlib
 import uuid
+import jwt
 import psycopg2
 import pytest
 import shutil
 import signal
 import subprocess
 import time
+import filecmp
+import difflib

 from contextlib import closing
 from pathlib import Path
@@ -17,6 +23,8 @@ from psycopg2.extensions import connection as PgConnection
 from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar, cast
 from typing_extensions import Literal

+import requests
+
 from .utils import (get_self_dir, mkdir_if_needed, subprocess_capture)
 """
 This file contains pytest fixtures. A fixture is a test resource that can be
@@ -42,7 +50,27 @@ Fn = TypeVar('Fn', bound=Callable[..., Any])
 DEFAULT_OUTPUT_DIR = 'test_output'
 DEFAULT_POSTGRES_DIR = 'tmp_install'

-DEFAULT_PAGESERVER_PORT = 64000
+BASE_PORT = 15000
+WORKER_PORT_NUM = 100
+
+def pytest_configure(config):
+    """
+    Ensure that no unwanted daemons are running before we start testing.
+    Check that we do not owerflow available ports range.
+    """
+    numprocesses = config.getoption('numprocesses')
+    if numprocesses is not None and BASE_PORT + numprocesses * WORKER_PORT_NUM > 32768: # do not use ephemeral ports
+         raise Exception('Too many workers configured. Cannot distrubute ports for services.')
+
+    # does not use -c as it is not supported on macOS
+    cmd = ['pgrep', 'pageserver|postgres|wal_acceptor']
+    result = subprocess.run(cmd, stdout=subprocess.DEVNULL)
+    if result.returncode == 0:
+        # returncode of 0 means it found something.
+        # This is bad; we don't want any of those processes polluting the
+        # result of the test.
+        # NOTE this shows as an internal pytest error, there might be a better way
+        raise Exception('found interfering processes running')


 def determine_scope(fixture_name: str, config: Any) -> str:
@@ -67,20 +95,6 @@ def zenfixture(func: Fn) -> Fn:
    return pytest.fixture(func, scope=scope)


-@pytest.fixture(autouse=True, scope='session')
-def safety_check() -> None:
-    """ Ensure that no unwanted daemons are running before we start testing. """
-
-    # does not use -c as it is not supported on macOS
-    cmd = ['pgrep', 'pageserver|postgres|wal_acceptor']
-    result = subprocess.run(cmd, stdout=subprocess.DEVNULL)
-    if result.returncode == 0:
-        # returncode of 0 means it found something.
-        # This is bad; we don't want any of those processes polluting the
-        # result of the test.
-        raise Exception('found interfering processes running')
-
-
 class PgProtocol:
    """ Reusable connection logic """
    def __init__(self, host: str, port: int, username: Optional[str] = None):
@@ -116,6 +130,21 @@ class PgProtocol:
        conn.autocommit = autocommit
        return conn

+    async def connect_async(self, *, dbname: str = 'postgres', username: Optional[str] = None, password: Optional[str] = None) -> asyncpg.Connection:
+        """
+        Connect to the node from async python.
+        Returns asyncpg's connection object.
+        """
+
+        conn = await asyncpg.connect(
+            host=self.host,
+            port=self.port,
+            database=dbname,
+            user=username or self.username,
+            password=password,
+        )
+        return conn
+
    def safe_psql(self, query: str, **kwargs: Any) -> List[Any]:
        """
        Execute query against the node and return all rows.
@@ -141,6 +170,7 @@ class ZenithCli:
        assert os.path.isdir(binpath)
        self.binpath = binpath
        self.bin_zenith = os.path.join(binpath, 'zenith')
+        self.repo_dir = repo_dir
        self.env = os.environ.copy()
        self.env['ZENITH_REPO_DIR'] = repo_dir
        self.env['POSTGRES_DISTRIB_DIR'] = pg_distrib_dir
@@ -162,12 +192,27 @@ class ZenithCli:

        args = [self.bin_zenith] + arguments
        print('Running command "{}"'.format(' '.join(args)))
-        return subprocess.run(args,
-                              env=self.env,
-                              check=True,
-                              universal_newlines=True,
-                              stdout=subprocess.PIPE,
-                              stderr=subprocess.PIPE)
+
+        # Interceipt CalledProcessError and print more info
+        try:
+            res = subprocess.run(args,
+                                env=self.env,
+                                check=True,
+                                universal_newlines=True,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
+        except subprocess.CalledProcessError as exc:
+            # this way command output will be in recorded and shown in CI in failure message
+            msg = f"""\
+            Run failed: {exc}
+              stdout: {exc.stdout}
+              stderr: {exc.stderr}
+            """
+            print(msg)
+
+            raise Exception(msg) from exc
+
+        return res


@zenfixture
@@ -175,25 +220,153 @@ def zenith_cli(zenith_binpath: str, repo_dir: str, pg_distrib_dir: str) -> Zenit
    return ZenithCli(zenith_binpath, repo_dir, pg_distrib_dir)


+class ZenithPageserverHttpClient(requests.Session):
+    def __init__(self, port: int, auth_token: Optional[str] = None) -> None:
+        super().__init__()
+        self.port = port
+        self.auth_token = auth_token
+
+        if auth_token is not None:
+            self.headers['Authorization'] = f'Bearer {auth_token}'
+
+    def check_status(self):
+        self.get(f"http://localhost:{self.port}/v1/status").raise_for_status()
+
+    def branch_list(self, tenant_id: uuid.UUID) -> List[Dict]:
+        res = self.get(f"http://localhost:{self.port}/v1/branch/{tenant_id.hex}")
+        res.raise_for_status()
+        return res.json()
+
+    def branch_create(self, tenant_id: uuid.UUID, name: str, start_point: str) -> Dict:
+        res = self.post(
+            f"http://localhost:{self.port}/v1/branch",
+            json={
+                'tenant_id': tenant_id.hex,
+                'name': name,
+                'start_point': start_point,
+            }
+        )
+        res.raise_for_status()
+        return res.json()
+
+    def branch_detail(self, tenant_id: uuid.UUID, name: str) -> Dict:
+        res = self.get(
+            f"http://localhost:{self.port}/v1/branch/{tenant_id.hex}/{name}",
+        )
+        res.raise_for_status()
+        return res.json()
+
+    def tenant_list(self) -> List[str]:
+        res = self.get(f"http://localhost:{self.port}/v1/tenant")
+        res.raise_for_status()
+        return res.json()
+
+    def tenant_create(self, tenant_id: uuid.UUID):
+        res = self.post(
+            f"http://localhost:{self.port}/v1/tenant",
+            json={
+                'tenant_id': tenant_id.hex,
+            },
+        )
+        res.raise_for_status()
+        return res.json()
+
+    def get_metrics(self) -> str:
+        res = self.get(f"http://localhost:{self.port}/metrics")
+        res.raise_for_status()
+        return res.text
+
+
+@dataclass
+class AuthKeys:
+    pub: bytes
+    priv: bytes
+
+    def generate_management_token(self):
+        token = jwt.encode({"scope": "pageserverapi"}, self.priv, algorithm="RS256")
+
+        # jwt.encode can return 'bytes' or 'str', depending on Python version or type
+        # hinting or something (not sure what). If it returned 'bytes', convert it to 'str'
+        # explicitly.
+        if isinstance(token, bytes):
+            token = token.decode()
+
+        return token
+
+    def generate_tenant_token(self, tenant_id):
+        token = jwt.encode({"scope": "tenant", "tenant_id": tenant_id}, self.priv, algorithm="RS256")
+
+        if isinstance(token, bytes):
+            token = token.decode()
+
+        return token
+
+
+@zenfixture
+def worker_seq_no(worker_id: str):
+    # worker_id is a pytest-xdist fixture
+    # it can be master or gw<number>
+    # parse it to always get a number
+    if worker_id == 'master':
+        return 0
+    assert worker_id.startswith('gw')
+    return int(worker_id[2:])
+
+
+@zenfixture
+def worker_base_port(worker_seq_no: int):
+    # so we divide ports in ranges of 100 ports
+    # so workers have disjoint set of ports for services
+    return BASE_PORT + worker_seq_no * WORKER_PORT_NUM
+
+class PortDistributor:
+    def __init__(self, base_port: int, port_number: int) -> None:
+        self.iterator = iter(range(base_port, base_port + port_number))
+
+    def get_port(self) -> int:
+        try:
+            return next(self.iterator)
+        except StopIteration:
+            raise RuntimeError('port range configured for test is exhausted, consider enlarging the range')
+
+
+@zenfixture
+def port_distributor(worker_base_port):
+    return PortDistributor(base_port=worker_base_port, port_number=WORKER_PORT_NUM)
+
+@dataclass
+class PageserverPort:
+    pg: int
+    http: int
+
+
 class ZenithPageserver(PgProtocol):
    """ An object representing a running pageserver. """
-    def __init__(self, zenith_cli: ZenithCli):
-        super().__init__(host='localhost', port=DEFAULT_PAGESERVER_PORT)
+    def __init__(self, zenith_cli: ZenithCli, repo_dir: str, port: PageserverPort):
+        super().__init__(host='localhost', port=port.pg)
        self.zenith_cli = zenith_cli
        self.running = False
        self.initial_tenant = None
+        self.repo_dir = repo_dir
+        self.service_port = port # do not shadow PgProtocol.port which is just int

    def init(self, enable_auth: bool = False) -> 'ZenithPageserver':
        """
        Initialize the repository, i.e. run "zenith init".
        Returns self.
        """
-        cmd = ['init']
+        cmd = ['init', f'--pageserver-pg-port={self.service_port.pg}', f'--pageserver-http-port={self.service_port.http}']
        if enable_auth:
            cmd.append('--enable-auth')
        self.zenith_cli.run(cmd)
        return self

+    def repo_dir(self) -> str:
+        """
+        Return path to repository dir
+        """
+        return self.zenith_cli.repo_dir
+
    def start(self) -> 'ZenithPageserver':
        """
        Start the page server.
@@ -224,9 +397,31 @@ class ZenithPageserver(PgProtocol):
    def __exit__(self, exc_type, exc, tb):
        self.stop()

+    @cached_property
+    def auth_keys(self) -> AuthKeys:
+        pub = (Path(self.repo_dir) / 'auth_public_key.pem').read_bytes()
+        priv = (Path(self.repo_dir) / 'auth_private_key.pem').read_bytes()
+        return AuthKeys(pub=pub, priv=priv)
+
+    def http_client(self, auth_token: Optional[str] = None):
+        return ZenithPageserverHttpClient(
+            port=self.service_port.http,
+            auth_token=auth_token,
+        )
+
+
+

@zenfixture
-def pageserver(zenith_cli: ZenithCli) -> Iterator[ZenithPageserver]:
+def pageserver_port(port_distributor: PortDistributor) -> PageserverPort:
+    pg = port_distributor.get_port()
+    http = port_distributor.get_port()
+    print(f"pageserver_port: pg={pg} http={http}")
+    return PageserverPort(pg=pg, http=http)
+
+
+@zenfixture
+def pageserver(zenith_cli: ZenithCli, repo_dir: str, pageserver_port: PageserverPort) -> Iterator[ZenithPageserver]:
    """
    The 'pageserver' fixture provides a Page Server that's up and running.

@@ -238,8 +433,7 @@ def pageserver(zenith_cli: ZenithCli) -> Iterator[ZenithPageserver]:
    By convention, the test branches are named after the tests. For example,
    test called 'test_foo' would create and use branches with the 'test_foo' prefix.
    """
-
-    ps = ZenithPageserver(zenith_cli).init().start()
+    ps = ZenithPageserver(zenith_cli=zenith_cli, repo_dir=repo_dir, port=pageserver_port).init().start()
    # For convenience in tests, create a branch from the freshly-initialized cluster.
    zenith_cli.run(["branch", "empty", "main"])

@@ -249,17 +443,85 @@ def pageserver(zenith_cli: ZenithCli) -> Iterator[ZenithPageserver]:
    print('Starting pageserver cleanup')
    ps.stop()

+class PgBin:
+    """ A helper class for executing postgres binaries """
+    def __init__(self, log_dir: str, pg_distrib_dir: str):
+        self.log_dir = log_dir
+        self.pg_install_path = pg_distrib_dir
+        self.pg_bin_path = os.path.join(self.pg_install_path, 'bin')
+        self.env = os.environ.copy()
+        self.env['LD_LIBRARY_PATH'] = os.path.join(self.pg_install_path, 'lib')
+
+    def _fixpath(self, command: List[str]) -> None:
+        if '/' not in command[0]:
+            command[0] = os.path.join(self.pg_bin_path, command[0])
+
+    def _build_env(self, env_add: Optional[Env]) -> Env:
+        if env_add is None:
+            return self.env
+        env = self.env.copy()
+        env.update(env_add)
+        return env
+
+    def run(self, command: List[str], env: Optional[Env] = None, cwd: Optional[str] = None) -> None:
+        """
+        Run one of the postgres binaries.
+
+        The command should be in list form, e.g. ['pgbench', '-p', '55432']
+
+        All the necessary environment variables will be set.
+
+        If the first argument (the command name) doesn't include a path (no '/'
+        characters present), then it will be edited to include the correct path.
+
+        If you want stdout/stderr captured to files, use `run_capture` instead.
+        """
+
+        self._fixpath(command)
+        print('Running command "{}"'.format(' '.join(command)))
+        env = self._build_env(env)
+        subprocess.run(command, env=env, cwd=cwd, check=True)
+
+    def run_capture(self,
+                    command: List[str],
+                    env: Optional[Env] = None,
+                    cwd: Optional[str] = None) -> None:
+        """
+        Run one of the postgres binaries, with stderr and stdout redirected to a file.
+
+        This is just like `run`, but for chatty programs.
+        """
+
+        self._fixpath(command)
+        print('Running command "{}"'.format(' '.join(command)))
+        env = self._build_env(env)
+        subprocess_capture(self.log_dir, command, env=env, cwd=cwd, check=True)
+
+
+@zenfixture
+def pg_bin(test_output_dir: str, pg_distrib_dir: str) -> PgBin:
+    return PgBin(test_output_dir, pg_distrib_dir)
+
+@pytest.fixture
+def pageserver_auth_enabled(zenith_cli: ZenithCli, repo_dir: str, pageserver_port: PageserverPort):
+    with ZenithPageserver(zenith_cli=zenith_cli, repo_dir=repo_dir, port=pageserver_port).init(enable_auth=True).start() as ps:
+        # For convenience in tests, create a branch from the freshly-initialized cluster.
+        zenith_cli.run(["branch", "empty", "main"])
+        yield ps
+

 class Postgres(PgProtocol):
    """ An object representing a running postgres daemon. """
-    def __init__(self, zenith_cli: ZenithCli, repo_dir: str, tenant_id: str, port: int):
+    def __init__(self, zenith_cli: ZenithCli, repo_dir: str, pg_bin: PgBin, tenant_id: str, port: int):
        super().__init__(host='localhost', port=port)

        self.zenith_cli = zenith_cli
        self.running = False
        self.repo_dir = repo_dir
        self.branch: Optional[str] = None  # dubious, see asserts below
+        self.pgdata_dir: Optional[str] = None # Path to computenode PGDATA
        self.tenant_id = tenant_id
+        self.pg_bin = pg_bin
        # path to conf is <repo_dir>/pgdatadirs/tenants/<tenant_id>/<branch_name>/postgresql.conf

    def create(
@@ -267,7 +529,6 @@ class Postgres(PgProtocol):
        branch: str,
        wal_acceptors: Optional[str] = None,
        config_lines: Optional[List[str]] = None,
-        config_only: bool = False,
    ) -> 'Postgres':
        """
        Create the pg data directory.
@@ -279,11 +540,11 @@ class Postgres(PgProtocol):
        if not config_lines:
            config_lines = []

-        if config_only:
-            self.zenith_cli.run(['pg', 'create', '--config-only', branch, f'--tenantid={self.tenant_id}'])
-        else:
-            self.zenith_cli.run(['pg', 'create', branch, f'--tenantid={self.tenant_id}'])
+        self.zenith_cli.run(['pg', 'create', branch, f'--tenantid={self.tenant_id}', f'--port={self.port}'])
        self.branch = branch
+        path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch
+        self.pgdata_dir = os.path.join(self.repo_dir, path)
+
        if wal_acceptors is not None:
            self.adjust_for_wal_acceptors(wal_acceptors)
        if config_lines is None:
@@ -299,27 +560,32 @@ class Postgres(PgProtocol):
        """

        assert self.branch is not None
-        self.zenith_cli.run(['pg', 'start', self.branch, f'--tenantid={self.tenant_id}'])
+
+        print(f"Starting postgres on branch {self.branch}")
+
+        run_result = self.zenith_cli.run(['pg', 'start', self.branch, f'--tenantid={self.tenant_id}', f'--port={self.port}'])
        self.running = True

+        print(f"stdout: {run_result.stdout}")
+
        return self

+    def pg_data_dir_path(self) -> str:
+        """ Path to data directory """
+        path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch
+        return os.path.join(self.repo_dir, path)
+
    def pg_xact_dir_path(self) -> str:
        """ Path to pg_xact dir """
-        path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch / 'pg_xact'
-        return os.path.join(self.repo_dir, path)
+        return os.path.join(self.pg_data_dir_path(), 'pg_xact')

    def pg_twophase_dir_path(self) -> str:
        """ Path to pg_twophase dir """
-        print(self.tenant_id)
-        print(self.branch)
-        path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch / 'pg_twophase'
-        return os.path.join(self.repo_dir, path)
+        return os.path.join(self.pg_data_dir_path(), 'pg_twophase')

    def config_file_path(self) -> str:
        """ Path to postgresql.conf """
-        filename = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch / 'postgresql.conf'
-        return os.path.join(self.repo_dir, filename)
+        return os.path.join(self.pg_data_dir_path(), 'postgresql.conf')

    def adjust_for_wal_acceptors(self, wal_acceptors: str) -> 'Postgres':
        """
@@ -397,7 +663,6 @@ class Postgres(PgProtocol):
            branch=branch,
            wal_acceptors=wal_acceptors,
            config_lines=config_lines,
-            config_only=True,
        ).start()

        return self
@@ -408,16 +673,16 @@ class Postgres(PgProtocol):
    def __exit__(self, exc_type, exc, tb):
        self.stop()

-
 class PostgresFactory:
    """ An object representing multiple running postgres daemons. """
-    def __init__(self, zenith_cli: ZenithCli, repo_dir: str, initial_tenant: str, base_port: int = 55431):
+    def __init__(self, zenith_cli: ZenithCli, repo_dir: str, pg_bin: PgBin, initial_tenant: str, port_distributor: PortDistributor):
        self.zenith_cli = zenith_cli
        self.repo_dir = repo_dir
        self.num_instances = 0
        self.instances: List[Postgres] = []
        self.initial_tenant: str = initial_tenant
-        self.base_port = base_port
+        self.port_distributor = port_distributor
+        self.pg_bin = pg_bin

    def create_start(
        self,
@@ -426,14 +691,13 @@ class PostgresFactory:
        wal_acceptors: Optional[str] = None,
        config_lines: Optional[List[str]] = None
    ) -> Postgres:
-
        pg = Postgres(
            zenith_cli=self.zenith_cli,
            repo_dir=self.repo_dir,
+            pg_bin=self.pg_bin,
            tenant_id=tenant_id or self.initial_tenant,
-            port=self.base_port + self.num_instances + 1,
+            port=self.port_distributor.get_port(),
        )
-
        self.num_instances += 1
        self.instances.append(pg)

@@ -454,8 +718,9 @@ class PostgresFactory:
        pg = Postgres(
            zenith_cli=self.zenith_cli,
            repo_dir=self.repo_dir,
+            pg_bin=self.pg_bin,
            tenant_id=tenant_id or self.initial_tenant,
-            port=self.base_port + self.num_instances + 1,
+            port=self.port_distributor.get_port(),
        )

        self.num_instances += 1
@@ -478,8 +743,9 @@ class PostgresFactory:
        pg = Postgres(
            zenith_cli=self.zenith_cli,
            repo_dir=self.repo_dir,
+            pg_bin=self.pg_bin,
            tenant_id=tenant_id or self.initial_tenant,
-            port=self.base_port + self.num_instances + 1,
+            port=self.port_distributor.get_port(),
        )

        self.num_instances += 1
@@ -503,8 +769,14 @@ def initial_tenant(pageserver: ZenithPageserver):


@zenfixture
-def postgres(zenith_cli: ZenithCli, initial_tenant: str, repo_dir: str) -> Iterator[PostgresFactory]:
-    pgfactory = PostgresFactory(zenith_cli, repo_dir, initial_tenant=initial_tenant)
+def postgres(zenith_cli: ZenithCli, initial_tenant: str, repo_dir: str, pg_bin: PgBin, port_distributor: PortDistributor) -> Iterator[PostgresFactory]:
+    pgfactory = PostgresFactory(
+        zenith_cli=zenith_cli,
+        repo_dir=repo_dir,
+        pg_bin=pg_bin,
+        initial_tenant=initial_tenant,
+        port_distributor=port_distributor,
+    )

    yield pgfactory

@@ -512,67 +784,6 @@ def postgres(zenith_cli: ZenithCli, initial_tenant: str, repo_dir: str) -> Itera
    print('Starting postgres cleanup')
    pgfactory.stop_all()

-
-class PgBin:
-    """ A helper class for executing postgres binaries """
-    def __init__(self, log_dir: str, pg_distrib_dir: str):
-        self.log_dir = log_dir
-        self.pg_install_path = pg_distrib_dir
-        self.pg_bin_path = os.path.join(self.pg_install_path, 'bin')
-        self.env = os.environ.copy()
-        self.env['LD_LIBRARY_PATH'] = os.path.join(self.pg_install_path, 'lib')
-
-    def _fixpath(self, command: List[str]) -> None:
-        if '/' not in command[0]:
-            command[0] = os.path.join(self.pg_bin_path, command[0])
-
-    def _build_env(self, env_add: Optional[Env]) -> Env:
-        if env_add is None:
-            return self.env
-        env = self.env.copy()
-        env.update(env_add)
-        return env
-
-    def run(self, command: List[str], env: Optional[Env] = None, cwd: Optional[str] = None) -> None:
-        """
-        Run one of the postgres binaries.
-
-        The command should be in list form, e.g. ['pgbench', '-p', '55432']
-
-        All the necessary environment variables will be set.
-
-        If the first argument (the command name) doesn't include a path (no '/'
-        characters present), then it will be edited to include the correct path.
-
-        If you want stdout/stderr captured to files, use `run_capture` instead.
-        """
-
-        self._fixpath(command)
-        print('Running command "{}"'.format(' '.join(command)))
-        env = self._build_env(env)
-        subprocess.run(command, env=env, cwd=cwd, check=True)
-
-    def run_capture(self,
-                    command: List[str],
-                    env: Optional[Env] = None,
-                    cwd: Optional[str] = None) -> None:
-        """
-        Run one of the postgres binaries, with stderr and stdout redirected to a file.
-
-        This is just like `run`, but for chatty programs.
-        """
-
-        self._fixpath(command)
-        print('Running command "{}"'.format(' '.join(command)))
-        env = self._build_env(env)
-        subprocess_capture(self.log_dir, command, env=env, cwd=cwd, check=True)
-
-
-@zenfixture
-def pg_bin(test_output_dir: str, pg_distrib_dir: str) -> PgBin:
-    return PgBin(test_output_dir, pg_distrib_dir)
-
-
 def read_pid(path: Path):
    """ Read content of file into number """
    return int(path.read_text())
@@ -581,10 +792,11 @@ def read_pid(path: Path):
@dataclass
 class WalAcceptor:
    """ An object representing a running wal acceptor daemon. """
-    bin_path: Path
+    wa_bin_path: Path
    data_dir: Path
    port: int
    num: int # identifier for logging
+    pageserver_port: int
    auth_token: Optional[str] = None

    def start(self) -> 'WalAcceptor':
@@ -592,13 +804,13 @@ class WalAcceptor:
        self.data_dir.mkdir(parents=True, exist_ok=True)
        self.pidfile.unlink(missing_ok=True)

-        cmd = [str(self.bin_path)]
+        cmd = [str(self.wa_bin_path)]
        cmd.extend(["-D", str(self.data_dir)])
-        cmd.extend(["-l", "localhost:{}".format(self.port)])
+        cmd.extend(["-l", f"localhost:{self.port}"])
        cmd.append("--daemonize")
        cmd.append("--no-sync")
        # Tell page server it can receive WAL from this WAL safekeeper
-        cmd.extend(["--pageserver", "localhost:{}".format(DEFAULT_PAGESERVER_PORT)])
+        cmd.extend(["--pageserver", f"localhost:{self.pageserver_port}"])
        cmd.extend(["--recall", "1 second"])
        print('Running command "{}"'.format(' '.join(cmd)))
        env = {'PAGESERVER_AUTH_TOKEN': self.auth_token} if self.auth_token else None
@@ -645,24 +857,25 @@ class WalAcceptor:

 class WalAcceptorFactory:
    """ An object representing multiple running wal acceptors. """
-    def __init__(self, zenith_binpath: Path, data_dir: Path):
-        self.wa_binpath = zenith_binpath / 'wal_acceptor'
+    def __init__(self, zenith_binpath: Path, data_dir: Path, pageserver_port: int, port_distributor: PortDistributor):
+        self.wa_bin_path = zenith_binpath / 'wal_acceptor'
        self.data_dir = data_dir
        self.instances: List[WalAcceptor] = []
-        self.initial_port = 54321
+        self.port_distributor = port_distributor
+        self.pageserver_port = pageserver_port

    def start_new(self, auth_token: Optional[str] = None) -> WalAcceptor:
        """
        Start new wal acceptor.
        """
-
        wa_num = len(self.instances)
        wa = WalAcceptor(
-            self.wa_binpath,
-            self.data_dir / "wal_acceptor_{}".format(wa_num),
-            self.initial_port + wa_num,
-            wa_num,
-            auth_token,
+            wa_bin_path=self.wa_bin_path,
+            data_dir=self.data_dir / "wal_acceptor_{}".format(wa_num),
+            port=self.port_distributor.get_port(),
+            num=wa_num,
+            pageserver_port=self.pageserver_port,
+            auth_token=auth_token,
        )
        wa.start()
        self.instances.append(wa)
@@ -687,9 +900,14 @@ class WalAcceptorFactory:


@zenfixture
-def wa_factory(zenith_binpath: str, repo_dir: str) -> Iterator[WalAcceptorFactory]:
+def wa_factory(zenith_binpath: str, repo_dir: str, pageserver_port: PageserverPort, port_distributor: PortDistributor) -> Iterator[WalAcceptorFactory]:
    """ Gives WalAcceptorFactory providing wal acceptors. """
-    wafactory = WalAcceptorFactory(Path(zenith_binpath), Path(repo_dir) / "wal_acceptors")
+    wafactory = WalAcceptorFactory(
+        zenith_binpath=Path(zenith_binpath),
+        data_dir=Path(repo_dir) / "wal_acceptors",
+        pageserver_port=pageserver_port.pg,
+        port_distributor=port_distributor,
+    )
    yield wafactory
    # After the yield comes any cleanup code we need.
    print('Starting wal acceptors cleanup')
@@ -793,3 +1011,85 @@ class TenantFactory:
@zenfixture
 def tenant_factory(zenith_cli: ZenithCli):
    return TenantFactory(zenith_cli)
+
+#
+# Test helpers
+#
+def list_files_to_compare(pgdata_dir: str):
+    pgdata_files = []
+    for root, _file, filenames in os.walk(pgdata_dir):
+        for filename in filenames:
+            rel_dir = os.path.relpath(root, pgdata_dir)
+            # Skip some dirs and files we don't want to compare
+            skip_dirs = ['pg_wal', 'pg_stat', 'pg_stat_tmp', 'pg_subtrans', 'pg_logical']
+            skip_files = ['pg_internal.init', 'pg.log', 'zenith.signal', 'postgresql.conf',
+                        'postmaster.opts', 'postmaster.pid', 'pg_control']
+            if rel_dir not in skip_dirs and filename not in skip_files:
+                rel_file = os.path.join(rel_dir, filename)
+                pgdata_files.append(rel_file)
+
+    pgdata_files.sort()
+    print(pgdata_files)
+    return pgdata_files
+
+# pg is the existing and running compute node, that we want to compare with a basebackup
+def check_restored_datadir_content(zenith_cli: ZenithCli, test_output_dir: str, pg: Postgres, pageserver_pg_port: int):
+
+    # Get the timeline ID of our branch. We need it for the 'basebackup' command
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute("SHOW zenith.zenith_timeline")
+            timeline = cur.fetchone()[0]
+
+    # stop postgres to ensure that files won't change
+    pg.stop()
+
+    # Take a basebackup from pageserver
+    restored_dir_path = os.path.join(test_output_dir, f"{pg.branch}_restored_datadir")
+    mkdir_if_needed(restored_dir_path)
+
+    psql_path = os.path.join(pg.pg_bin.pg_bin_path, 'psql')
+
+    cmd = rf"""
+        {psql_path}                                    \
+            --no-psqlrc                                \
+            postgres://localhost:{pageserver_pg_port}  \
+            -c 'basebackup {pg.tenant_id} {timeline}'  \
+         | tar -x -C {restored_dir_path}
+    """
+
+    subprocess.check_call(cmd, shell=True)
+
+    # list files we're going to compare
+    pgdata_files = list_files_to_compare(pg.pgdata_dir)
+    restored_files = list_files_to_compare(restored_dir_path)
+
+    # check that file sets are equal
+    assert pgdata_files == restored_files
+
+    # compare content of the files
+    # filecmp returns (match, mismatch, error) lists
+    # We've already filtered all mismatching files in list_files_to_compare(),
+    # so here expect that the content is identical
+    (match, mismatch, error) = filecmp.cmpfiles(pg.pgdata_dir,
+                                                restored_dir_path,
+                                                pgdata_files,
+                                                shallow=False)
+    print('filecmp result mismatch and error lists:')
+    print(mismatch)
+    print(error)
+
+    for f in mismatch:
+
+        f1 = os.path.join(pg.pgdata_dir, f)
+        f2 = os.path.join(restored_dir_path, f)
+        stdout_filename = "{}.filediff".format(f2)
+
+        with open(stdout_filename, 'w') as stdout_f:
+            subprocess.run("xxd -b {} > {}.hex ".format(f1, f1), shell=True)
+            subprocess.run("xxd -b {} > {}.hex ".format(f2, f2), shell=True)
+
+            cmd = ['diff {}.hex {}.hex'.format(f1, f2)]
+            subprocess.run(cmd, stdout=stdout_f, shell=True)
+
+    assert (mismatch, error) == ([], [])
--- a/test_runner/performance/test_bulk_insert.py
+++ b/test_runner/performance/test_bulk_insert.py
@@ -0,0 +1,60 @@
+import os
+from contextlib import closing
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
+
+def get_timeline_size(repo_dir: str, tenantid: str, timelineid: str):
+    path = "{}/tenants/{}/timelines/{}".format(repo_dir, tenantid, timelineid)
+
+    totalbytes = 0
+    for root, dirs, files in os.walk(path):
+        for name in files:
+            totalbytes += os.path.getsize(os.path.join(root, name))
+
+        if 'wal' in dirs:
+            dirs.remove('wal')  # don't visit 'wal' subdirectory
+
+    return totalbytes
+
+#
+# Run bulk INSERT test.
+#
+# Collects metrics:
+#
+# 1. Time to INSERT 5 million rows
+# 2. Disk writes
+# 3. Disk space used
+#
+def test_bulk_insert(postgres: PostgresFactory, pageserver: ZenithPageserver, pg_bin, zenith_cli, zenbenchmark, repo_dir: str):
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_bulk_insert", "empty"])
+
+    pg = postgres.create_start('test_bulk_insert')
+    print("postgres is running on 'test_bulk_insert' branch")
+
+    # Open a connection directly to the page server that we'll use to force
+    # flushing the layers to disk
+    psconn = pageserver.connect();
+    pscur = psconn.cursor()
+
+    # Get the timeline ID of our branch. We need it for the 'do_gc' command
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute("SHOW zenith.zenith_timeline")
+            timeline = cur.fetchone()[0]
+
+            cur.execute("create table huge (i int, j int);")
+
+            # Run INSERT, recording the time and I/O it takes
+            with zenbenchmark.record_pageserver_writes(pageserver, 'pageserver_writes'):
+                with zenbenchmark.record_duration('insert'):
+                    cur.execute("insert into huge values (generate_series(1, 5000000), 0);")
+
+                    # Flush the layers from memory to disk. This is included in the reported
+                    # time and I/O
+                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+
+            # Report disk space used by the repository
+            timeline_size = get_timeline_size(repo_dir, pageserver.initial_tenant, timeline)
+            zenbenchmark.record('size', timeline_size / (1024*1024), 'MB')
--- a/test_runner/performance/test_perf_pgbench.py
+++ b/test_runner/performance/test_perf_pgbench.py
@@ -0,0 +1,68 @@
+import os
+from contextlib import closing
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
+
+def get_timeline_size(repo_dir: str, tenantid: str, timelineid: str):
+    path = "{}/tenants/{}/timelines/{}".format(repo_dir, tenantid, timelineid)
+
+    totalbytes = 0
+    for root, dirs, files in os.walk(path):
+        for name in files:
+            totalbytes += os.path.getsize(os.path.join(root, name))
+
+        if 'wal' in dirs:
+            dirs.remove('wal')  # don't visit 'wal' subdirectory
+
+    return totalbytes
+
+#
+# Run a very short pgbench test.
+#
+# Collects three metrics:
+#
+# 1. Time to initialize the pgbench database (pgbench -s5 -i)
+# 2. Time to run 5000 pgbench transactions
+# 3. Disk space used
+#
+def test_pgbench(postgres: PostgresFactory, pageserver: ZenithPageserver, pg_bin, zenith_cli, zenbenchmark, repo_dir: str):
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_pgbench_perf", "empty"])
+
+    pg = postgres.create_start('test_pgbench_perf')
+    print("postgres is running on 'test_pgbench_perf' branch")
+
+    # Open a connection directly to the page server that we'll use to force
+    # flushing the layers to disk
+    psconn = pageserver.connect();
+    pscur = psconn.cursor()
+
+    # Get the timeline ID of our branch. We need it for the 'do_gc' command
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute("SHOW zenith.zenith_timeline")
+            timeline = cur.fetchone()[0]
+
+    connstr = pg.connstr()
+
+    # Initialize pgbench database, recording the time and I/O it takes
+    with zenbenchmark.record_pageserver_writes(pageserver, 'pageserver_writes'):
+        with zenbenchmark.record_duration('init'):
+            pg_bin.run_capture(['pgbench', '-s5', '-i', connstr])
+
+            # Flush the layers from memory to disk. This is included in the reported
+            # time and I/O
+            pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+
+    # Run pgbench for 5000 transactions
+    with zenbenchmark.record_duration('5000_xacts'):
+        pg_bin.run_capture(['pgbench', '-c1', '-t5000', connstr])
+
+    # Flush the layers to disk again. This is *not' included in the reported time,
+    # though.
+    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+
+    # Report disk space used by the repository
+    timeline_size = get_timeline_size(repo_dir, pageserver.initial_tenant, timeline)
+    zenbenchmark.record('size', timeline_size / (1024*1024), 'MB')
--- a/vendor/postgres
+++ b/vendor/postgres
--- a/walkeeper/Cargo.toml
+++ b/walkeeper/Cargo.toml
@@ -13,16 +13,11 @@ bytes = "1.0.1"
 byteorder = "1.4.3"
 fs2 = "0.4.3"
 lazy_static = "1.4.0"
-slog-stdlog = "4.1.0"
-slog-async = "2.6.0"
-slog-scope = "4.4.0"
-slog-term = "2.8.0"
-slog = "2.7.0"
 log = "0.4.14"
 clap = "2.33.0"
 daemonize = "0.4.1"
 rust-s3 = { version = "0.27.0-rc4", features = ["no-verify-ssl"] }
-tokio = { version = "1.3.0", features = ["full"] }
+tokio = "1.11"
 tokio-stream = { version = "0.1.4" }
 postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
--- a/walkeeper/src/bin/wal_acceptor.rs
+++ b/walkeeper/src/bin/wal_acceptor.rs
@@ -1,15 +1,14 @@
 //
 // Main entry point for the wal_acceptor executable
 //
-use anyhow::{Context, Result};
+use anyhow::Result;
 use clap::{App, Arg};
 use daemonize::Daemonize;
 use log::*;
-use slog::Drain;
+use std::env;
 use std::path::{Path, PathBuf};
 use std::thread;
-use std::{env, io};
-use std::{fs::File, fs::OpenOptions};
+use zenith_utils::logging;

 use walkeeper::s3_offload;
 use walkeeper::wal_service;
@@ -113,20 +112,7 @@ fn main() -> Result<()> {

 fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<()> {
    let log_filename = conf.data_dir.join("wal_acceptor.log");
-    // Don't open the same file for output multiple times;
-    // the different fds could overwrite each other's output.
-    let log_file = OpenOptions::new()
-        .create(true)
-        .append(true)
-        .open(&log_filename)
-        .with_context(|| format!("failed to open {:?}", &log_filename))?;
-
-    // Initialize logger
-    let logger_file = log_file.try_clone().unwrap();
-    let _scope_guard = init_logging(&conf, logger_file)?;
-    let _log_guard = slog_stdlog::init().unwrap();
-    // Note: this `info!(...)` macro comes from `log` crate
-    info!("standard logging redirected to slog");
+    let (_scope_guard, log_file) = logging::init(log_filename, conf.daemonize)?;

    if conf.daemonize {
        info!("daemonizing...");
@@ -179,29 +165,3 @@ fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<()> {
    }
    Ok(())
 }
-
-fn init_logging(
-    conf: &WalAcceptorConf,
-    log_file: File,
-) -> Result<slog_scope::GlobalLoggerGuard, io::Error> {
-    if conf.daemonize {
-        let decorator = slog_term::PlainSyncDecorator::new(log_file);
-        let drain = slog_term::CompactFormat::new(decorator).build();
-        let drain = slog::Filter::new(drain, |record: &slog::Record| {
-            record.level().is_at_least(slog::Level::Info)
-        });
-        let drain = std::sync::Mutex::new(drain).fuse();
-        let logger = slog::Logger::root(drain, slog::o!());
-        Ok(slog_scope::set_global_logger(logger))
-    } else {
-        let decorator = slog_term::TermDecorator::new().build();
-        let drain = slog_term::FullFormat::new(decorator).build().fuse();
-        let drain = slog::Filter::new(drain, |record: &slog::Record| {
-            record.level().is_at_least(slog::Level::Info)
-        })
-        .fuse();
-        let drain = slog_async::Async::new(drain).chan_size(1000).build().fuse();
-        let logger = slog::Logger::root(drain, slog::o!());
-        Ok(slog_scope::set_global_logger(logger))
-    }
-}
--- a/walkeeper/src/lib.rs
+++ b/walkeeper/src/lib.rs
@@ -5,6 +5,7 @@ use std::time::Duration;
 pub mod receive_wal;
 pub mod replication;
 pub mod s3_offload;
+pub mod safekeeper;
 pub mod send_wal;
 pub mod timeline;
 pub mod wal_service;
--- a/walkeeper/src/receive_wal.rs
+++ b/walkeeper/src/receive_wal.rs
@@ -1,142 +1,26 @@
-//! This implements the Safekeeper protocol, picking up immediately after the "START_WAL_PUSH" message
-//!
-//! FIXME: better description needed here
+//! Safekeeper communication endpoint to wal proposer (compute node).
+//! Gets messages from the network, passes them down to consensus module and
+//! sends replies back.

 use anyhow::{bail, Context, Result};
-use bincode::config::Options;
-use bytes::{Buf, Bytes};
+use bytes::Bytes;
 use log::*;
 use postgres::{Client, Config, NoTls};
-use serde::{Deserialize, Serialize};
-use std::cmp::{max, min};
-use std::fs::{self, File, OpenOptions};
-use std::io::{Read, Seek, SeekFrom, Write};
+
 use std::net::SocketAddr;
-use std::str;
-use std::sync::Arc;
 use std::thread;
 use std::thread::sleep;
-use zenith_utils::bin_ser::{self, le_coder, LeSer};
-use zenith_utils::connstring::connection_host_port;
-use zenith_utils::lsn::Lsn;
-use zenith_utils::postgres_backend::PostgresBackend;
-use zenith_utils::pq_proto::{BeMessage, FeMessage, SystemId};
-use zenith_utils::zid::{ZTenantId, ZTimelineId};

-use crate::replication::HotStandbyFeedback;
+use crate::safekeeper::AcceptorProposerMessage;
+use crate::safekeeper::ProposerAcceptorMessage;
+
 use crate::send_wal::SendWalHandler;
-use crate::timeline::{Timeline, TimelineTools};
+use crate::timeline::TimelineTools;
 use crate::WalAcceptorConf;
-use postgres_ffi::xlog_utils::{TimeLineID, XLogFileName, MAX_SEND_SIZE, XLOG_BLCKSZ};
-
-pub const SK_MAGIC: u32 = 0xcafeceefu32;
-pub const SK_FORMAT_VERSION: u32 = 1;
-const SK_PROTOCOL_VERSION: u32 = 1;
-const UNKNOWN_SERVER_VERSION: u32 = 0;
-const END_OF_STREAM: Lsn = Lsn(0);
-pub const CONTROL_FILE_NAME: &str = "safekeeper.control";
-
-/// Unique node identifier used by Paxos
-#[derive(Debug, Clone, Copy, Ord, PartialOrd, PartialEq, Eq, Serialize, Deserialize)]
-pub struct NodeId {
-    term: u64,
-    uuid: [u8; 16],
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub struct ServerInfo {
-    /// proposer-safekeeper protocol version
-    pub protocol_version: u32,
-    /// Postgres server version
-    pub pg_version: u32,
-    pub node_id: NodeId,
-    pub system_id: SystemId,
-    /// Zenith timelineid
-    pub timeline_id: ZTimelineId,
-    pub wal_end: Lsn,
-    pub timeline: TimeLineID,
-    pub wal_seg_size: u32,
-    pub tenant_id: ZTenantId,
-}
-
-/// Vote request sent from proposer to safekeepers
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-struct RequestVote {
-    node_id: NodeId,
-    /// volume commit LSN
-    vcl: Lsn,
-    /// new epoch when safekeeper reaches vcl
-    epoch: u64,
-}
-
-/// Information of about storage node
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub struct SafeKeeperInfo {
-    /// magic for verifying content the control file
-    pub magic: u32,
-    /// safekeeper format version
-    pub format_version: u32,
-    /// safekeeper's epoch
-    pub epoch: u64,
-    /// information about server
-    pub server: ServerInfo,
-    /// part of WAL acknowledged by quorum
-    pub commit_lsn: Lsn,
-    /// locally flushed part of WAL
-    pub flush_lsn: Lsn,
-    /// minimal LSN which may be needed for recovery of some safekeeper: min(commit_lsn) for all safekeepers
-    pub restart_lsn: Lsn,
-}
-
-impl SafeKeeperInfo {
-    pub fn new() -> SafeKeeperInfo {
-        SafeKeeperInfo {
-            magic: SK_MAGIC,
-            format_version: SK_FORMAT_VERSION,
-            epoch: 0,
-            server: ServerInfo {
-                protocol_version: SK_PROTOCOL_VERSION, /* proposer-safekeeper protocol version */
-                pg_version: UNKNOWN_SERVER_VERSION,    /* Postgres server version */
-                node_id: NodeId {
-                    term: 0,
-                    uuid: [0; 16],
-                },
-                system_id: 0, /* Postgres system identifier */
-                timeline_id: ZTimelineId::from([0u8; 16]),
-                wal_end: Lsn(0),
-                timeline: 0,
-                wal_seg_size: 0,
-                tenant_id: ZTenantId::from([0u8; 16]),
-            },
-            commit_lsn: Lsn(0),  /* part of WAL acknowledged by quorum */
-            flush_lsn: Lsn(0),   /* locally flushed part of WAL */
-            restart_lsn: Lsn(0), /* minimal LSN which may be needed for recovery of some safekeeper */
-        }
-    }
-}
-
-/// Request with WAL message sent from proposer to safekeeper.
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-struct SafeKeeperRequest {
-    /// Sender's node identifier (looks like we do not need it for TCP streaming connection)
-    sender_id: NodeId,
-    /// start position of message in WAL
-    begin_lsn: Lsn,
-    /// end position of message in WAL
-    end_lsn: Lsn,
-    /// restart LSN position  (minimal LSN which may be needed by proposer to perform recovery)
-    restart_lsn: Lsn,
-    /// LSN committed by quorum of safekeepers
-    commit_lsn: Lsn,
-}
-
-/// Report safekeeper state to proposer
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-struct SafeKeeperResponse {
-    epoch: u64,
-    flush_lsn: Lsn,
-    hs_feedback: HotStandbyFeedback,
-}
+use zenith_utils::connstring::connection_host_port;
+use zenith_utils::postgres_backend::PostgresBackend;
+use zenith_utils::pq_proto::{BeMessage, FeMessage};
+use zenith_utils::zid::{ZTenantId, ZTimelineId};

 pub struct ReceiveWalConn<'pg> {
    /// Postgres connection
@@ -162,8 +46,8 @@ fn request_callback(conf: WalAcceptorConf, timelineid: ZTimelineId, tenantid: ZT
    let me_conf: Config = me_connstr.parse().unwrap();
    let (host, port) = connection_host_port(&me_conf);
    let callme = format!(
-        "callmemaybe {} {} host={} port={} options='-c ztimelineid={}'",
-        tenantid, timelineid, host, port, timelineid,
+        "callmemaybe {} {} host={} port={} options='-c ztimelineid={} ztenantid={}'",
+        tenantid, timelineid, host, port, timelineid, tenantid,
    );

    loop {
@@ -190,7 +74,7 @@ fn request_callback(conf: WalAcceptorConf, timelineid: ZTimelineId, tenantid: ZT

 impl<'pg> ReceiveWalConn<'pg> {
    pub fn new(pg: &'pg mut PostgresBackend) -> Result<ReceiveWalConn<'pg>> {
-        let peer_addr = pg.get_peer_addr()?;
+        let peer_addr = *pg.get_peer_addr();
        Ok(ReceiveWalConn {
            pg_backend: pg,
            peer_addr,
@@ -206,315 +90,58 @@ impl<'pg> ReceiveWalConn<'pg> {
        }
    }

-    // Read the result of a `CopyData` message sent from the postgres instance
-    //
-    // As the trait bound implies, this always encodes little-endian.
-    fn read_msg<T: LeSer>(&mut self) -> Result<T> {
+    // Read and parse message sent from the postgres instance
+    fn read_msg(&mut self) -> Result<ProposerAcceptorMessage> {
        let data = self.read_msg_bytes()?;
-        // Taken directly from `LeSer::des`:
-        let value = le_coder()
-            .reject_trailing_bytes()
-            .deserialize(&data)
-            .or(Err(bin_ser::DeserializeError::BadInput))?;
-        Ok(value)
+        ProposerAcceptorMessage::parse(data)
    }

-    // Writes the value into a `CopyData` message sent to the postgres instance
-    fn write_msg<T: LeSer>(&mut self, value: &T) -> Result<()> {
+    // Send message to the postgres
+    fn write_msg(&mut self, msg: &AcceptorProposerMessage) -> Result<()> {
        let mut buf = Vec::new();
-        value.ser_into(&mut buf)?;
+        msg.serialize(&mut buf)?;
        self.pg_backend.write_message(&BeMessage::CopyData(&buf))?;
        Ok(())
    }

    /// Receive WAL from wal_proposer
    pub fn run(&mut self, swh: &mut SendWalHandler) -> Result<()> {
-        let mut this_timeline: Option<Arc<Timeline>> = None;
-
        // Notify the libpq client that it's allowed to send `CopyData` messages
        self.pg_backend
            .write_message(&BeMessage::CopyBothResponse)?;

        // Receive information about server
-        let server_info = self
-            .read_msg::<ServerInfo>()
-            .context("Failed to receive server info")?;
-        info!(
-            "Start handshake with wal_proposer {} sysid {} timeline {} tenant {}",
-            self.peer_addr, server_info.system_id, server_info.timeline_id, server_info.tenant_id,
-        );
-        // FIXME: also check that the system identifier matches
-        this_timeline.set(server_info.timeline_id)?;
-        this_timeline.get().load_control_file(&swh.conf)?;
-
-        let mut my_info = this_timeline.get().get_info();
-
-        /* Check protocol compatibility */
-        if server_info.protocol_version != SK_PROTOCOL_VERSION {
-            bail!(
-                "Incompatible protocol version {}, expected {}",
-                server_info.protocol_version,
-                SK_PROTOCOL_VERSION
-            );
-        }
-        /* Postgres upgrade is not treated as fatal error */
-        if server_info.pg_version != my_info.server.pg_version
-            && my_info.server.pg_version != UNKNOWN_SERVER_VERSION
-        {
-            info!(
-                "Incompatible server version {}, expected {}",
-                server_info.pg_version, my_info.server.pg_version
-            );
+        let mut msg = self
+            .read_msg()
+            .context("failed to receive proposer greeting")?;
+        let tenant_id: ZTenantId;
+        match msg {
+            ProposerAcceptorMessage::Greeting(ref greeting) => {
+                info!(
+                    "start handshake with wal proposer {} sysid {} timeline {}",
+                    self.peer_addr, greeting.system_id, greeting.tli,
+                );
+                tenant_id = greeting.tenant_id;
+            }
+            _ => bail!("unexpected message {:?} instead of greeting", msg),
        }

-        /* Update information about server, but preserve locally stored node_id */
-        let node_id = my_info.server.node_id;
-        my_info.server = server_info.clone();
-        my_info.server.node_id = node_id;
-
-        /* Calculate WAL end based on local data */
-        let (flush_lsn, timeline_id) = this_timeline.find_end_of_wal(&swh.conf.data_dir, true);
-        my_info.flush_lsn = flush_lsn;
-        my_info.server.timeline = timeline_id;
-
-        info!(
-            "find_end_of_wal in {:?}: timeline={} flush_lsn={}",
-            &swh.conf.data_dir, timeline_id, flush_lsn
-        );
-
-        /* Report my identifier to proposer */
-        self.write_msg(&my_info)?;
-
-        /* Wait for vote request */
-        let prop = self
-            .read_msg::<RequestVote>()
-            .context("Failed to read vote request")?;
-        /* This is Paxos check which should ensure that only one master can perform commits */
-        if prop.node_id < my_info.server.node_id {
-            /* Send my node-id to inform proposer that it's candidate was rejected */
-            self.write_msg(&my_info.server.node_id)?;
-            bail!(
-                "Reject connection attempt with term {} because my term is {}",
-                prop.node_id.term,
-                my_info.server.node_id.term,
-            );
-        }
-        my_info.server.node_id = prop.node_id;
-        this_timeline.get().set_info(&my_info);
-        /* Need to persist our vote first */
-        this_timeline.get().save_control_file(true)?;
-
-        let mut flushed_restart_lsn = Lsn(0);
-        let wal_seg_size = server_info.wal_seg_size as usize;
-
-        /* Acknowledge the proposed candidate by returning it to the proposer */
-        self.write_msg(&prop.node_id)?;
-
+        // if requested, ask pageserver to fetch wal from us
+        // xxx: this place seems not really fitting
        if swh.conf.pageserver_addr.is_some() {
            // Need to establish replication channel with page server.
            // Add far as replication in postgres is initiated by receiver, we should use callme mechanism
            let conf = swh.conf.clone();
-            let timelineid = this_timeline.get().timelineid;
-            let tenantid = server_info.tenant_id;
+            let timelineid = swh.timeline.get().timelineid;
            thread::spawn(move || {
-                request_callback(conf, timelineid, tenantid);
+                request_callback(conf, timelineid, tenant_id);
            });
        }

-        info!(
-            "Start streaming from timeline {} tenant {} address {:?} flush_lsn={}",
-            server_info.timeline_id, server_info.tenant_id, self.peer_addr, my_info.flush_lsn
-        );
-
-        // Main loop
        loop {
-            let mut sync_control_file = false;
-
-            /* Receive message header */
-            let msg_bytes = self.read_msg_bytes()?;
-            let mut msg_reader = msg_bytes.reader();
-
-            let req = SafeKeeperRequest::des_from(&mut msg_reader)
-                .context("Failed to get WAL message header")?;
-            if req.sender_id != my_info.server.node_id {
-                bail!("Sender NodeId is changed");
-            }
-            if req.begin_lsn == END_OF_STREAM {
-                info!("Server stops streaming");
-                break;
-            }
-            let start_pos = req.begin_lsn;
-            let end_pos = req.end_lsn;
-            let rec_size = end_pos.checked_sub(start_pos).unwrap().0 as usize;
-            assert!(rec_size <= MAX_SEND_SIZE);
-
-            debug!(
-                "received for {} bytes between {} and {}",
-                rec_size, start_pos, end_pos,
-            );
-
-            /* Receive message body (from the rest of the message) */
-            let mut buf = Vec::with_capacity(rec_size);
-            msg_reader.read_to_end(&mut buf)?;
-            assert_eq!(buf.len(), rec_size);
-
-            /* Save message in file */
-            Self::write_wal_file(
-                swh,
-                start_pos,
-                timeline_id,
-                this_timeline.get(),
-                wal_seg_size,
-                &buf,
-            )?;
-
-            my_info.restart_lsn = req.restart_lsn;
-            my_info.commit_lsn = req.commit_lsn;
-
-            /*
-             * Epoch switch happen when written WAL record cross the boundary.
-             * The boundary is maximum of last WAL position at this node (FlushLSN) and global
-             * maximum (vcl) determined by WAL proposer during handshake.
-             * Switching epoch means that node completes recovery and start writing in the WAL new data.
-             */
-            if my_info.epoch < prop.epoch && end_pos > max(my_info.flush_lsn, prop.vcl) {
-                info!("Switch to new epoch {}", prop.epoch);
-                my_info.epoch = prop.epoch; /* bump epoch */
-                sync_control_file = true;
-            }
-            if end_pos > my_info.flush_lsn {
-                my_info.flush_lsn = end_pos;
-            }
-            /*
-             * Update restart LSN in control file.
-             * To avoid negative impact on performance of extra fsync, do it only
-             * when restart_lsn delta exceeds WAL segment size.
-             */
-            sync_control_file |= flushed_restart_lsn + (wal_seg_size as u64) < my_info.restart_lsn;
-            this_timeline.get().save_control_file(sync_control_file)?;
-
-            if sync_control_file {
-                flushed_restart_lsn = my_info.restart_lsn;
-            }
-
-            /* Report flush position */
-            //info!("Confirm LSN: {:X}/{:>08X}", (end_pos>>32) as u32, end_pos as u32);
-            let resp = SafeKeeperResponse {
-                epoch: my_info.epoch,
-                flush_lsn: end_pos,
-                hs_feedback: this_timeline.get().get_hs_feedback(),
-            };
-            self.write_msg(&resp)?;
-
-            /*
-             * Ping wal sender that new data is available.
-             * FlushLSN (end_pos) can be smaller than commitLSN in case we are at catching-up safekeeper.
-             */
-            this_timeline
-                .get()
-                .notify_wal_senders(min(req.commit_lsn, end_pos));
+            let reply = swh.timeline.get().process_msg(&msg)?;
+            self.write_msg(&reply)?;
+            msg = self.read_msg()?;
        }
-
-        Ok(())
-    }
-
-    fn write_wal_file(
-        swh: &SendWalHandler,
-        startpos: Lsn,
-        timeline_id: TimeLineID,
-        timeline: &Arc<Timeline>,
-        wal_seg_size: usize,
-        buf: &[u8],
-    ) -> Result<()> {
-        let mut bytes_left: usize = buf.len();
-        let mut bytes_written: usize = 0;
-        let mut partial;
-        let mut start_pos = startpos;
-        const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ];
-
-        /* Extract WAL location for this block */
-        let mut xlogoff = start_pos.segment_offset(wal_seg_size) as usize;
-
-        while bytes_left != 0 {
-            let bytes_to_write;
-
-            /*
-             * If crossing a WAL boundary, only write up until we reach wal
-             * segment size.
-             */
-            if xlogoff + bytes_left > wal_seg_size {
-                bytes_to_write = wal_seg_size - xlogoff;
-            } else {
-                bytes_to_write = bytes_left;
-            }
-
-            /* Open file */
-            let segno = start_pos.segment_number(wal_seg_size);
-            let wal_file_name = XLogFileName(timeline_id, segno, wal_seg_size);
-            let wal_file_path = swh
-                .conf
-                .data_dir
-                .join(timeline.timelineid.to_string())
-                .join(wal_file_name.clone());
-            let wal_file_partial_path = swh
-                .conf
-                .data_dir
-                .join(timeline.timelineid.to_string())
-                .join(wal_file_name.clone() + ".partial");
-
-            {
-                let mut wal_file: File;
-                /* Try to open already completed segment */
-                if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_path) {
-                    wal_file = file;
-                    partial = false;
-                } else if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_partial_path)
-                {
-                    /* Try to open existed partial file */
-                    wal_file = file;
-                    partial = true;
-                } else {
-                    /* Create and fill new partial file */
-                    partial = true;
-                    match OpenOptions::new()
-                        .create(true)
-                        .write(true)
-                        .open(&wal_file_partial_path)
-                    {
-                        Ok(mut file) => {
-                            for _ in 0..(wal_seg_size / XLOG_BLCKSZ) {
-                                file.write_all(&ZERO_BLOCK)?;
-                            }
-                            wal_file = file;
-                        }
-                        Err(e) => {
-                            error!("Failed to open log file {:?}: {}", &wal_file_path, e);
-                            return Err(e.into());
-                        }
-                    }
-                }
-                wal_file.seek(SeekFrom::Start(xlogoff as u64))?;
-                wal_file.write_all(&buf[bytes_written..(bytes_written + bytes_to_write)])?;
-
-                // Flush file is not prohibited
-                if !swh.conf.no_sync {
-                    wal_file.sync_all()?;
-                }
-            }
-            /* Write was successful, advance our position */
-            bytes_written += bytes_to_write;
-            bytes_left -= bytes_to_write;
-            start_pos += bytes_to_write as u64;
-            xlogoff += bytes_to_write;
-
-            /* Did we reach the end of a WAL segment? */
-            if start_pos.segment_offset(wal_seg_size) == 0 {
-                xlogoff = 0;
-                if partial {
-                    fs::rename(&wal_file_partial_path, &wal_file_path)?;
-                }
-            }
-        }
-        Ok(())
    }
 }
--- a/Show More
+++ b/Show More