Fixes to checkpoint.nextXid accounting:

- Advance next_xid only when incoming xlog record has one. - Advance next_xid by one. Remove XID_CHECKPOINT_INTERVAL.
Add batch_wraparound/test_xid_wraparound.py
2026-07-07 14:10:43 +00:00 · 2021-09-01 12:26:57 +03:00 · 2021-09-01 12:26:57 +03:00 · 2021-09-01 09:42:18 +03:00 · 2021-08-31 13:09:16 -07:00 · 2021-08-31 14:10:50 +03:00
99 changed files with 5752 additions and 5633 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -10,6 +10,16 @@ executors:
      - image: cimg/rust:1.52.1

 jobs:
+  check-codestyle:
+    executor: zenith-build-executor
+    steps:
+      - checkout
+
+      - run:
+          name: rustfmt
+          when: always
+          command: |
+            cargo fmt --all -- --check

  # A job to build postgres
  build-postgres:
@@ -229,7 +239,7 @@ jobs:
          when: always
          command: |
            du -sh /tmp/test_output/*
-            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "wal_acceptor.log" ! -name "regression.diffs" -delete
+            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "wal_acceptor.log" ! -name "regression.diffs" ! -name "junit.xml" -delete
            du -sh /tmp/test_output/*
      - store_artifacts:
          path: /tmp/test_output
@@ -237,9 +247,27 @@ jobs:
      - store_test_results:
          path: /tmp/test_output

+  # Build zenithdb/zenith:latest image and push it to Docker hub
+  docker-image:
+    docker:
+      - image: cimg/base:2021.04
+    steps:
+      - checkout
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - run:
+          name: Init postgres submodule
+          command: git submodule update --init --depth 1
+      - run:
+          name: Build and push Docker image
+          command: |
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
+            docker build -t zenithdb/zenith:latest . && docker push zenithdb/zenith:latest
+
 workflows:
  build_and_test:
    jobs:
+      - check-codestyle
      - build-postgres
      - build-zenith:
          name: build-zenith-<< matrix.build_type >>
@@ -257,6 +285,14 @@ workflows:
          needs_postgres_source: true
          requires:
            - build-zenith-<< matrix.build_type >>
+      - run-pytest:
+          name: xid wraparound tests << matrix.build_type >>
+          matrix:
+            parameters:
+              build_type: ["debug", "release"]
+          test_selection: batch_wraparound
+          requires:
+            - build-zenith-<< matrix.build_type >>
      - run-pytest:
          name: other tests << matrix.build_type >>
          matrix:
@@ -265,3 +301,20 @@ workflows:
          test_selection: batch_others
          requires:
            - build-zenith-<< matrix.build_type >>
+      - run-pytest:
+          name: benchmarks
+          build_type: release
+          test_selection: performance
+          requires:
+            - build-zenith-release
+      - docker-image:
+          # Context gives an ability to login
+          context: Docker Hub
+          # Build image only for commits to main
+          filters:
+            branches:
+              only:
+                - main
+          requires:
+            - pg_regress tests release
+            - other tests release
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,4 +1,4 @@
 [submodule "vendor/postgres"]
 	path = vendor/postgres
-	url = https://github.com/libzenith/postgres
+	url = https://github.com/zenithdb/postgres
 	branch = main
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -154,9 +154,9 @@ dependencies = [

 [[package]]
 name = "bindgen"
-version = "0.57.0"
+version = "0.59.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd4865004a46a0aafb2a0a5eb19d3c9fc46ee5f063a6cfc605c69ac9ecf5263d"
+checksum = "453c49e5950bb0eb63bb3df640e31618846c89d5b7faa54040d76e98e0134375"
 dependencies = [
 "bitflags",
 "cexpr",
@@ -181,6 +181,18 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"

+[[package]]
+name = "bitvec"
+version = "0.19.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8942c8d352ae1838c9dda0b0ca2ab657696ef2232a20147cf1b30ae1a9cb4321"
+dependencies = [
+ "funty",
+ "radium",
+ "tap",
+ "wyz",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.9.0"
@@ -234,15 +246,12 @@ name = "cc"
 version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2"
-dependencies = [
- "jobserver",
-]

 [[package]]
 name = "cexpr"
-version = "0.4.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27"
+checksum = "db507a7679252d2276ed0dd8113c6875ec56d3089f9225b2b42c30cc1f8e5c89"
 dependencies = [
 "nom",
 ]
@@ -312,6 +321,7 @@ dependencies = [
 "postgres_ffi",
 "rand",
 "regex",
+ "reqwest",
 "serde",
 "serde_json",
 "tar",
@@ -537,10 +547,10 @@ dependencies = [
 ]

 [[package]]
-name = "fs_extra"
-version = "1.2.0"
+name = "funty"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394"
+checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7"

 [[package]]
 name = "futures"
@@ -703,15 +713,6 @@ version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"

-[[package]]
-name = "heck"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
-dependencies = [
- "unicode-segmentation",
-]
-
 [[package]]
 name = "hermit-abi"
 version = "0.1.19"
@@ -865,15 +866,6 @@ version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"

-[[package]]
-name = "jobserver"
-version = "0.1.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "972f5ae5d1cb9c6ae417789196c803205313edde988685da5e3aae0827b9e7fd"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "js-sys"
 version = "0.3.51"
@@ -925,18 +917,6 @@ dependencies = [
 "winapi",
 ]

-[[package]]
-name = "librocksdb-sys"
-version = "6.17.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5da125e1c0f22c7cae785982115523a0738728498547f415c9054cb17c7e89f9"
-dependencies = [
- "bindgen",
- "cc",
- "glob",
- "libc",
-]
-
 [[package]]
 name = "lock_api"
 version = "0.4.4"
@@ -1073,10 +1053,12 @@ dependencies = [

 [[package]]
 name = "nom"
-version = "5.1.2"
+version = "6.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af"
+checksum = "e7413f999671bd4745a7b624bd370a569fb6bc574b23c83a3c5ed2e453f3d5e2"
 dependencies = [
+ "bitvec",
+ "funty",
 "memchr",
 "version_check",
 ]
@@ -1197,10 +1179,10 @@ dependencies = [
 "clap",
 "crc32c",
 "daemonize",
- "fs_extra",
 "futures",
 "hex",
 "humantime",
+ "hyper",
 "lazy_static",
 "log",
 "postgres",
@@ -1209,22 +1191,18 @@ dependencies = [
 "postgres_ffi",
 "rand",
 "regex",
- "rocksdb",
- "rust-s3",
+ "routerify",
 "scopeguard",
 "serde",
 "serde_json",
 "slog",
- "slog-async",
 "slog-scope",
 "slog-stdlog",
 "slog-term",
 "tar",
 "thiserror",
 "tokio",
- "tokio-stream",
 "toml",
- "walkdir",
 "workspace_hack",
 "zenith_metrics",
 "zenith_utils",
@@ -1401,6 +1379,7 @@ dependencies = [
 "memoffset",
 "rand",
 "regex",
+ "serde",
 "thiserror",
 "workspace_hack",
 "zenith_utils",
@@ -1412,30 +1391,6 @@ version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"

-[[package]]
-name = "proc-macro-error"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
-dependencies = [
- "proc-macro-error-attr",
- "proc-macro2",
- "quote",
- "syn",
- "version_check",
-]
-
-[[package]]
-name = "proc-macro-error-attr"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
-dependencies = [
- "proc-macro2",
- "quote",
- "version_check",
-]
-
 [[package]]
 name = "proc-macro-hack"
 version = "0.5.19"
@@ -1513,6 +1468,12 @@ dependencies = [
 "proc-macro2",
 ]

+[[package]]
+name = "radium"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8"
+
 [[package]]
 name = "rand"
 version = "0.8.4"
@@ -1649,13 +1610,16 @@ dependencies = [
 ]

 [[package]]
-name = "rocksdb"
-version = "0.16.0"
+name = "routerify"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c749134fda8bfc90d0de643d59bfc841dcb3ac8a1062e12b6754bd60235c48b3"
+checksum = "0c6bb49594c791cadb5ccfa5f36d41b498d40482595c199d10cd318800280bd9"
 dependencies = [
- "libc",
- "librocksdb-sys",
+ "http",
+ "hyper",
+ "lazy_static",
+ "percent-encoding",
+ "regex",
 ]

 [[package]]
@@ -1870,9 +1834,9 @@ dependencies = [

 [[package]]
 name = "shlex"
-version = "0.1.1"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
+checksum = "42a568c8f2cd051a4d283bd6eb0343ac214c1b0f1ac19f93e1175b2dee38c73d"

 [[package]]
 name = "signal-hook-registry"
@@ -1965,21 +1929,6 @@ version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"

-[[package]]
-name = "snapfile"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "aversion",
- "bookfile",
- "hex",
- "rand",
- "serde",
- "structopt",
- "tempfile",
- "zenith_utils",
-]
-
 [[package]]
 name = "socket2"
 version = "0.4.0"
@@ -2012,30 +1961,6 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"

-[[package]]
-name = "structopt"
-version = "0.3.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69b041cdcb67226aca307e6e7be44c8806423d83e018bd662360a93dabce4d71"
-dependencies = [
- "clap",
- "lazy_static",
- "structopt-derive",
-]
-
-[[package]]
-name = "structopt-derive"
-version = "0.4.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7813934aecf5f51a54775e00068c237de98489463968231a51746bbbc03f9c10"
-dependencies = [
- "heck",
- "proc-macro-error",
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "subtle"
 version = "2.4.1"
@@ -2059,6 +1984,12 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60"

+[[package]]
+name = "tap"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
+
 [[package]]
 name = "tar"
 version = "0.4.35"
@@ -2344,12 +2275,6 @@ dependencies = [
 "tinyvec",
 ]

-[[package]]
-name = "unicode-segmentation"
-version = "1.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
-
 [[package]]
 name = "unicode-width"
 version = "0.1.8"
@@ -2609,6 +2534,12 @@ dependencies = [
 "syn",
 ]

+[[package]]
+name = "wyz"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214"
+
 [[package]]
 name = "xattr"
 version = "0.2.2"
@@ -2663,7 +2594,9 @@ dependencies = [
 "log",
 "postgres",
 "rand",
+ "routerify",
 "serde",
+ "serde_json",
 "thiserror",
 "tokio",
 "workspace_hack",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,7 +9,6 @@ members = [
    "zenith",
    "zenith_metrics",
    "zenith_utils",
-    "snapfile",
 ]

 [profile.release]
--- a/83
+++ b/83
@@ -1,94 +1,77 @@
 #
 # Docker image for console integration testing.
 #
-# We may also reuse it in CI to unify installation process and as a general binaries building
-# tool for production servers.
-#
-# Dynamic linking is used for librocksdb and libstdc++ bacause librocksdb-sys calls
-# bindgen with "dynamic" feature flag. This also prevents usage of dockerhub alpine-rust
-# images which are statically linked and have guards against any dlopen. I would rather
-# prefer all static binaries so we may change the way librocksdb-sys builds or wait until
-# we will have our own storage and drop rockdb dependency.
-#
-# Cargo-chef is used to separate dependencies building from main binaries building. This
-# way `docker build` will download and install dependencies only of there are changes to
-# out Cargo.toml files.
-#
-

 #
-# build postgres separately -- this layer will be rebuilt only if one of
-# mentioned paths will get any changes
+# Build Postgres separately --- this layer will be rebuilt only if one of
+# mentioned paths will get any changes.
 #
-FROM alpine:3.13 as pg-build
-RUN apk add --update clang llvm compiler-rt compiler-rt-static lld musl-dev binutils \
-                     make bison flex readline-dev zlib-dev perl linux-headers libseccomp-dev
-WORKDIR zenith
+FROM zenithdb/build:buster AS pg-build
+WORKDIR /zenith
 COPY ./vendor/postgres vendor/postgres
 COPY ./Makefile Makefile
-# Build using clang and lld
-RUN CC='clang' LD='lld' CFLAGS='-fuse-ld=lld --rtlib=compiler-rt' make postgres -j4
+RUN make -j $(getconf _NPROCESSORS_ONLN) -s postgres

 #
 # Calculate cargo dependencies.
 # This will always run, but only generate recipe.json with list of dependencies without
 # installing them.
 #
-FROM alpine:20210212 as cargo-deps-inspect
-RUN apk add --update rust cargo
-RUN cargo install cargo-chef
-WORKDIR zenith
+FROM zenithdb/build:buster AS cargo-deps-inspect
+WORKDIR /zenith
 COPY . .
-RUN cargo chef prepare --recipe-path recipe.json
+RUN cargo chef prepare --recipe-path /zenith/recipe.json

 #
 # Build cargo dependencies.
-# This temp cantainner would be build only if recipe.json was changed.
+# This temp cantainner should be rebuilt only if recipe.json was changed.
 #
-FROM alpine:20210212 as deps-build
-RUN apk add --update rust cargo openssl-dev clang build-base
-# rust-rocksdb can be built against system-wide rocksdb -- that saves about
-# 10 minutes during build. Rocksdb apk package is in testing now, but use it
-# anyway. In case of any troubles we can download and build rocksdb here manually
-# (to cache it as a docker layer).
-RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
-WORKDIR zenith
+FROM zenithdb/build:buster AS deps-build
+WORKDIR /zenith
 COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
-COPY --from=cargo-deps-inspect /root/.cargo/bin/cargo-chef /root/.cargo/bin/
+COPY --from=cargo-deps-inspect /usr/local/cargo/bin/cargo-chef /usr/local/cargo/bin/
 COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
 RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json

 #
 # Build zenith binaries
 #
-FROM alpine:20210212 as build
-RUN apk add --update rust cargo openssl-dev clang build-base
-RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
-WORKDIR zenith
+FROM zenithdb/build:buster AS build
+WORKDIR /zenith
 COPY . .
 # Copy cached dependencies
 COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
 COPY --from=deps-build /zenith/target target
-COPY --from=deps-build /root/.cargo /root/.cargo
+COPY --from=deps-build /usr/local/cargo/ /usr/local/cargo/
 RUN cargo build --release

 #
 # Copy binaries to resulting image.
-# build-base hare to provide libstdc++ (it will also bring gcc, but leave it this way until we figure
-# out how to statically link rocksdb or avoid it at all).
 #
-FROM alpine:3.13
-RUN apk add --update openssl build-base libseccomp-dev
-RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb
+FROM debian:buster-slim
+WORKDIR /data
+
+RUN apt-get update && apt-get -yq install librocksdb-dev libseccomp-dev openssl && \
+    mkdir zenith_install
+
 COPY --from=build /zenith/target/release/pageserver /usr/local/bin
 COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin
 COPY --from=build /zenith/target/release/proxy /usr/local/bin
-COPY --from=pg-build /zenith/tmp_install /usr/local
+COPY --from=pg-build /zenith/tmp_install postgres_install
 COPY docker-entrypoint.sh /docker-entrypoint.sh

-RUN addgroup zenith && adduser -h /data -D -G zenith zenith
+# Remove build artifacts (~ 500 MB)
+RUN rm -rf postgres_install/build && \
+    # 'Install' Postgres binaries locally
+    cp -r postgres_install/* /usr/local/ && \
+    # Prepare an archive of Postgres binaries (should be around 11 MB)
+    # and keep it inside container for an ease of deploy pipeline.
+    cd postgres_install && tar -czf /data/postgres_install.tar.gz . && cd .. && \
+    rm -rf postgres_install
+
+RUN useradd -d /data zenith && chown -R zenith:zenith /data
+
 VOLUME ["/data"]
-WORKDIR /data
 USER zenith
 EXPOSE 6400
 ENTRYPOINT ["/docker-entrypoint.sh"]
--- a/Dockerfile.alpine
+++ b/Dockerfile.alpine
@@ -0,0 +1,95 @@
+#
+# Docker image for console integration testing.
+#
+# We may also reuse it in CI to unify installation process and as a general binaries building
+# tool for production servers.
+#
+# Dynamic linking is used for librocksdb and libstdc++ bacause librocksdb-sys calls
+# bindgen with "dynamic" feature flag. This also prevents usage of dockerhub alpine-rust
+# images which are statically linked and have guards against any dlopen. I would rather
+# prefer all static binaries so we may change the way librocksdb-sys builds or wait until
+# we will have our own storage and drop rockdb dependency.
+#
+# Cargo-chef is used to separate dependencies building from main binaries building. This
+# way `docker build` will download and install dependencies only of there are changes to
+# out Cargo.toml files.
+#
+
+
+#
+# build postgres separately -- this layer will be rebuilt only if one of
+# mentioned paths will get any changes
+#
+FROM alpine:3.13 as pg-build
+RUN apk add --update clang llvm compiler-rt compiler-rt-static lld musl-dev binutils \
+                     make bison flex readline-dev zlib-dev perl linux-headers libseccomp-dev
+WORKDIR zenith
+COPY ./vendor/postgres vendor/postgres
+COPY ./Makefile Makefile
+# Build using clang and lld
+RUN CC='clang' LD='lld' CFLAGS='-fuse-ld=lld --rtlib=compiler-rt' make postgres -j4
+
+#
+# Calculate cargo dependencies.
+# This will always run, but only generate recipe.json with list of dependencies without
+# installing them.
+#
+FROM alpine:20210212 as cargo-deps-inspect
+RUN apk add --update rust cargo
+RUN cargo install cargo-chef
+WORKDIR zenith
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+#
+# Build cargo dependencies.
+# This temp cantainner would be build only if recipe.json was changed.
+#
+FROM alpine:20210212 as deps-build
+RUN apk add --update rust cargo openssl-dev clang build-base
+# rust-rocksdb can be built against system-wide rocksdb -- that saves about
+# 10 minutes during build. Rocksdb apk package is in testing now, but use it
+# anyway. In case of any troubles we can download and build rocksdb here manually
+# (to cache it as a docker layer).
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
+WORKDIR zenith
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=cargo-deps-inspect /root/.cargo/bin/cargo-chef /root/.cargo/bin/
+COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
+RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json
+
+#
+# Build zenith binaries
+#
+FROM alpine:20210212 as build
+RUN apk add --update rust cargo openssl-dev clang build-base
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
+WORKDIR zenith
+COPY . .
+# Copy cached dependencies
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=deps-build /zenith/target target
+COPY --from=deps-build /root/.cargo /root/.cargo
+RUN cargo build --release
+
+#
+# Copy binaries to resulting image.
+# build-base hare to provide libstdc++ (it will also bring gcc, but leave it this way until we figure
+# out how to statically link rocksdb or avoid it at all).
+#
+FROM alpine:3.13
+RUN apk add --update openssl build-base libseccomp-dev
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb
+COPY --from=build /zenith/target/release/pageserver /usr/local/bin
+COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin
+COPY --from=build /zenith/target/release/proxy /usr/local/bin
+COPY --from=pg-build /zenith/tmp_install /usr/local
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+
+RUN addgroup zenith && adduser -h /data -D -G zenith zenith
+VOLUME ["/data"]
+WORKDIR /data
+USER zenith
+EXPOSE 6400
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["pageserver"]
--- a/Dockerfile.build
+++ b/Dockerfile.build
@@ -0,0 +1,15 @@
+#
+# Image with all the required dependencies to build https://github.com/zenithdb/zenith
+# and Postgres from https://github.com/zenithdb/postgres
+# Also includes some rust development and build tools.
+#
+FROM rust:slim-buster
+WORKDIR /zenith
+
+# Install postgres and zenith build dependencies
+# clang is for rocksdb
+RUN apt-get update && apt-get -yq install automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \
+                                          libseccomp-dev pkg-config libssl-dev librocksdb-dev clang
+
+# Install rust tools
+RUN rustup component add clippy && cargo install cargo-chef cargo-audit
--- a/12
+++ b/12
@@ -21,6 +21,7 @@ all: zenith postgres
 ### Zenith Rust bits
 #
 # The 'postgres_ffi' depends on the Postgres headers.
+.PHONY: zenith
 zenith: postgres-headers
 	cargo build

@@ -37,15 +38,18 @@ tmp_install/build/config.status:
 		--prefix=$(abspath tmp_install) > configure.log)

 # nicer alias for running 'configure'
+.PHONY: postgres-configure
 postgres-configure: tmp_install/build/config.status

 # Install the PostgreSQL header files into tmp_install/include
+.PHONY: postgres-headers
 postgres-headers: postgres-configure
 	+@echo "Installing PostgreSQL headers"
 	$(MAKE) -C tmp_install/build/src/include MAKELEVEL=0 install


 # Compile and install PostgreSQL and contrib/zenith
+.PHONY: postgres
 postgres: postgres-configure
 	+@echo "Compiling PostgreSQL"
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 install
@@ -67,4 +71,10 @@ distclean:
 	rm -rf tmp_install
 	cargo clean

-.PHONY: postgres-configure postgres postgres-headers zenith
+.PHONY: fmt
+fmt:
+	./pre-commit.py --fix-inplace
+
+.PHONY: setup-pre-commit-hook
+setup-pre-commit-hook:
+	ln -s -f ../../pre-commit.py .git/hooks/pre-commit
--- a/README.md
+++ b/README.md
@@ -2,6 +2,22 @@

 Zenith substitutes PostgreSQL storage layer and redistributes data across a cluster of nodes

+## Architecture overview
+
+A Zenith installation consists of Compute nodes and Storage engine.
+
+Compute nodes are stateles PostgreSQL nodes, backed by zenith storage.
+
+Zenith storage engine consists of two major components:
+- Pageserver. Scalable storage backend for compute nodes.
+- WAL service. The service that receives WAL from compute node and ensures that it is stored durably.
+
+Pageserver consists of:
+- Repository - Zenith storage implementation.
+- WAL receiver - service that receives WAL from WAL service and stores it in the repository.
+- Page service - service that communicates with compute nodes and responds with pages from the repository.
+- WAL redo - service that builds pages from base images and WAL records on Page service request.
+
 ## Running local installation

 1. Install build dependencies and other useful packages
@@ -12,15 +28,12 @@ apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libsec
 libssl-dev clang
 ```

-[Rust] 1.48 or later is also required.
+[Rust] 1.52 or later is also required.

 To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively.

 To run the integration tests (not required to use the code), install
-Python (3.6 or higher), and install python3 packages with `pip` (called `pip3` on some systems):
-```
-pip install pytest psycopg2
-```
+Python (3.6 or higher), and install python3 packages with `pipenv` using `pipenv install` in the project directory.

 2. Build zenith and patched postgres
 ```sh
@@ -54,7 +67,7 @@ main	127.0.0.1:55432	0/1609610	running

 4. Now it is possible to connect to postgres and run some queries:
 ```text
-> psql -p55432 -h 127.0.0.1 postgres
+> psql -p55432 -h 127.0.0.1 -U zenith_admin postgres
 postgres=# CREATE TABLE t(key int primary key, value text);
 CREATE TABLE
 postgres=# insert into t values(1,1);
@@ -84,7 +97,7 @@ waiting for server to start.... done

 # this new postgres instance will have all the data from 'main' postgres,
 # but all modifications would not affect data in original postgres
-> psql -p55433 -h 127.0.0.1 postgres
+> psql -p55433 -h 127.0.0.1 -U zenith_admin postgres
 postgres=# select * from t;
 key | value
 -----+-------
@@ -98,7 +111,7 @@ INSERT 0 1
 ## Running tests

 ```sh
-git clone --recursive https://github.com/libzenith/zenith.git
+git clone --recursive https://github.com/zenithdb/zenith.git
 make # builds also postgres and installs it to ./tmp_install
 cd test_runner
 pytest
@@ -106,55 +119,15 @@ pytest

 ## Documentation

-Now we use README files to cover design ideas and overall architecture for each module.
-And rustdoc style documentation comments.
+Now we use README files to cover design ideas and overall architecture for each module and `rustdoc` style documentation comments. See also [/docs/](/docs/) a top-level overview of all available markdown documentation.

-To view your documentation in a browser, try running `cargo doc --no-deps --open`
+- [/docs/sourcetree.md](/docs/sourcetree.md) contains overview of source tree layout.

-## Source tree layout
+To view your `rustdoc` documentation in a browser, try running `cargo doc --no-deps --open`

-`/control_plane`:
+## Join the development

-Local control plane.
-Functions to start, configure and stop pageserver and postgres instances running as a local processes.
-Intended to be used in integration tests and in CLI tools for local installations.
-
-`/zenith`
-
-Main entry point for the 'zenith' CLI utility.
-TODO: Doesn't it belong to control_plane?
-
-`/postgres_ffi`:
-
-Utility functions for interacting with PostgreSQL file formats.
-Misc constants, copied from PostgreSQL headers.
-
-`/zenith_utils`:
-
-Helpers that are shared between other crates in this repository.
-
-`/walkeeper`:
-
-WAL safekeeper (also known as WAL acceptor). Written in Rust.
-
-`/pageserver`:
-
-Page Server. Written in Rust.
-
-Depends on the modified 'postgres' binary for WAL redo.
-
-`/vendor/postgres`:
-
-PostgreSQL source tree, with the modifications needed for Zenith.
-
-`/vendor/postgres/contrib/zenith`:
-
-PostgreSQL extension that implements storage manager API and network communications with remote page server.
-
-`/test_runner`:
-
-Integration tests, written in Python using the `pytest` framework.
-
-`test_runner/zenith_regress`:
-
-Quick way to add new SQL regression test to integration tests set.
+- Read `CONTRIBUTING.md` to learn about project code style and practices.
+- Use glossary in [/docs/glossary.md](/docs/glossary.md)
+- To get familiar with a source tree layout, use [/docs/sourcetree.md](/docs/sourcetree.md).
+- To learn more about PostgreSQL internals, check http://www.interdb.jp/pg/index.html
--- a/control_plane/Cargo.toml
+++ b/control_plane/Cargo.toml
@@ -20,6 +20,7 @@ bytes = "1.0.1"
 nix = "0.20"
 url = "2.2.2"
 hex = { version = "0.4.3", features = ["serde"] }
+reqwest = { version = "0.11", features = ["blocking", "json"] }

 pageserver = { path = "../pageserver" }
 walkeeper = { path = "../walkeeper" }
--- a/control_plane/src/compute.rs
+++ b/control_plane/src/compute.rs
@@ -1,3 +1,4 @@
+use std::fs::{self, File, OpenOptions};
 use std::io::Write;
 use std::net::SocketAddr;
 use std::net::TcpStream;
@@ -6,10 +7,6 @@ use std::process::Command;
 use std::sync::Arc;
 use std::time::Duration;
 use std::{collections::BTreeMap, path::PathBuf};
-use std::{
-    fs::{self, File, OpenOptions},
-    io::Read,
-};

 use anyhow::{Context, Result};
 use lazy_static::lazy_static;
@@ -234,6 +231,8 @@ impl PostgresNode {
        })
    }

+    /// Get basebackup from the pageserver as a tar archive and extract it
+    /// to the `self.pgdata()` directory.
    pub fn do_basebackup(&self) -> Result<()> {
        let pgdata = self.pgdata();

@@ -243,26 +242,20 @@ impl PostgresNode {
            .page_server_psql_client()
            .with_context(|| "connecting to page server failed")?;

-        let mut copyreader = client
+        let copyreader = client
            .copy_out(sql.as_str())
            .with_context(|| "page server 'basebackup' command failed")?;

-        // FIXME: Currently, we slurp the whole tarball into memory, and then extract it,
-        // but we really should do this:
-        //let mut ar = tar::Archive::new(copyreader);
-        let mut buf = vec![];
-        copyreader
-            .read_to_end(&mut buf)
-            .with_context(|| "reading base backup from page server failed")?;
-        let mut ar = tar::Archive::new(buf.as_slice());
-        ar.unpack(&pgdata)
+        // Read the archive directly from the `CopyOutReader`
+        tar::Archive::new(copyreader)
+            .unpack(&pgdata)
            .with_context(|| "extracting page backup failed")?;

        Ok(())
    }

-    // Connect to a page server, get base backup, and untar it to initialize a
-    // new data directory
+    /// Connect to a pageserver, get basebackup, and untar it to initialize a
+    /// new data directory
    pub fn init_from_page_server(&self, auth_type: AuthType, config_only: bool) -> Result<()> {
        let pgdata = self.pgdata();

@@ -320,7 +313,7 @@ impl PostgresNode {

        // Never clean up old WAL. TODO: We should use a replication
        // slot or something proper, to prevent the compute node
-        // from removing WAL that hasn't been streamed to the safekeepr or
+        // from removing WAL that hasn't been streamed to the safekeeper or
        // page server yet. (gh issue #349)
        self.append_conf("postgresql.conf", "wal_keep_size='10TB'\n")?;

@@ -332,7 +325,7 @@ impl PostgresNode {
        };

        // Configure that node to take pages from pageserver
-        let (host, port) = connection_host_port(&self.pageserver.connection_config);
+        let (host, port) = connection_host_port(&self.pageserver.pg_connection_config);
        self.append_conf(
            "postgresql.conf",
            format!(
--- a/control_plane/src/local_env.rs
+++ b/control_plane/src/local_env.rs
@@ -42,9 +42,6 @@ pub struct LocalEnv {
    #[serde(with = "hex")]
    pub tenantid: ZTenantId,

-    // Repository format, 'rocksdb' or 'layered' or None for default
-    pub repository_format: Option<String>,
-
    // jwt auth token used for communication with pageserver
    pub auth_token: String,

@@ -104,7 +101,6 @@ pub fn init(
    remote_pageserver: Option<&str>,
    tenantid: ZTenantId,
    auth_type: AuthType,
-    repository_format: Option<&str>,
 ) -> Result<()> {
    // check if config already exists
    let base_path = base_path();
@@ -180,7 +176,6 @@ pub fn init(
            base_data_dir: base_path,
            remotes: BTreeMap::default(),
            tenantid,
-            repository_format: repository_format.map(|x| x.into()),
            auth_token,
            auth_type,
            private_key_path,
@@ -199,7 +194,6 @@ pub fn init(
            base_data_dir: base_path,
            remotes: BTreeMap::default(),
            tenantid,
-            repository_format: repository_format.map(|x| x.into()),
            auth_token,
            auth_type,
            private_key_path,
--- a/control_plane/src/storage.rs
+++ b/control_plane/src/storage.rs
@@ -5,10 +5,13 @@ use std::process::Command;
 use std::thread;
 use std::time::Duration;

-use anyhow::{anyhow, bail, Result};
+use anyhow::{anyhow, bail, ensure, Result};
 use nix::sys::signal::{kill, Signal};
 use nix::unistd::Pid;
+use pageserver::http::models::{BranchCreateRequest, TenantCreateRequest};
 use postgres::{Config, NoTls};
+use reqwest::blocking::{Client, RequestBuilder};
+use reqwest::{IntoUrl, Method, StatusCode};
 use zenith_utils::postgres_backend::AuthType;
 use zenith_utils::zid::ZTenantId;

@@ -17,6 +20,8 @@ use crate::read_pidfile;
 use pageserver::branches::BranchInfo;
 use zenith_utils::connstring::connection_address;

+const HTTP_BASE_URL: &str = "http://127.0.0.1:9898/v1";
+
 //
 // Control routines for pageserver.
 //
@@ -25,13 +30,15 @@ use zenith_utils::connstring::connection_address;
 #[derive(Debug)]
 pub struct PageServerNode {
    pub kill_on_exit: bool,
-    pub connection_config: Config,
+    pub pg_connection_config: Config,
    pub env: LocalEnv,
+    pub http_client: Client,
+    pub http_base_url: String,
 }

 impl PageServerNode {
    pub fn from_env(env: &LocalEnv) -> PageServerNode {
-        let password = if matches!(env.auth_type, AuthType::ZenithJWT) {
+        let password = if env.auth_type == AuthType::ZenithJWT {
            &env.auth_token
        } else {
            ""
@@ -39,8 +46,10 @@ impl PageServerNode {

        PageServerNode {
            kill_on_exit: false,
-            connection_config: Self::default_config(password), // default
+            pg_connection_config: Self::default_config(password), // default
            env: env.clone(),
+            http_client: Client::new(),
+            http_base_url: HTTP_BASE_URL.to_owned(),
        }
    }

@@ -50,12 +59,7 @@ impl PageServerNode {
            .unwrap()
    }

-    pub fn init(
-        &self,
-        create_tenant: Option<&str>,
-        enable_auth: bool,
-        repository_format: Option<&str>,
-    ) -> Result<()> {
+    pub fn init(&self, create_tenant: Option<&str>, enable_auth: bool) -> Result<()> {
        let mut cmd = Command::new(self.env.pageserver_bin()?);
        let mut args = vec![
            "--init",
@@ -70,10 +74,6 @@ impl PageServerNode {
            args.extend(&["--auth-type", "ZenithJWT"]);
        }

-        if let Some(repo_format) = repository_format {
-            args.extend(&["--repository-format", repo_format]);
-        }
-
        create_tenant.map(|tenantid| args.extend(&["--create-tenant", tenantid]));
        let status = cmd
            .args(args)
@@ -100,7 +100,7 @@ impl PageServerNode {
    pub fn start(&self) -> Result<()> {
        println!(
            "Starting pageserver at '{}' in {}",
-            connection_address(&self.connection_config),
+            connection_address(&self.pg_connection_config),
            self.repo_path().display()
        );

@@ -120,7 +120,7 @@ impl PageServerNode {
        // It takes a while for the page server to start up. Wait until it is
        // open for business.
        for retries in 1..15 {
-            match self.page_server_psql_client() {
+            match self.check_status() {
                Ok(_) => {
                    println!("Pageserver started");
                    return Ok(());
@@ -145,7 +145,7 @@ impl PageServerNode {
        }

        // wait for pageserver stop
-        let address = connection_address(&self.connection_config);
+        let address = connection_address(&self.pg_connection_config);
        for _ in 0..5 {
            let stream = TcpStream::connect(&address);
            thread::sleep(Duration::from_secs(1));
@@ -160,50 +160,64 @@ impl PageServerNode {
    }

    pub fn page_server_psql(&self, sql: &str) -> Vec<postgres::SimpleQueryMessage> {
-        let mut client = self.connection_config.connect(NoTls).unwrap();
+        let mut client = self.pg_connection_config.connect(NoTls).unwrap();

        println!("Pageserver query: '{}'", sql);
        client.simple_query(sql).unwrap()
    }

    pub fn page_server_psql_client(&self) -> Result<postgres::Client, postgres::Error> {
-        self.connection_config.connect(NoTls)
+        self.pg_connection_config.connect(NoTls)
    }

-    pub fn tenants_list(&self) -> Result<Vec<String>> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query("tenant_list")?;
-        let tenants_json = query_result
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
-            })
-            .flatten()
-            .ok_or_else(|| anyhow!("missing tenants"))?;
-
-        Ok(serde_json::from_str(tenants_json)?)
+    fn http_request<U: IntoUrl>(&self, method: Method, url: U) -> RequestBuilder {
+        let mut builder = self.http_client.request(method, url);
+        if self.env.auth_type == AuthType::ZenithJWT {
+            builder = builder.bearer_auth(&self.env.auth_token)
+        }
+        builder
    }

-    pub fn tenant_create(&self, tenantid: &ZTenantId) -> Result<()> {
-        let mut client = self.page_server_psql_client()?;
-        client.simple_query(format!("tenant_create {}", tenantid).as_str())?;
+    pub fn check_status(&self) -> Result<()> {
+        let status = self
+            .http_request(Method::GET, format!("{}/{}", self.http_base_url, "status"))
+            .send()?
+            .status();
+        ensure!(
+            status == StatusCode::OK,
+            format!("got unexpected response status {}", status)
+        );
        Ok(())
    }

-    pub fn branches_list(&self, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query(&format!("branch_list {}", tenantid))?;
-        let branches_json = query_result
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
-            })
-            .flatten()
-            .ok_or_else(|| anyhow!("missing branches"))?;
+    pub fn tenant_list(&self) -> Result<Vec<String>> {
+        Ok(self
+            .http_request(Method::GET, format!("{}/{}", self.http_base_url, "tenant"))
+            .send()?
+            .error_for_status()?
+            .json()?)
+    }

-        Ok(serde_json::from_str(branches_json)?)
+    pub fn tenant_create(&self, tenantid: ZTenantId) -> Result<()> {
+        Ok(self
+            .http_request(Method::POST, format!("{}/{}", self.http_base_url, "tenant"))
+            .json(&TenantCreateRequest {
+                tenant_id: tenantid,
+            })
+            .send()?
+            .error_for_status()?
+            .json()?)
+    }
+
+    pub fn branch_list(&self, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
+        Ok(self
+            .http_request(
+                Method::GET,
+                format!("{}/branch/{}", self.http_base_url, tenantid),
+            )
+            .send()?
+            .error_for_status()?
+            .json()?)
    }

    pub fn branch_create(
@@ -212,29 +226,16 @@ impl PageServerNode {
        startpoint: &str,
        tenantid: &ZTenantId,
    ) -> Result<BranchInfo> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query(
-            format!("branch_create {} {} {}", tenantid, branch_name, startpoint).as_str(),
-        )?;
-
-        let branch_json = query_result
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
+        Ok(self
+            .http_request(Method::POST, format!("{}/{}", self.http_base_url, "branch"))
+            .json(&BranchCreateRequest {
+                tenant_id: tenantid.to_owned(),
+                name: branch_name.to_owned(),
+                start_point: startpoint.to_owned(),
            })
-            .flatten()
-            .ok_or_else(|| anyhow!("missing branch"))?;
-
-        let res: BranchInfo = serde_json::from_str(branch_json).map_err(|e| {
-            anyhow!(
-                "failed to parse branch_create response: {}: {}",
-                branch_json,
-                e
-            )
-        })?;
-
-        Ok(res)
+            .send()?
+            .error_for_status()?
+            .json()?)
    }

    // TODO: make this a separate request type and avoid loading all the branches
@@ -243,14 +244,14 @@ impl PageServerNode {
        tenantid: &ZTenantId,
        branch_name: &str,
    ) -> Result<BranchInfo> {
-        let branch_infos = self.branches_list(tenantid)?;
-        let branche_by_name: Result<HashMap<String, BranchInfo>> = branch_infos
+        let branch_infos = self.branch_list(tenantid)?;
+        let branch_by_name: Result<HashMap<String, BranchInfo>> = branch_infos
            .into_iter()
            .map(|branch_info| Ok((branch_info.name.clone(), branch_info)))
            .collect();
-        let branche_by_name = branche_by_name?;
+        let branch_by_name = branch_by_name?;

-        let branch = branche_by_name
+        let branch = branch_by_name
            .get(branch_name)
            .ok_or_else(|| anyhow!("Branch {} not found", branch_name))?;

--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -1,6 +1,8 @@
 #!/bin/sh
+set -eux
+
 if [ "$1" = 'pageserver' ]; then
-    if [ ! -d "/data/timelines" ]; then
+    if [ ! -d "/data/tenants" ]; then
        echo "Initializing pageserver data directory"
        pageserver --init -D /data --postgres-distrib /usr/local
    fi
--- a/docs/README.md
+++ b/docs/README.md
@@ -0,0 +1,13 @@
+# Zenith documentation
+
+## Table of contents
+
+- [authentication.md](authentication.md) — pageserver JWT authentication.
+- [docker.md](docker.md) — Docker images and building pipeline.
+- [glossary.md](glossary.md) — Glossary of all the terms used in codebase.
+- [multitenancy.md](multitenancy.md) — how multitenancy is organized in the pageserver and Zenith CLI.
+- [sourcetree.md](sourcetree.md) — Overview of the source tree layeout.
+- [pageserver/README](/pageserver/README) — pageserver overview.
+- [postgres_ffi/README](/postgres_ffi/README) — Postgres FFI overview.
+- [test_runner/README.md](/test_runner/README.md) — tests infrastructure overview.
+- [walkeeper/README](/walkeeper/README.md) — WAL service overview.
--- a/docs/docker.md
+++ b/docs/docker.md
@@ -0,0 +1,38 @@
+# Docker images of Zenith
+
+## Images
+
+Currently we build two main images:
+
+- [zenithdb/zenith](https://hub.docker.com/repository/docker/zenithdb/zenith) — image with pre-built `pageserver`, `wal_acceptor` and `proxy` binaries and all the required runtime dependencies. Built from [/Dockerfile](/Dockerfile).
+- [zenithdb/compute-node](https://hub.docker.com/repository/docker/zenithdb/compute-node) — compute node image with pre-built Postgres binaries from [zenithdb/postgres](https://github.com/zenithdb/postgres).
+
+And two intermediate images used either to reduce build time or to deliver some additional binary tools from other repos:
+
+- [zenithdb/build](https://hub.docker.com/repository/docker/zenithdb/build) — image with all the dependencies required to build Zenith and compute node images. This image is based on `rust:slim-buster`, so it also has a proper `rust` environment. Built from [/Dockerfile.build](/Dockerfile.build).
+- [zenithdb/compute-tools](https://hub.docker.com/repository/docker/zenithdb/compute-tools) — compute node configuration management tools.
+
+## Building pipeline
+
+1. Image `zenithdb/compute-tools` is re-built automatically.
+
+2. Image `zenithdb/build` is built manually. If you want to introduce any new compile time dependencies to Zenith or compute node you have to update this image as well, build it and push to Docker Hub.
+
+Build:
+```sh
+docker build -t zenithdb/build:buster -f Dockerfile.build .
+```
+
+Login:
+```sh
+docker login
+```
+
+Push to Docker Hub:
+```sh
+docker push zenithdb/build:buster
+```
+
+3. Image `zenithdb/compute-node` is built independently in the [zenithdb/postgres](https://github.com/zenithdb/postgres) repo.
+
+4. Image `zenithdb/zenith` is built in this repo after a successful `release` tests run and pushed to Docker Hub automatically.
--- a/docs/glossary.md
+++ b/docs/glossary.md
@@ -0,0 +1,185 @@
+# Glossary
+
+### Authentication
+
+### Base image (page image)
+
+### Basebackup
+
+A tarball with files needed to bootstrap a compute node[] and a corresponding command to create it.
+NOTE:It has nothing to do with PostgreSQL pg_basebackup.
+
+### Branch
+
+We can create branch at certain LSN using `zenith branch` command.
+Each Branch lives in a corresponding timeline[] and has an ancestor[].
+
+
+### Checkpoint (PostgreSQL)
+
+NOTE: This is an overloaded term.
+
+A checkpoint record in the WAL marks a point in the WAL sequence at which it is guaranteed that all data files have been updated with all information from shared memory modified before that checkpoint; 
+
+### Checkpoint (Layered repository)
+
+NOTE: This is an overloaded term.
+
+Whenever enough WAL has been accumulated in memory, the page server []
+writes out the changes in memory into new layer files[]. This process
+is called "checkpointing". The page server only creates layer files for
+relations that have been modified since the last checkpoint. 
+
+### Compute node
+
+Stateless Postgres node that stores data in pageserver.
+
+### Garbage collection
+
+### Fork
+
+Each of the separate segmented file sets in which a relation is stored. The main fork is where the actual data resides. There also exist two secondary forks for metadata: the free space map and the visibility map.
+Each PostgreSQL fork is considered a separate relish.
+
+### Layer file
+
+Layered repository on-disk format is based on immutable files.  The
+files are called "layer files". Each file corresponds to one 10 MB
+segment of a PostgreSQL relation fork. There are two kinds of layer
+files: image files and delta files. An image file contains a
+"snapshot" of the segment at a particular LSN, and a delta file
+contains WAL records applicable to the segment, in a range of LSNs.
+
+### Layered repository
+
+### LSN
+
+
+### Page (block)
+
+The basic structure used to store relation data. All pages are of the same size.
+This is the unit of data exchange between compute node and pageserver.
+
+### Pageserver
+
+Zenith storage engine: repositories + wal receiver + page service + wal redo.
+
+### Page service
+
+The Page Service listens for GetPage@LSN requests from the Compute Nodes,
+and responds with pages from the repository.
+
+
+### PITR (Point-in-time-recovery)
+
+PostgreSQL's ability to restore up to a specified LSN.
+
+### Primary node
+
+
+### Proxy
+
+Postgres protocol proxy/router.
+This service listens psql port, can check auth via external service
+and create new databases and accounts (control plane API in our case).
+
+### Relation
+
+The generic term in PostgreSQL for all objects in a database that have a name and a list of attributes defined in a specific order.
+
+### Relish
+
+We call each relation and other file that is stored in the
+repository a "relish". It comes from "rel"-ish, as in "kind of a
+rel", because it covers relations as well as other things that are
+not relations, but are treated similarly for the purposes of the
+storage layer.
+
+### Replication slot
+
+
+### Replica node
+
+
+### Repository
+
+Repository stores multiple timelines, forked off from the same initial call to 'initdb'
+and has associated WAL redo service.
+One repository corresponds to one Tenant.
+
+### Retention policy
+
+How much history do we need to keep around for PITR and read-only nodes?
+
+### SLRU
+
+SLRUs include pg_clog, pg_multixact/members, and
+pg_multixact/offsets. There are other SLRUs in PostgreSQL, but
+they don't need to be stored permanently (e.g. pg_subtrans),
+or we do not support them in zenith yet (pg_commit_ts).
+Each SLRU segment is considered a separate relish[].
+
+### Tenant (Multitenancy)
+Tenant represents a single customer, interacting with Zenith.
+Wal redo[] activity, timelines[], snapshots[] are managed for each tenant independently.
+One pageserver[] can serve multiple tenants at once.
+One safekeeper 
+
+See `docs/multitenancy.md` for more.
+
+### Timeline
+
+Timeline accepts page changes and serves get_page_at_lsn() and
+get_rel_size() requests. The term "timeline" is used internally
+in the system, but to users they are exposed as "branches", with
+human-friendly names.
+
+NOTE: this has nothing to do with PostgreSQL WAL timelines.
+
+### XLOG
+
+PostgreSQL alias for WAL[].
+
+### WAL (Write-ahead log)
+
+The journal that keeps track of the changes in the database cluster as user- and system-invoked operations take place. It comprises many individual WAL records[] written sequentially to WAL files[].
+
+### WAL acceptor, WAL proposer
+
+In the context of the consensus algorithm, the Postgres
+compute node is also known as the WAL proposer, and the safekeeper is also known
+as the acceptor. Those are the standard terms in the Paxos algorithm.
+
+### WAL receiver (WAL decoder)
+
+The WAL receiver connects to the external WAL safekeeping service (or
+directly to the primary) using PostgreSQL physical streaming
+replication, and continuously receives WAL. It decodes the WAL records,
+and stores them to the repository.
+
+We keep one WAL receiver active per timeline.
+
+### WAL record
+
+A low-level description of an individual data change.
+
+### WAL redo
+
+A service that runs PostgreSQL in a special wal_redo mode
+to apply given WAL records over an old page image and return new page image.
+
+### WAL safekeeper
+
+One node that participates in the quorum. All the safekeepers
+together form the WAL service.
+
+### WAL segment (WAL file)
+
+Also known as WAL segment or WAL segment file. Each of the sequentially-numbered files that provide storage space for WAL. The files are all of the same predefined size and are written in sequential order, interspersing changes as they occur in multiple simultaneous sessions.
+
+### WAL service
+
+The service as whole that ensures that WAL is stored durably.
+
+### Web console
+
--- a/docs/sourcetree.md
+++ b/docs/sourcetree.md
@@ -0,0 +1,81 @@
+## Source tree layout
+
+Below you will find a brief overview of each subdir in the source tree in alphabetical order.
+
+`/control_plane`:
+
+Local control plane.
+Functions to start, configure and stop pageserver and postgres instances running as a local processes.
+Intended to be used in integration tests and in CLI tools for local installations.
+
+`/docs`:
+
+Documentaion of the Zenith features and concepts.
+Now it is mostly dev documentation.
+
+`/monitoring`:
+
+TODO
+
+`/pageserver`:
+
+Zenith storage service.
+The pageserver has a few different duties:
+
+- Store and manage the data.
+- Generate a tarball with files needed to bootstrap ComputeNode.
+- Respond to GetPage@LSN requests from the Compute Nodes.
+- Receive WAL from the WAL service and decode it.
+- Replay WAL that's applicable to the chunks that the Page Server maintains
+
+For more detailed info, see `/pageserver/README`
+
+`/postgres_ffi`:
+
+Utility functions for interacting with PostgreSQL file formats.
+Misc constants, copied from PostgreSQL headers.
+
+`/proxy`:
+
+Postgres protocol proxy/router.
+This service listens psql port, can check auth via external service
+and create new databases and accounts (control plane API in our case).
+
+`/test_runner`:
+
+Integration tests, written in Python using the `pytest` framework.
+
+`/vendor/postgres`:
+
+PostgreSQL source tree, with the modifications needed for Zenith.
+
+`/vendor/postgres/contrib/zenith`:
+
+PostgreSQL extension that implements storage manager API and network communications with remote page server.
+
+`/vendor/postgres/contrib/zenith_test_utils`:
+
+PostgreSQL extension that contains functions needed for testing and debugging.
+
+`/walkeeper`:
+
+The zenith WAL service that receives WAL from a primary compute nodes and streams it to the pageserver.
+It acts as a holding area and redistribution center for recently generated WAL.
+
+For more detailed info, see `/walkeeper/README`
+
+`/workspace_hack`:
+The workspace_hack crate exists only to pin down some dependencies.
+
+`/zenith`
+
+Main entry point for the 'zenith' CLI utility.
+TODO: Doesn't it belong to control_plane?
+
+`/zenith_metrics`:
+
+Helpers for exposing Prometheus metrics from the server.
+
+`/zenith_utils`:
+
+Helpers that are shared between other crates in this repository.
--- a/pageserver/Cargo.toml
+++ b/pageserver/Cargo.toml
@@ -14,33 +14,28 @@ regex = "1.4.5"
 bytes = { version = "1.0.1", features = ['serde'] }
 byteorder = "1.4.3"
 futures = "0.3.13"
+hyper = "0.14"
 lazy_static = "1.4.0"
 slog-stdlog = "4.1.0"
-slog-async = "2.6.0"
 slog-scope = "4.4.0"
 slog-term = "2.8.0"
 slog = "2.7.0"
 log = "0.4.14"
 clap = "2.33.0"
 daemonize = "0.4.1"
-rust-s3 = { version = "0.27.0-rc4", features = ["no-verify-ssl"] }
 tokio = { version = "1.5.0", features = ["full"] }
-tokio-stream = { version = "0.1.5" }
 postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-# by default rust-rocksdb tries to build a lot of compression algos. Use lz4 only for now as it is simplest dependency.
-rocksdb = { version = "0.16.0", features = ["lz4"], default-features = false }
+routerify = "2"
 anyhow = "1.0"
 crc32c = "0.6.0"
-walkdir = "2"
 thiserror = "1.0"
-hex = "0.4.3"
+hex = { version = "0.4.3", features = ["serde"] }
 tar = "0.4.33"
 humantime = "2.1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
-fs_extra = "1.2.0"
 toml = "0.5"
 scopeguard = "1.1.0"

--- a/pageserver/README
+++ b/pageserver/README
@@ -1,16 +1,5 @@
 ## Page server architecture

-The Page Server is responsible for all operations on a number of
-"chunks" of relation data. A chunk corresponds to a PostgreSQL
-relation segment (i.e. one max. 1 GB file in the data directory), but
-it holds all the different versions of every page in the segment that
-are still needed by the system.
-
-Currently we do not specifically organize data in chunks.
-All page images and corresponding WAL records are stored as entries in a key-value storage,
-where StorageKey is a zenith_timeline_id + BufferTag + LSN.
-
-
 The Page Server has a few different duties:

 - Respond to GetPage@LSN requests from the Compute Nodes
@@ -19,10 +8,11 @@ The Page Server has a few different duties:
 - Backup to S3


-The Page Server consists of multiple threads that operate on a shared
-cache of page versions:


+The Page Server consists of multiple threads that operate on a shared
+repository of page versions:
+
                                           | WAL
                                           V
                                   +--------------+
@@ -34,16 +24,14 @@ cache of page versions:
                  +---------+                              ..........            |    |
                  |         |                              .        .            |    |
 GetPage@LSN      |         |                              . backup .  ------->  | S3 |
------------->    |  Page   |         page cache           .        .            |    |
+------------->    |  Page   |         repository           .        .            |    |
                  | Service |                              ..........            |    |
   page           |         |                                                    +----+
 <-------------    |         |
-                  +---------+
-
-                             ...................................
-                             .                                 .
-                             . Garbage Collection / Compaction .
-                             ...................................
+                  +---------+      +--------------------+
+		                   |   Checkpointing /  |
+				   | Garbage collection |
+                                   +--------------------+

 Legend:

@@ -63,7 +51,7 @@ Page Service
 ------------

 The Page Service listens for GetPage@LSN requests from the Compute Nodes,
-and responds with pages from the page cache.
+and responds with pages from the repository.


 WAL Receiver
@@ -72,47 +60,59 @@ WAL Receiver
 The WAL receiver connects to the external WAL safekeeping service (or
 directly to the primary) using PostgreSQL physical streaming
 replication, and continuously receives WAL. It decodes the WAL records,
-and stores them to the page cache.
+and stores them to the repository.


-Page Cache
+Repository
 ----------

-The Page Cache is a switchboard to access different Repositories.
+The repository stores all the page versions, or WAL records needed to
+reconstruct them. Each tenant has a separate Repository, which is
+stored in the .zenith/tenants/<tenantid> directory.

-#### Repository
-Repository corresponds to one .zenith directory.
-Repository is needed to manage Timelines.
+Repository is an abstract trait, defined in `repository.rs`. It is
+implemented by the LayeredRepository object in
+`layered_repository.rs`. There is only that one implementation of the
+Repository trait, but it's still a useful abstraction that keeps the
+interface for the low-level storage functionality clean. The layered
+storage format is described in layered_repository/README.md.

-#### Timeline
-Timeline is a page cache workhorse that accepts page changes
-and serves get_page_at_lsn() and get_rel_size() requests.
-Note: this has nothing to do with PostgreSQL WAL timeline.
+Each repository consists of multiple Timelines. Timeline is a
+workhorse that accepts page changes from the WAL, and serves
+get_page_at_lsn() and get_rel_size() requests. Note: this has nothing
+to do with PostgreSQL WAL timeline. The term "timeline" is mostly
+interchangeable with "branch", there is a one-to-one mapping from
+branch to timeline. A timeline has a unique ID within the tenant,
+represented as 16-byte hex string that never changes, whereas a
+branch is a user-given name for a timeline.

-#### Branch
-We can create branch at certain LSN.
-Each Branch lives in a corresponding timeline and has an ancestor.
-
-To get full snapshot of data at certain moment we need to traverse timeline and its ancestors.
-
-#### ObjectRepository
-ObjectRepository implements Repository and has associated ObjectStore and WAL redo service.
-
-#### ObjectStore
-ObjectStore is an interface for key-value store for page images and wal records.
-Currently it has one implementation - RocksDB.
-
-#### WAL redo service
-WAL redo service - service that runs PostgreSQL in a special wal_redo mode
-to apply given WAL records over an old page image and return new page image.
+Each repository also has a WAL redo manager associated with it, see
+`walredo.rs`. The WAL redo manager is used to replay PostgreSQL WAL
+records, whenever we need to reconstruct a page version from WAL to
+satisfy a GetPage@LSN request, or to avoid accumulating too much WAL
+for a page. The WAL redo manager uses a Postgres process running in
+special zenith wal-redo mode to do the actual WAL redo, and
+communicates with the process using a pipe.


-TODO: Garbage Collection / Compaction
-------------------------------------
+Checkpointing / Garbage Collection
+----------------------------------

-Periodically, the Garbage Collection / Compaction thread runs
-and applies pending WAL records, and removes old page versions that
-are no longer needed.
+Periodically, the checkpointer thread wakes up and performs housekeeping
+duties on the repository. It has two duties:
+
+### Checkpointing
+
+Flush WAL that has accumulated in memory to disk, so that the old WAL
+can be truncated away in the WAL safekeepers. Also, to free up memory
+for receiving new WAL. This process is called "checkpointing". It's
+similar to checkpointing in PostgreSQL or other DBMSs, but in the page
+server, checkpointing happens on a per-segment basis.
+
+### Garbage collection
+
+Remove old on-disk layer files that are no longer needed according to the
+PITR retention policy


 TODO: Backup service
--- a/pageserver/src/basebackup.rs
+++ b/pageserver/src/basebackup.rs
@@ -179,18 +179,18 @@ impl<'a> Basebackup<'a> {
    // Extract twophase state files
    //
    fn add_twophase_file(&mut self, xid: TransactionId) -> anyhow::Result<()> {
-        if let Ok(img) =
-            self.timeline
-                .get_page_at_lsn_nowait(RelishTag::TwoPhase { xid }, 0, self.lsn)
-        {
-            let mut buf = BytesMut::new();
-            buf.extend_from_slice(&img[..]);
-            let crc = crc32c::crc32c(&img[..]);
-            buf.put_u32_le(crc);
-            let path = format!("pg_twophase/{:>08X}", xid);
-            let header = new_tar_header(&path, buf.len() as u64)?;
-            self.ar.append(&header, &buf[..])?;
-        }
+        let img = self
+            .timeline
+            .get_page_at_lsn_nowait(RelishTag::TwoPhase { xid }, 0, self.lsn)?;
+
+        let mut buf = BytesMut::new();
+        buf.extend_from_slice(&img[..]);
+        let crc = crc32c::crc32c(&img[..]);
+        buf.put_u32_le(crc);
+        let path = format!("pg_twophase/{:>08X}", xid);
+        let header = new_tar_header(&path, buf.len() as u64)?;
+        self.ar.append(&header, &buf[..])?;
+
        Ok(())
    }

--- a/pageserver/src/bin/pageserver.rs
+++ b/pageserver/src/bin/pageserver.rs
@@ -10,7 +10,6 @@ use std::{
    path::{Path, PathBuf},
    process::exit,
    str::FromStr,
-    sync::Arc,
    thread,
    time::Duration,
 };
@@ -20,8 +19,8 @@ use anyhow::{ensure, Result};
 use clap::{App, Arg, ArgMatches};
 use daemonize::Daemonize;

-use pageserver::{branches, logger, page_cache, page_service, PageServerConf, RepositoryFormat};
-use zenith_utils::http_endpoint;
+use pageserver::{branches, http, logger, page_service, tenant_mgr, PageServerConf};
+use zenith_utils::http::endpoint;

 const DEFAULT_LISTEN_ADDR: &str = "127.0.0.1:64000";
 const DEFAULT_HTTP_ENDPOINT_ADDR: &str = "127.0.0.1:9898";
@@ -41,7 +40,6 @@ struct CfgFileParams {
    pg_distrib_dir: Option<String>,
    auth_validation_public_key_path: Option<String>,
    auth_type: Option<String>,
-    repository_format: Option<String>,
 }

 impl CfgFileParams {
@@ -59,7 +57,6 @@ impl CfgFileParams {
            pg_distrib_dir: get_arg("postgres-distrib"),
            auth_validation_public_key_path: get_arg("auth-validation-public-key-path"),
            auth_type: get_arg("auth-type"),
-            repository_format: get_arg("repository-format"),
        }
    }

@@ -76,7 +73,6 @@ impl CfgFileParams {
                .auth_validation_public_key_path
                .or(other.auth_validation_public_key_path),
            auth_type: self.auth_type.or(other.auth_type),
-            repository_format: self.repository_format.or(other.repository_format),
        }
    }

@@ -136,16 +132,6 @@ impl CfgFileParams {
            );
        }

-        let repository_format = match self.repository_format.as_ref() {
-            Some(repo_format_str) if repo_format_str == "rocksdb" => RepositoryFormat::RocksDb,
-            Some(repo_format_str) if repo_format_str == "layered" => RepositoryFormat::Layered,
-            Some(repo_format_str) => anyhow::bail!(
-                "invalid --repository-format '{}', must be 'rocksdb' or 'layered'",
-                repo_format_str
-            ),
-            None => RepositoryFormat::Layered, // default
-        };
-
        Ok(PageServerConf {
            daemonize: false,

@@ -162,8 +148,6 @@ impl CfgFileParams {

            auth_validation_public_key_path,
            auth_type,
-
-            repository_format,
        })
    }
 }
@@ -235,12 +219,6 @@ fn main() -> Result<()> {
                .takes_value(true)
                .help("Authentication scheme type. One of: Trust, MD5, ZenithJWT"),
        )
-        .arg(
-            Arg::with_name("repository-format")
-                .long("repository-format")
-                .takes_value(true)
-                .help("Which repository implementation to use, 'rocksdb' or 'layered'"),
-        )
        .get_matches();

    let workdir = Path::new(arg_matches.value_of("workdir").unwrap_or(".zenith"));
@@ -323,29 +301,34 @@ fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
        }
    }

+    // initialize authentication for incoming connections
+    let auth = match &conf.auth_type {
+        AuthType::Trust | AuthType::MD5 => None,
+        AuthType::ZenithJWT => {
+            // unwrap is ok because check is performed when creating config, so path is set and file exists
+            let key_path = conf.auth_validation_public_key_path.as_ref().unwrap();
+            Some(JwtAuth::from_key_path(key_path)?.into())
+        }
+    };
+    info!("Using auth: {:#?}", conf.auth_type);
+
    // Spawn a new thread for the http endpoint
+    let cloned = auth.clone();
    thread::Builder::new()
-        .name("Metrics thread".into())
-        .spawn(move || http_endpoint::thread_main(conf.http_endpoint_addr.clone()))?;
+        .name("http_endpoint_thread".into())
+        .spawn(move || {
+            let router = http::make_router(conf, cloned);
+            endpoint::serve_thread_main(router, conf.http_endpoint_addr.clone())
+        })?;

    // Check that we can bind to address before starting threads to simplify shutdown
    // sequence if port is occupied.
    info!("Starting pageserver on {}", conf.listen_addr);
    let pageserver_listener = TcpListener::bind(conf.listen_addr.clone())?;

-    // Initialize page cache, this will spawn walredo_thread
-    page_cache::init(conf);
+    // Initialize tenant manager.
+    tenant_mgr::init(conf);

-    // initialize authentication for incoming connections
-    let auth = match &conf.auth_type {
-        AuthType::Trust | AuthType::MD5 => Arc::new(None),
-        AuthType::ZenithJWT => {
-            // unwrap is ok because check is performed when creating config, so path is set and file exists
-            let key_path = conf.auth_validation_public_key_path.as_ref().unwrap();
-            Arc::new(Some(JwtAuth::from_key_path(key_path)?))
-        }
-    };
-    info!("Using auth: {:#?}", conf.auth_type);
    // Spawn a thread to listen for connections. It will spawn further threads
    // for each connection.
    let page_service_thread = thread::Builder::new()
--- a/pageserver/src/branches.rs
+++ b/pageserver/src/branches.rs
@@ -20,15 +20,15 @@ use log::*;
 use zenith_utils::lsn::Lsn;

 use crate::logger;
-use crate::object_repository::ObjectRepository;
-use crate::page_cache;
 use crate::restore_local_repo;
+use crate::tenant_mgr;
 use crate::walredo::WalRedoManager;
-use crate::{repository::Repository, PageServerConf, RepositoryFormat};
+use crate::{repository::Repository, PageServerConf};

 #[derive(Serialize, Deserialize, Clone)]
 pub struct BranchInfo {
    pub name: String,
+    #[serde(with = "hex")]
    pub timeline_id: ZTimelineId,
    pub latest_valid_lsn: Option<Lsn>,
    pub ancestor_id: Option<String>,
@@ -46,15 +46,22 @@ pub fn init_pageserver(conf: &'static PageServerConf, create_tenant: Option<&str
    let (_scope_guard, _log_file) = logger::init_logging(&conf, "pageserver.log")?;
    let _log_guard = slog_stdlog::init()?;

+    // We don't use the real WAL redo manager, because we don't want to spawn the WAL redo
+    // process during repository initialization.
+    //
+    // FIXME: That caused trouble, because the WAL redo manager spawned a thread that launched
+    // initdb in the background, and it kept running even after the "zenith init" had exited.
+    // In tests, we started the  page server immediately after that, so that initdb was still
+    // running in the background, and we failed to run initdb again in the same directory. This
+    // has been solved for the rapid init+start case now, but the general race condition remains
+    // if you restart the server quickly. The WAL redo manager doesn't use a separate thread
+    // anymore, but I think that could still happen.
+    let dummy_redo_mgr = Arc::new(crate::walredo::DummyRedoManager {});
+
    if let Some(tenantid) = create_tenant {
        let tenantid = ZTenantId::from_str(tenantid)?;
        println!("initializing tenantid {}", tenantid);
-        create_repo(
-            conf,
-            tenantid,
-            Arc::new(crate::walredo::DummyRedoManager {}),
-        )
-        .with_context(|| "failed to create repo")?;
+        create_repo(conf, tenantid, dummy_redo_mgr).with_context(|| "failed to create repo")?;
    }
    fs::create_dir_all(conf.tenants_path())?;

@@ -87,31 +94,11 @@ pub fn create_repo(

    let tli = create_timeline(conf, None, &tenantid)?;

-    // We don't use page_cache here, because we don't want to spawn the WAL redo thread during
-    // repository initialization.
-    //
-    // FIXME: That caused trouble, because the WAL redo thread launched initdb in the background,
-    // and it kept running even after the "zenith init" had exited. In tests, we started the
-    // page server immediately after that, so that initdb was still running in the background,
-    // and we failed to run initdb again in the same directory. This has been solved for the
-    // rapid init+start case now, but the general race condition remains if you restart the
-    // server quickly.
-    let repo: Arc<dyn Repository + Sync + Send> =
-        match conf.repository_format {
-            RepositoryFormat::Layered => Arc::new(
-                crate::layered_repository::LayeredRepository::new(conf, wal_redo_manager, tenantid),
-            ),
-            RepositoryFormat::RocksDb => {
-                let obj_store = crate::rocksdb_storage::RocksObjectStore::create(conf, &tenantid)?;
-
-                Arc::new(ObjectRepository::new(
-                    conf,
-                    Arc::new(obj_store),
-                    wal_redo_manager,
-                    tenantid,
-                ))
-            }
-        };
+    let repo = Arc::new(crate::layered_repository::LayeredRepository::new(
+        conf,
+        wal_redo_manager,
+        tenantid,
+    ));

    // Load data into pageserver
    // TODO To implement zenith import we need to
@@ -214,7 +201,7 @@ pub(crate) fn get_tenants(conf: &PageServerConf) -> Result<Vec<String>> {
 }

 pub(crate) fn get_branches(conf: &PageServerConf, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
-    let repo = page_cache::get_repository_for_tenant(tenantid)?;
+    let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;

    // Each branch has a corresponding record (text file) in the refs/branches
    // with timeline_id.
@@ -270,7 +257,7 @@ pub(crate) fn create_branch(
    startpoint_str: &str,
    tenantid: &ZTenantId,
 ) -> Result<BranchInfo> {
-    let repo = page_cache::get_repository_for_tenant(tenantid)?;
+    let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;

    if conf.branch_path(branchname, tenantid).exists() {
        anyhow::bail!("branch {} already exists", branchname);
@@ -283,7 +270,7 @@ pub(crate) fn create_branch(
        let end_of_wal = repo
            .get_timeline(startpoint.timelineid)?
            .get_last_record_lsn();
-        println!("branching at end of WAL: {}", end_of_wal);
+        info!("branching at end of WAL: {}", end_of_wal);
        startpoint.lsn = end_of_wal;
    }

--- a/pageserver/src/http/mod.rs
+++ b/pageserver/src/http/mod.rs
@@ -0,0 +1,3 @@
+pub mod models;
+pub mod routes;
+pub use routes::make_router;
--- a/pageserver/src/http/models.rs
+++ b/pageserver/src/http/models.rs
@@ -0,0 +1,17 @@
+use serde::{Deserialize, Serialize};
+
+use crate::ZTenantId;
+
+#[derive(Serialize, Deserialize)]
+pub struct BranchCreateRequest {
+    #[serde(with = "hex")]
+    pub tenant_id: ZTenantId,
+    pub name: String,
+    pub start_point: String,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct TenantCreateRequest {
+    #[serde(with = "hex")]
+    pub tenant_id: ZTenantId,
+}
--- a/pageserver/src/http/openapi_spec.yml
+++ b/pageserver/src/http/openapi_spec.yml
@@ -0,0 +1,239 @@
+openapi: "3.0.2"
+info:
+  title: Page Server API
+  version: "1.0"
+servers:
+  - url: ""
+paths:
+  /v1/status:
+    description: Healthcheck endpoint
+    get:
+      description: Healthcheck
+      security: []
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+  /v1/branch/{tenant_id}:
+    parameters:
+      - name: tenant_id
+        in: path
+        required: true
+        schema:
+          type: string
+          format: hex
+    get:
+      description: Get branches for tenant
+      responses:
+        "200":
+          description: BranchInfo
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/BranchInfo"
+        "400":
+          description: Error when no tenant id found in path
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+  /v1/branch/:
+    post:
+      description: Create branch
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - "tenant_id"
+                - "name"
+                - "start_point"
+              properties:
+                tenant_id:
+                  type: string
+                  format: hex
+                name:
+                  type: string
+                start_point:
+                  type: string
+      responses:
+        "201":
+          description: BranchInfo
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/BranchInfo"
+        "400":
+          description: Malformed branch create request
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+  /v1/tenant/:
+    get:
+      description: Get tenants list
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+    post:
+      description: Create tenant
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - "tenant_id"
+              properties:
+                tenant_id:
+                  type: string
+                  format: hex
+      responses:
+        "201":
+          description: CREATED
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+        "400":
+          description: Malformed tenant create request
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+
+components:
+  securitySchemes:
+    JWT:
+      type: http
+      scheme: bearer
+      bearerFormat: JWT
+  schemas:
+    BranchInfo:
+      type: object
+      required:
+        - name
+        - timeline_id
+      properties:
+        name:
+          type: string
+        timeline_id:
+          type: string
+          format: hex
+        ancestor_id:
+          type: string
+        ancestor_lsn:
+          type: string
+    Error:
+      type: object
+      required:
+        - msg
+      properties:
+        msg:
+          type: string
+    UnauthorizedError:
+      type: object
+      required:
+        - msg
+      properties:
+        msg:
+          type: string
+    ForbiddenError:
+      type: object
+      required:
+        - msg
+      properties:
+        msg:
+          type: string
+
+security:
+  - JWT: []
--- a/pageserver/src/http/routes.rs
+++ b/pageserver/src/http/routes.rs
@@ -0,0 +1,166 @@
+use std::sync::Arc;
+
+use anyhow::Result;
+use hyper::header;
+use hyper::StatusCode;
+use hyper::{Body, Request, Response, Uri};
+use routerify::{ext::RequestExt, RouterBuilder};
+use zenith_utils::auth::JwtAuth;
+use zenith_utils::http::endpoint::attach_openapi_ui;
+use zenith_utils::http::endpoint::auth_middleware;
+use zenith_utils::http::endpoint::check_permission;
+use zenith_utils::http::error::ApiError;
+use zenith_utils::http::{
+    endpoint,
+    error::HttpErrorBody,
+    json::{json_request, json_response},
+};
+
+use super::models::BranchCreateRequest;
+use super::models::TenantCreateRequest;
+use crate::{
+    branches::{self},
+    tenant_mgr, PageServerConf, ZTenantId,
+};
+
+#[derive(Debug)]
+struct State {
+    conf: &'static PageServerConf,
+    auth: Option<Arc<JwtAuth>>,
+    allowlist_routes: Vec<Uri>,
+}
+
+impl State {
+    fn new(conf: &'static PageServerConf, auth: Option<Arc<JwtAuth>>) -> Self {
+        let allowlist_routes = ["/v1/status", "/v1/doc", "/swagger.yml"]
+            .iter()
+            .map(|v| v.parse().unwrap())
+            .collect::<Vec<_>>();
+        Self {
+            conf,
+            auth,
+            allowlist_routes,
+        }
+    }
+}
+
+#[inline(always)]
+fn get_state(request: &Request<Body>) -> &State {
+    request
+        .data::<Arc<State>>()
+        .expect("unknown state type")
+        .as_ref()
+}
+
+#[inline(always)]
+fn get_config(request: &Request<Body>) -> &'static PageServerConf {
+    get_state(request).conf
+}
+
+// healthcheck handler
+async fn status_handler(_: Request<Body>) -> Result<Response<Body>, ApiError> {
+    Ok(Response::builder()
+        .status(StatusCode::OK)
+        .header(header::CONTENT_TYPE, "application/json")
+        .body(Body::from("{}"))
+        .map_err(ApiError::from_err)?)
+}
+
+async fn branch_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let request_data: BranchCreateRequest = json_request(&mut request).await?;
+
+    check_permission(&request, Some(request_data.tenant_id))?;
+
+    let response_data = tokio::task::spawn_blocking(move || {
+        branches::create_branch(
+            get_config(&request),
+            &request_data.name,
+            &request_data.start_point,
+            &request_data.tenant_id,
+        )
+    })
+    .await
+    .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::CREATED, response_data)?)
+}
+
+async fn branch_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let tenantid: ZTenantId = match request.param("tenant_id") {
+        Some(arg) => arg
+            .parse()
+            .map_err(|_| ApiError::BadRequest("failed to parse tenant id".to_string()))?,
+        None => {
+            return Err(ApiError::BadRequest(
+                "no tenant id specified in path param".to_string(),
+            ))
+        }
+    };
+
+    check_permission(&request, Some(tenantid))?;
+
+    let response_data = tokio::task::spawn_blocking(move || {
+        crate::branches::get_branches(get_config(&request), &tenantid)
+    })
+    .await
+    .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::OK, response_data)?)
+}
+
+async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    // check for management permission
+    check_permission(&request, None)?;
+
+    let response_data =
+        tokio::task::spawn_blocking(move || crate::branches::get_tenants(get_config(&request)))
+            .await
+            .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::OK, response_data)?)
+}
+
+async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    // check for management permission
+    check_permission(&request, None)?;
+
+    let request_data: TenantCreateRequest = json_request(&mut request).await?;
+
+    let response_data = tokio::task::spawn_blocking(move || {
+        tenant_mgr::create_repository_for_tenant(get_config(&request), request_data.tenant_id)
+    })
+    .await
+    .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::CREATED, response_data)?)
+}
+
+async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
+    json_response(
+        StatusCode::NOT_FOUND,
+        HttpErrorBody::from_msg("page not found".to_owned()),
+    )
+}
+
+pub fn make_router(
+    conf: &'static PageServerConf,
+    auth: Option<Arc<JwtAuth>>,
+) -> RouterBuilder<hyper::Body, ApiError> {
+    let spec = include_bytes!("openapi_spec.yml");
+    let mut router = attach_openapi_ui(endpoint::make_router(), spec, "/swagger.yml", "/v1/doc");
+    if auth.is_some() {
+        router = router.middleware(auth_middleware(|request| {
+            let state = get_state(request);
+            if state.allowlist_routes.contains(request.uri()) {
+                None
+            } else {
+                state.auth.as_deref()
+            }
+        }))
+    }
+
+    router
+        .data(Arc::new(State::new(conf, auth)))
+        .get("/v1/status", status_handler)
+        .get("/v1/branch/:tenant_id", branch_list_handler)
+        .post("/v1/branch", branch_create_handler)
+        .get("/v1/tenant", tenant_list_handler)
+        .post("/v1/tenant", tenant_create_handler)
+        .any(handler_404)
+}
--- a/pageserver/src/layered_repository.rs
+++ b/pageserver/src/layered_repository.rs
@@ -1,13 +1,12 @@
 //!
-//! Zenith repository implementation that keeps old data in "snapshot files", and
-//! the recent changes in memory. See layered_repository/snapshot_layer.rs and
-//! layered_repository/inmemory_layer.rs, respectively. The functions here are
-//! responsible for locating the correct layer for the get/put call, tracing
-//! timeline branching history as needed.
+//! Zenith repository implementation that keeps old data in files on disk, and
+//! the recent changes in memory. See layered_repository/*_layer.rs files.
+//! The functions here are responsible for locating the correct layer for the
+//! get/put call, tracing timeline branching history as needed.
 //!
-//! The snapshot files are stored in the .zenith/tenants/<tenantid>/timelines/<timelineid>
+//! The files are stored in the .zenith/tenants/<tenantid>/timelines/<timelineid>
 //! directory. See layered_repository/README for how the files are managed.
-//! In addition to the snapshot files, there is a metadata file in the same
+//! In addition to the layer files, there is a metadata file in the same
 //! directory that contains information about the timeline, in particular its
 //! parent timeline, and the last LSN that has been written to disk.
 //!
@@ -24,30 +23,34 @@ use std::fs;
 use std::fs::File;
 use std::io::Write;
 use std::ops::Bound::Included;
-use std::sync::{Arc, Mutex};
+use std::sync::{Arc, Mutex, RwLock};
 use std::time::{Duration, Instant};

 use crate::relish::*;
-use crate::repository::{GcResult, History, Repository, Timeline, WALRecord};
+use crate::repository::{GcResult, Repository, Timeline, WALRecord};
 use crate::restore_local_repo::import_timeline_wal;
 use crate::walredo::WalRedoManager;
 use crate::PageServerConf;
 use crate::{ZTenantId, ZTimelineId};

+use zenith_metrics::{register_histogram, Histogram};
 use zenith_metrics::{register_histogram_vec, HistogramVec};
 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::{AtomicLsn, Lsn};
 use zenith_utils::seqwait::SeqWait;

+mod blob;
+mod delta_layer;
+mod filename;
+mod image_layer;
 mod inmemory_layer;
 mod layer_map;
-mod page_history;
-mod snapshot_layer;
 mod storage_layer;

+use delta_layer::DeltaLayer;
+use image_layer::ImageLayer;
 use inmemory_layer::InMemoryLayer;
 use layer_map::LayerMap;
-use snapshot_layer::SnapshotLayer;
 use storage_layer::{Layer, PageReconstructData, SegmentTag, RELISH_SEG_SIZE};

 static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
@@ -55,14 +58,14 @@ static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
 // Timeout when waiting for WAL receiver to catch up to an LSN given in a GetPage@LSN call.
 static TIMEOUT: Duration = Duration::from_secs(60);

-// Perform a checkpoint in the GC thread, when the LSN has advanced this much since
-// last checkpoint. This puts a backstop on how much WAL needs to be re-digested if
-// the page server is restarted.
+// Flush out an inmemory layer, if it's holding WAL older than this.
+// This puts a backstop on how much WAL needs to be re-digested if the
+// page server crashes.
 //
 // FIXME: This current value is very low. I would imagine something like 1 GB or 10 GB
 // would be more appropriate. But a low value forces the code to be exercised more,
 // which is good for now to trigger bugs.
-static CHECKPOINT_INTERVAL: u64 = 16 * 1024 * 1024;
+static OLDEST_INMEM_DISTANCE: u64 = 16 * 1024 * 1024;

 // Metrics collected on operations on the storage repository.
 lazy_static! {
@@ -74,6 +77,15 @@ lazy_static! {
    .expect("failed to define a metric");
 }

+// Metrics collected on operations on the storage repository.
+lazy_static! {
+    static ref RECONSTRUCT_TIME: Histogram = register_histogram!(
+        "pageserver_getpage_reconstruct_time",
+        "FIXME Time spent on storage operations"
+    )
+    .expect("failed to define a metric");
+}
+
 ///
 /// Repository consists of multiple timelines. Keep them in a hash table.
 ///
@@ -104,9 +116,8 @@ impl Repository for LayeredRepository {
        std::fs::create_dir_all(self.conf.timeline_path(&timelineid, &self.tenantid))?;

        let metadata = TimelineMetadata {
-            last_valid_lsn: start_lsn,
-            last_record_lsn: start_lsn,
-            prev_record_lsn: Lsn(0),
+            disk_consistent_lsn: start_lsn,
+            prev_record_lsn: None,
            ancestor_timeline: None,
            ancestor_lsn: start_lsn,
        };
@@ -135,9 +146,8 @@ impl Repository for LayeredRepository {
        // There is initially no data in it, but all the read-calls know to look
        // into the ancestor.
        let metadata = TimelineMetadata {
-            last_valid_lsn: start_lsn,
-            last_record_lsn: start_lsn,
-            prev_record_lsn: src_timeline.get_prev_record_lsn(),
+            disk_consistent_lsn: start_lsn,
+            prev_record_lsn: Some(src_timeline.get_prev_record_lsn()), // FIXME not atomic with start_lsn
            ancestor_timeline: Some(src),
            ancestor_lsn: start_lsn,
        };
@@ -200,7 +210,7 @@ impl LayeredRepository {
                    self.walredo_mgr.clone(),
                )?;

-                // List the snapshot layers on disk, and load them into the layer map
+                // List the layers on disk, and load them into the layer map
                timeline.load_layer_map()?;

                // Load any new WAL after the last checkpoint into memory.
@@ -262,11 +272,9 @@ impl LayeredRepository {
            {
                let timelines = self.timelines.lock().unwrap();
                for (_timelineid, timeline) in timelines.iter() {
-                    let distance = u64::from(timeline.last_valid_lsn.load())
-                        - u64::from(timeline.last_checkpoint_lsn.load());
-                    if distance > CHECKPOINT_INTERVAL {
-                        timeline.checkpoint()?;
-                    }
+                    STORAGE_TIME
+                        .with_label_values(&["checkpoint_timed"])
+                        .observe_closure_duration(|| timeline.checkpoint_internal(false))?
                }
                // release lock on 'timelines'
            }
@@ -322,7 +330,7 @@ impl LayeredRepository {
    // 2. Scan all timelines, and on each timeline, make note of the
    //    all the points where other timelines have been branched off.
    //    We will refrain from removing page versions at those LSNs.
-    // 3. For each timeline, scan all snapshot files on the timeline.
+    // 3. For each timeline, scan all layer files on the timeline.
    //    Remove all files for which a newer file exists and which
    //    don't cover any branch point LSNs.
    //
@@ -334,7 +342,7 @@ impl LayeredRepository {
        &self,
        target_timelineid: Option<ZTimelineId>,
        horizon: u64,
-        _compact: bool,
+        compact: bool,
    ) -> Result<GcResult> {
        let mut totals: GcResult = Default::default();
        let now = Instant::now();
@@ -398,6 +406,13 @@ impl LayeredRepository {
            let last_lsn = timeline.get_last_valid_lsn();

            if let Some(cutoff) = last_lsn.checked_sub(horizon) {
+                // If GC was explicitly requested by the admin, force flush all in-memory
+                // layers to disk first, so that they too can be garbage collected. That's
+                // used in tests, so we want as deterministic results as possible.
+                if compact {
+                    timeline.checkpoint()?;
+                }
+
                let result = timeline.gc_timeline(branchpoints, cutoff)?;

                totals += result;
@@ -410,15 +425,43 @@ impl LayeredRepository {
 }

 /// Metadata stored on disk for each timeline
+///
+/// The fields correspond to the values we hold in memory, in LayeredTimeline.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct TimelineMetadata {
-    last_valid_lsn: Lsn,
-    last_record_lsn: Lsn,
-    prev_record_lsn: Lsn,
+    disk_consistent_lsn: Lsn,
+
+    // This is only set if we know it. We track it in memory when the page
+    // server is running, but we only track the value corresponding to
+    // 'last_record_lsn', not 'disk_consistent_lsn' which can lag behind by a
+    // lot. We only store it in the metadata file when we flush *all* the
+    // in-memory data so that 'last_record_lsn' is the same as
+    // 'disk_consistent_lsn'.  That's OK, because after page server restart, as
+    // soon as we reprocess at least one record, we will have a valid
+    // 'prev_record_lsn' value in memory again. This is only really needed when
+    // doing a clean shutdown, so that there is no more WAL beyond
+    // 'disk_consistent_lsn'
+    prev_record_lsn: Option<Lsn>,
+
    ancestor_timeline: Option<ZTimelineId>,
    ancestor_lsn: Lsn,
 }

+#[derive(Debug, Clone)]
+struct RecordLsn {
+    last: Lsn,
+    prev: Lsn,
+}
+
+impl RecordLsn {
+    fn advance(&mut self, lsn: Lsn) {
+        assert!(self.last <= lsn);
+        let new_prev = self.last;
+        self.last = lsn;
+        self.prev = new_prev;
+    }
+}
+
 pub struct LayeredTimeline {
    conf: &'static PageServerConf,

@@ -437,10 +480,10 @@ pub struct LayeredTimeline {
    // versions have already been garbage collected away, we should
    // throw an error, but we don't track that currently.
    //
-    // last_record_lsn points to the end of last processed WAL record.
+    // record_lsn.last points to the end of last processed WAL record.
    // It can lag behind last_valid_lsn, if the WAL receiver has
-    // received some WAL after the end of last record, but not the
-    // whole next record yet. In the page cache, we care about
+    // received some WAL after the end of last record, but not the whole
+    // next record yet. For get_page_at_lsn requests, we care about
    // last_valid_lsn, but if the WAL receiver needs to restart the
    // streaming, it needs to restart at the end of last record, so we
    // track them separately. last_record_lsn should perhaps be in
@@ -448,16 +491,25 @@ pub struct LayeredTimeline {
    // all three values together.
    //
    // We also remember the starting point of the previous record in
-    // 'prev_record_lsn'. It's used to set the xl_prev pointer of the
+    // 'record_lsn.prev'. It's used to set the xl_prev pointer of the
    // first WAL record when the node is started up. But here, we just
-    // keep track of it. FIXME: last_record_lsn and prev_record_lsn
-    // should be updated atomically together.
+    // keep track of it.
    //
+    // When advancing last_valid_lsn and record_lsn simultaneously, we MUST
+    // advance last_valid_lsn before record_lsn.
+    // This is so if a reader wishes, it can read record_lsn and then
+    // last_valid_lsn, and find that last_valid_lsn >= record_lsn.
    last_valid_lsn: SeqWait<Lsn>,
-    last_record_lsn: AtomicLsn,
-    prev_record_lsn: AtomicLsn,
+    record_lsn: RwLock<RecordLsn>,

-    last_checkpoint_lsn: AtomicLsn,
+    // All WAL records have been processed and stored durably on files on
+    // local disk, up to this LSN. On crash and restart, we need to re-process
+    // the WAL starting from this point.
+    //
+    // Some later WAL records might have been processed and also flushed to disk
+    // already, so don't be surprised to see some, but there's no guarantee on
+    // them yet.
+    disk_consistent_lsn: AtomicLsn,

    // Parent timeline that this timeline was branched from, and the LSN
    // of the branch point.
@@ -486,7 +538,8 @@ impl Timeline for LayeredTimeline {
        let seg = SegmentTag::from_blknum(rel, blknum);

        if let Some((layer, lsn)) = self.get_layer_for_read(seg, lsn)? {
-            self.materialize_page(seg, blknum, lsn, &*layer)
+            RECONSTRUCT_TIME
+                .observe_closure_duration(|| self.materialize_page(seg, blknum, lsn, &*layer))
        } else {
            bail!("relish {} not found at {}", rel, lsn);
        }
@@ -554,7 +607,11 @@ impl Timeline for LayeredTimeline {
        let mut all_rels = HashSet::new();
        let mut timeline = self;
        loop {
-            let rels = timeline.layers.lock().unwrap().list_rels(spcnode, dbnode)?;
+            let rels = timeline
+                .layers
+                .lock()
+                .unwrap()
+                .list_rels(spcnode, dbnode, lsn)?;

            all_rels.extend(rels.iter());

@@ -566,26 +623,6 @@ impl Timeline for LayeredTimeline {
            }
        }

-        // Now we have a list of all rels that appeared anywhere in the history. Filter
-        // out relations that were dropped.
-        //
-        // FIXME: We should pass the LSN argument to the calls above, and avoid scanning
-        // dropped relations in the first place.
-        let mut res: Result<()> = Ok(());
-        all_rels.retain(
-            |reltag| match self.get_rel_exists(RelishTag::Relation(*reltag), lsn) {
-                Ok(exists) => {
-                    info!("retain: {} -> {}", *reltag, exists);
-                    exists
-                }
-                Err(err) => {
-                    res = Err(err);
-                    false
-                }
-            },
-        );
-        res?;
-
        Ok(all_rels)
    }

@@ -608,32 +645,9 @@ impl Timeline for LayeredTimeline {
            }
        }

-        // Now we have a list of all nonrels that appeared anywhere in the history. Filter
-        // out dropped ones.
-        //
-        // FIXME: We should pass the LSN argument to the calls above, and avoid scanning
-        // dropped relations in the first place.
-        let mut res: Result<()> = Ok(());
-        all_rels.retain(|tag| match self.get_rel_exists(*tag, lsn) {
-            Ok(exists) => {
-                info!("retain: {} -> {}", *tag, exists);
-                exists
-            }
-            Err(err) => {
-                res = Err(err);
-                false
-            }
-        });
-        res?;
-
        Ok(all_rels)
    }

-    fn history<'a>(&'a self) -> Result<Box<dyn History + 'a>> {
-        // This is needed by the push/pull functionality. Not implemented yet.
-        todo!();
-    }
-
    fn put_wal_record(&self, rel: RelishTag, blknum: u32, rec: WALRecord) -> Result<()> {
        if !rel.is_blocky() && blknum != 0 {
            bail!(
@@ -736,14 +750,7 @@ impl Timeline for LayeredTimeline {
        Ok(())
    }

-    fn put_page_image(
-        &self,
-        rel: RelishTag,
-        blknum: u32,
-        lsn: Lsn,
-        img: Bytes,
-        _update_meta: bool,
-    ) -> Result<()> {
+    fn put_page_image(&self, rel: RelishTag, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()> {
        if !rel.is_blocky() && blknum != 0 {
            bail!(
                "invalid request for block {} for non-blocky relish {}",
@@ -758,28 +765,16 @@ impl Timeline for LayeredTimeline {
        layer.put_page_image(blknum, lsn, img)
    }

-    fn put_raw_data(
-        &self,
-        _tag: crate::object_key::ObjectTag,
-        _lsn: Lsn,
-        _data: &[u8],
-    ) -> Result<()> {
-        // FIXME: This doesn't make much sense for the layered storage format,
-        // it's pretty tightly coupled with the way the object store stores
-        // things.
-        bail!("put_raw_data not implemented");
-    }
-
    /// Public entry point for checkpoint(). All the logic is in the private
    /// checkpoint_internal function, this public facade just wraps it for
    /// metrics collection.
    fn checkpoint(&self) -> Result<()> {
        STORAGE_TIME
-            .with_label_values(&["checkpoint"])
-            .observe_closure_duration(|| self.checkpoint_internal())
+            .with_label_values(&["checkpoint_force"])
+            .observe_closure_duration(|| self.checkpoint_internal(true))
    }

-    /// Remember that WAL has been received and added to the page cache up to the given LSN
+    /// Remember that WAL has been received and added to the timeline up to the given LSN
    fn advance_last_valid_lsn(&self, lsn: Lsn) {
        let old = self.last_valid_lsn.advance(lsn);

@@ -790,7 +785,7 @@ impl Timeline for LayeredTimeline {
        if lsn < old {
            // Should never be called with an LSN older than the last
            // record LSN, though.
-            let last_record_lsn = self.last_record_lsn.load();
+            let last_record_lsn = self.get_last_record_lsn();
            if lsn < last_record_lsn {
                warn!(
                    "attempted to move last valid LSN backwards beyond last record LSN (last record {}, new {})",
@@ -801,11 +796,18 @@ impl Timeline for LayeredTimeline {
    }

    fn init_valid_lsn(&self, lsn: Lsn) {
+        // These writes must be specified in the order mentioned on the field comments.
        let old = self.last_valid_lsn.advance(lsn);
        assert!(old == Lsn(0));
-        let old = self.last_record_lsn.fetch_max(lsn);
-        assert!(old == Lsn(0));
-        self.prev_record_lsn.store(Lsn(0));
+
+        {
+            let mut record_lsn = self.record_lsn.write().unwrap();
+
+            assert_eq!(record_lsn.last, Lsn(0));
+            assert_eq!(record_lsn.prev, Lsn(0));
+
+            record_lsn.advance(lsn);
+        }
    }

    fn get_last_valid_lsn(&self) -> Lsn {
@@ -813,35 +815,35 @@ impl Timeline for LayeredTimeline {
    }

    ///
-    /// Remember the (end of) last valid WAL record remembered in the page cache.
+    /// Remember the (end of) last valid WAL record remembered in the timeline.
    ///
    /// NOTE: this updates last_valid_lsn as well.
    ///
    fn advance_last_record_lsn(&self, lsn: Lsn) {
-        // Can't move backwards.
-        let old = self.last_record_lsn.fetch_max(lsn);
-        assert!(old <= lsn);
+        // These writes must be specified in the order mentioned on the field comments.
+        let old_valid_lsn = self.last_valid_lsn.advance(lsn);

-        // Use old value of last_record_lsn as prev_record_lsn
-        self.prev_record_lsn.fetch_max(old);
-
-        // Also advance last_valid_lsn
-        let old = self.last_valid_lsn.advance(lsn);
        // Can't move backwards.
-        if lsn < old {
+        if lsn < old_valid_lsn {
            warn!(
                "attempted to move last record LSN backwards (was {}, new {})",
-                old, lsn
+                old_valid_lsn, lsn
            );
        }
+
+        {
+            let mut record_lsn = self.record_lsn.write().unwrap();
+            assert!(record_lsn.last <= lsn);
+            record_lsn.advance(lsn);
+        }
    }

    fn get_last_record_lsn(&self) -> Lsn {
-        self.last_record_lsn.load()
+        self.record_lsn.read().unwrap().last
    }

    fn get_prev_record_lsn(&self) -> Lsn {
-        self.prev_record_lsn.load()
+        self.record_lsn.read().unwrap().prev
    }
 }

@@ -865,10 +867,14 @@ impl LayeredTimeline {

            walredo_mgr,

-            last_valid_lsn: SeqWait::new(metadata.last_valid_lsn),
-            last_record_lsn: AtomicLsn::new(metadata.last_record_lsn.0),
-            prev_record_lsn: AtomicLsn::new(metadata.prev_record_lsn.0),
-            last_checkpoint_lsn: AtomicLsn::new(metadata.last_valid_lsn.0),
+            // initialize in-memory 'last_valid_lsn' and 'last_record_lsn' from
+            // 'disk_consistent_lsn'.
+            last_valid_lsn: SeqWait::new(metadata.disk_consistent_lsn),
+            record_lsn: RwLock::new(RecordLsn {
+                last: metadata.disk_consistent_lsn,
+                prev: metadata.prev_record_lsn.unwrap_or(Lsn(0)),
+            }),
+            disk_consistent_lsn: AtomicLsn::new(metadata.disk_consistent_lsn.0),

            ancestor_timeline: ancestor,
            ancestor_lsn: metadata.ancestor_lsn,
@@ -877,7 +883,7 @@ impl LayeredTimeline {
    }

    ///
-    /// Load the list of snapshot files from disk, populating the layer map
+    /// Scan the timeline directory to populate the layer map
    ///
    fn load_layer_map(&self) -> anyhow::Result<()> {
        info!(
@@ -885,19 +891,51 @@ impl LayeredTimeline {
            self.timelineid
        );
        let mut layers = self.layers.lock().unwrap();
-        let snapfiles =
-            SnapshotLayer::list_snapshot_files(self.conf, self.timelineid, self.tenantid)?;
+        let (imgfilenames, mut deltafilenames) =
+            filename::list_files(self.conf, self.timelineid, self.tenantid)?;
+
+        // First create ImageLayer structs for each image file.
+        for filename in imgfilenames.iter() {
+            let layer = ImageLayer::new(self.conf, self.timelineid, self.tenantid, filename);

-        for layer_rc in snapfiles.iter() {
            info!(
-                "found layer {} {}-{} {} on timeline {}",
-                layer_rc.get_seg_tag(),
-                layer_rc.get_start_lsn(),
-                layer_rc.get_end_lsn(),
-                layer_rc.is_dropped(),
+                "found layer {} {} on timeline {}",
+                layer.get_seg_tag(),
+                layer.get_start_lsn(),
                self.timelineid
            );
-            layers.insert_historic(Arc::clone(layer_rc));
+            layers.insert_historic(Arc::new(layer));
+        }
+
+        // Then for the Delta files. The delta files are created in order starting
+        // from the oldest file, because each DeltaLayer needs a reference to its
+        // predecessor.
+        deltafilenames.sort();
+
+        for filename in deltafilenames.iter() {
+            let predecessor = layers.get(&filename.seg, filename.start_lsn);
+
+            let predecessor_str: String = if let Some(prec) = &predecessor {
+                prec.filename().display().to_string()
+            } else {
+                "none".to_string()
+            };
+
+            let layer = DeltaLayer::new(
+                self.conf,
+                self.timelineid,
+                self.tenantid,
+                filename,
+                predecessor,
+            );
+
+            info!(
+                "found layer {} on timeline {}, predecessor: {}",
+                layer.filename().display(),
+                self.timelineid,
+                predecessor_str,
+            );
+            layers.insert_historic(Arc::new(layer));
        }

        Ok(())
@@ -1004,23 +1042,23 @@ impl LayeredTimeline {
        let layer;
        if let Some((prev_layer, _prev_lsn)) = self.get_layer_for_read(seg, lsn)? {
            // Create new entry after the previous one.
-            let lsn;
+            let start_lsn;
            if prev_layer.get_timeline_id() != self.timelineid {
                // First modification on this timeline
-                lsn = self.ancestor_lsn;
+                start_lsn = self.ancestor_lsn;
                trace!(
                    "creating file for write for {} at branch point {}/{}",
                    seg,
                    self.timelineid,
-                    lsn
+                    start_lsn
                );
            } else {
-                lsn = prev_layer.get_end_lsn();
+                start_lsn = prev_layer.get_end_lsn();
                trace!(
                    "creating file for write for {} after previous layer {}/{}",
                    seg,
                    self.timelineid,
-                    lsn
+                    start_lsn
                );
            }
            trace!(
@@ -1029,12 +1067,12 @@ impl LayeredTimeline {
                prev_layer.get_start_lsn(),
                prev_layer.get_end_lsn()
            );
-            layer = InMemoryLayer::copy_snapshot(
+            layer = InMemoryLayer::create_successor_layer(
                self.conf,
-                &self,
-                &*prev_layer,
+                prev_layer,
                self.timelineid,
                self.tenantid,
+                start_lsn,
                lsn,
            )?;
        } else {
@@ -1046,7 +1084,8 @@ impl LayeredTimeline {
                lsn
            );

-            layer = InMemoryLayer::create(self.conf, self.timelineid, self.tenantid, seg, lsn)?;
+            layer =
+                InMemoryLayer::create(self.conf, self.timelineid, self.tenantid, seg, lsn, lsn)?;
        }

        let mut layers = self.layers.lock().unwrap();
@@ -1089,10 +1128,17 @@ impl LayeredTimeline {
    ///
    /// NOTE: This has nothing to do with checkpoint in PostgreSQL. We don't
    /// know anything about them here in the repository.
-    fn checkpoint_internal(&self) -> Result<()> {
+    fn checkpoint_internal(&self, force: bool) -> Result<()> {
+        // To hold the invariant last_valid_lsn >= last_record_lsn,
+        // we must follow the read order specified in the field comments.
+
+        let RecordLsn {
+            last: last_record_lsn,
+            prev: prev_record_lsn,
+        } = self.record_lsn.read().unwrap().clone();
+
        let last_valid_lsn = self.last_valid_lsn.load();
-        let last_record_lsn = self.last_record_lsn.load();
-        let prev_record_lsn = self.prev_record_lsn.load();
+
        trace!(
            "checkpointing timeline {} at {}",
            self.timelineid,
@@ -1106,79 +1152,88 @@ impl LayeredTimeline {
        // requests will block.
        let mut layers = self.layers.lock().unwrap();

-        // Walk through each in-memory, and write any dirty data to disk,
-        // as snapshot files.
-        //
-        // We currently write a new snapshot file for every relation
-        // that was modified, if there has been any changes at all.
-        // It would be smarter to only flush out in-memory layers that
-        // have accumulated a fair amount of changes. Note that the
-        // start and end LSNs of snapshot files belonging to different
-        // relations don't have to line up, although currently they do
-        // because of the way this works. So you could have a snapshot
-        // file covering LSN range 100-200 for one relation, and a
-        // snapshot file covering 150-250 for another relation. The
-        // read functions should even cope with snapshot files
-        // covering overlapping ranges for the same relation, although
-        // that situation never arises currently.
-        //
-        // Note: We aggressively freeze and unload all the layer
-        // structs. Even if a layer is actively being used. This
-        // keeps memory usage in check, but is probably too
-        // aggressive. Some kind of LRU policy would be appropriate.
+        // Take the in-memory layer with the oldest WAL record. If it's older
+        // than the threshold, write it out to disk as a new image and delta file.
+        // Repeat until all remaining in-memory layers are within the threshold.
        //
+        // That's necessary to limit the amount of WAL that needs to be kept
+        // in the safekeepers, and that needs to be reprocessed on page server
+        // crash. TODO: It's not a great policy for keeping memory usage in
+        // check, though. We should also aim at flushing layers that consume
+        // a lot of memory and/or aren't receiving much updates anymore.
+        let mut disk_consistent_lsn = last_record_lsn;
+        while let Some(oldest_layer) = layers.peek_oldest_open() {
+            // Does this layer need freezing?
+            let oldest_pending_lsn = oldest_layer.get_oldest_pending_lsn();
+            let distance = last_valid_lsn.0 - oldest_pending_lsn.0;
+            if !force && distance < OLDEST_INMEM_DISTANCE {
+                info!(
+                    "the oldest layer is now {} which is {} bytes behind last_valid_lsn",
+                    oldest_layer.get_seg_tag(),
+                    distance
+                );
+                disk_consistent_lsn = oldest_pending_lsn;
+                break;
+            }

-        // Call freeze() on any unfrozen layers (that is, layers that haven't
-        // been written to disk yet).
-        // Call unload() on all frozen layers, to release memory.
-        let mut iter = layers.iter_open_layers();
-        while let Some(layer) = iter.next() {
-            let (new_historic, new_open) = layer.freeze(last_valid_lsn, &self)?;
+            // freeze it
+            let (new_historics, new_open) = oldest_layer.freeze(last_valid_lsn, &self)?;

            // replace this layer with the new layers that 'freeze' returned
-            // (removes it if new_open is None)
-            iter.replace(new_open);
-
-            if let Some(historic) = new_historic {
-                trace!(
-                    "freeze returned layer {} {}-{}",
-                    historic.get_seg_tag(),
-                    historic.get_start_lsn(),
-                    historic.get_end_lsn()
-                );
-                iter.insert_historic(historic);
+            layers.pop_oldest_open();
+            if let Some(n) = new_open {
+                layers.insert_open(n);
+            }
+            for n in new_historics {
+                layers.insert_historic(n);
            }
        }

+        // Call unload() on all frozen layers, to release memory.
+        // TODO: On-disk layers shouldn't consume much memory to begin with,
+        // so this shouldn't be necessary. But currently the DeltaLayer and
+        // ImageLayer code slurps the whole file into memory, so they do in
+        // fact consume a lot of memory.
        for layer in layers.iter_historic_layers() {
            layer.unload()?;
        }

-        // Also save the metadata, with updated last_valid_lsn and last_record_lsn, to a
-        // file in the timeline dir. The metadata reflects the last_valid_lsn as it was
-        // when we *started* the checkpoint, so that after crash, the WAL receiver knows
-        // to restart the streaming from that WAL position.
-        let ancestor_timelineid = if let Some(x) = &self.ancestor_timeline {
-            Some(x.timelineid)
-        } else {
-            None
+        // Save the metadata, with updated 'disk_consistent_lsn', to a
+        // file in the timeline dir. After crash, we will restart WAL
+        // streaming and processing from that point.
+
+        // We can only save a valid 'prev_record_lsn' value on disk if we
+        // flushed *all* in-memory changes to disk. We only track
+        // 'prev_record_lsn' in memory for the latest processed record, so we
+        // don't remember what the correct value that corresponds to some old
+        // LSN is. But if we flush everything, then the value corresponding
+        // current 'last_record_lsn' is correct and we can store it on disk.
+        let ondisk_prev_record_lsn = {
+            if disk_consistent_lsn == last_record_lsn {
+                Some(prev_record_lsn)
+            } else {
+                None
+            }
        };
+
+        let ancestor_timelineid = self.ancestor_timeline.as_ref().map(|x| x.timelineid);
+
        let metadata = TimelineMetadata {
-            last_valid_lsn: last_valid_lsn,
-            last_record_lsn: last_record_lsn,
-            prev_record_lsn: prev_record_lsn,
+            disk_consistent_lsn: disk_consistent_lsn,
+            prev_record_lsn: ondisk_prev_record_lsn,
            ancestor_timeline: ancestor_timelineid,
            ancestor_lsn: self.ancestor_lsn,
        };
        LayeredRepository::save_metadata(self.conf, self.timelineid, self.tenantid, &metadata)?;

-        self.last_checkpoint_lsn.store(last_valid_lsn);
+        // Also update the in-memory copy
+        self.disk_consistent_lsn.store(disk_consistent_lsn);

        Ok(())
    }

    ///
-    /// Garbage collect snapshot files on a timeline that are no longer needed.
+    /// Garbage collect layer files on a timeline that are no longer needed.
    ///
    /// The caller specifies how much history is needed with the two arguments:
    ///
@@ -1196,36 +1251,34 @@ impl LayeredTimeline {
    /// to figure out what read-only nodes might actually need.)
    ///
    /// Currently, we don't make any attempt at removing unneeded page versions
-    /// within a snapshot file. We can only remove the whole file if it's fully
+    /// within a layer file. We can only remove the whole file if it's fully
    /// obsolete.
    ///
    pub fn gc_timeline(&self, retain_lsns: Vec<Lsn>, cutoff: Lsn) -> Result<GcResult> {
        let now = Instant::now();
        let mut result: GcResult = Default::default();

-        // Scan all snapshot files in the directory. For each file, if a newer file
-        // exists, we can remove the old one.
-        self.checkpoint()?;
-
-        let mut layers = self.layers.lock().unwrap();
-
        info!(
            "running GC on timeline {}, cutoff {}",
            self.timelineid, cutoff
        );

-        let mut layers_to_remove: Vec<Arc<SnapshotLayer>> = Vec::new();
+        let mut layers_to_remove: Vec<Arc<dyn Layer>> = Vec::new();

+        // Scan all layer files in the directory. For each file, if a newer file
+        // exists, we can remove the old one.
+        //
        // Determine for each file if it needs to be retained
        // FIXME: also scan open in-memory layers. Normally we cannot remove the
        // latest layer of any seg, but if it was unlinked it's possible
+        let mut layers = self.layers.lock().unwrap();
        'outer: for l in layers.iter_historic_layers() {
            let seg = l.get_seg_tag();

            if seg.rel.is_relation() {
-                result.snapshot_relfiles_total += 1;
+                result.ondisk_relfiles_total += 1;
            } else {
-                result.snapshot_nonrelfiles_total += 1;
+                result.ondisk_nonrelfiles_total += 1;
            }

            // Is it newer than cutoff point?
@@ -1238,9 +1291,9 @@ impl LayeredTimeline {
                    cutoff
                );
                if seg.rel.is_relation() {
-                    result.snapshot_relfiles_needed_by_cutoff += 1;
+                    result.ondisk_relfiles_needed_by_cutoff += 1;
                } else {
-                    result.snapshot_nonrelfiles_needed_by_cutoff += 1;
+                    result.ondisk_nonrelfiles_needed_by_cutoff += 1;
                }
                continue 'outer;
            }
@@ -1257,20 +1310,21 @@ impl LayeredTimeline {
                        *retain_lsn
                    );
                    if seg.rel.is_relation() {
-                        result.snapshot_relfiles_needed_by_branches += 1;
+                        result.ondisk_relfiles_needed_by_branches += 1;
                    } else {
-                        result.snapshot_nonrelfiles_needed_by_branches += 1;
+                        result.ondisk_nonrelfiles_needed_by_branches += 1;
                    }
                    continue 'outer;
                }
            }

-            // Unless the relation was dropped, is there a later snapshot file for this relation?
-            if !l.is_dropped() && !layers.newer_layer_exists(l.get_seg_tag(), l.get_end_lsn()) {
+            // Unless the relation was dropped, is there a later image file for this relation?
+            if !l.is_dropped() && !layers.newer_image_layer_exists(l.get_seg_tag(), l.get_end_lsn())
+            {
                if seg.rel.is_relation() {
-                    result.snapshot_relfiles_not_updated += 1;
+                    result.ondisk_relfiles_not_updated += 1;
                } else {
-                    result.snapshot_nonrelfiles_not_updated += 1;
+                    result.ondisk_nonrelfiles_not_updated += 1;
                }
                continue 'outer;
            }
@@ -1295,15 +1349,15 @@ impl LayeredTimeline {

            if doomed_layer.is_dropped() {
                if doomed_layer.get_seg_tag().rel.is_relation() {
-                    result.snapshot_relfiles_dropped += 1;
+                    result.ondisk_relfiles_dropped += 1;
                } else {
-                    result.snapshot_nonrelfiles_dropped += 1;
+                    result.ondisk_nonrelfiles_dropped += 1;
                }
            } else {
                if doomed_layer.get_seg_tag().rel.is_relation() {
-                    result.snapshot_relfiles_removed += 1;
+                    result.ondisk_relfiles_removed += 1;
                } else {
-                    result.snapshot_nonrelfiles_removed += 1;
+                    result.ondisk_nonrelfiles_removed += 1;
                }
            }
        }
--- a/pageserver/src/layered_repository/README.md
+++ b/pageserver/src/layered_repository/README.md
@@ -11,45 +11,48 @@ to one 10 MB slice of a PostgreSQL relation fork. The snapshot files
 for each timeline are stored in the timeline's subdirectory under
 .zenith/tenants/<tenantid>/timelines.

-The files are named like this:
+There are two kind of snapshot file: base images, and deltas. A base
+image file contains a snapshot of a segment as it was at one LSN,
+whereas a delta file contains modifications to a segment - mostly in
+the form of WAL records - in a range of LSN
+
+base image file:
+
+    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>
+
+delta file:

    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>_<end LSN>

 For example:

+    rel_1663_13990_2609_0_10_000000000169C348
    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000

-Some non-relation files are also stored in repository. For example,
-a CLOG segment would be named like this:
+In addition to the relations, with "rel_*" prefix, we use the same
+format for storing various smaller files from the PostgreSQL data
+directory. They will use different suffixes and the naming scheme up
+to the LSNs vary. The Zenith source code uses the term "relish" to
+mean "a relation, or other file that's treated like a relation in the
+storage" For example, a base image of a CLOG segment would be named
+like this:

-    pg_xact_0000_0_00000000198B06B0_00000000198C2550
+    pg_xact_0000_0_00000000198B06B0

 There is no difference in how the relation and non-relation files are
 managed, except that the first part of file names is different.
 Internally, the relations and non-relation files that are managed in
 the versioned store are together called "relishes".

-Each snapshot file contains a full snapshot, that is, full copy of all
-pages in the relation, as of the "start LSN". It also contains all WAL
-records applicable to the relation between the start and end
-LSNs. With this information, the page server can reconstruct any page
-version of the relation in the LSN range.
-
 If a file has been dropped, the last snapshot file for it is created
 with the _DROPPED suffix, e.g.

    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000_DROPPED

-In addition to the relations, with "rel_*" prefix, we use the same
-format for storing various smaller files from the PostgreSQL data
-directory. They will use different suffixes and the naming scheme
-up to the LSN range varies. The Zenith source code uses the term
-"relish" to mean "a relation, or other file that's treated like a
-relation in the storage"

 ## Notation used in this document

-The full path of a snapshot file looks like this:
+The full path of a delta file looks like this:

    .zenith/tenants/941ddc8604413b88b3d208bddf90396c/timelines/4af489b06af8eed9e27a841775616962/rel_1663_13990_2609_0_10_000000000169C348_0000000001702000

@@ -57,9 +60,10 @@ For simplicity, the examples below use a simplified notation for the
 paths.  The tenant ID is left out, the timeline ID is replaced with
 the human-readable branch name, and spcnode+dbnode+relnode+forkum+segno
 with a human-readable table name. The LSNs are also shorter. For
-example, a snapshot file for 'orders' table on 'main' branch, with LSN
-range 100-200 would be:
+example, a base image file at LSN 100 and a delta file between 100-200
+for 'orders' table on 'main' branch is represented like this:

+    main/orders_100
    main/orders_100_200


@@ -68,10 +72,14 @@ range 100-200 would be:
 Let's start with a simple example with a system that contains one
 branch called 'main' and two tables, 'orders' and 'customers'. The end
 of WAL is currently at LSN 250. In this starting situation, you would
-have two files on disk:
+have these files on disk:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
+	main/customers_100
 	main/customers_100_200
+	main/customers_200

 In addition to those files, the recent changes between LSN 200 and the
 end of WAL at 250 are kept in memory. If the page server crashes, the
@@ -87,20 +95,33 @@ the last checkpoint happened at LSN 400 but there hasn't been any
 recent changes to 'customers' table, you would have these files on
 disk:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
+	main/orders_300
 	main/orders_300_400
+	main/orders_400
+	main/customers_100
 	main/customers_100_200
+	main/customers_200

 If the customers table is modified later, a new file is created for it
 at the next checkpoint. The new file will cover the "gap" from the
 last snapshot file, so the LSN ranges are always contiguous:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
+	main/orders_300
 	main/orders_300_400
+	main/orders_400
+	main/customers_100
 	main/customers_100_200
+	main/customers_200
 	main/customers_200_500
+	main/customers_500

 ## Reading page versions

@@ -120,18 +141,6 @@ relation at the start LSN and the WAL, reconstructing the page
 involves replaying any WAL records applicable to the page between LSNs
 200-250, starting from the base image at LSN 200.

-A request at a file boundary can be satisfied using either file. For
-example, if there are two files on disk:
-
-	main/orders_100_200
-	main/orders_200_300
-
-And a request comes with LSN 200, either file can be used for it. It
-is better to use the later file, however, because it contains an
-already materialized version of all the pages at LSN 200. Using the
-first file, you would need to apply any WAL records between 100 and
-200 to reconstruct the requested page.
-
 # Multiple branches

 Imagine that a child branch is created at LSN 250:
@@ -145,12 +154,20 @@ Imagine that a child branch is created at LSN 250:
 Then, the 'orders' table is updated differently on the 'main' and
 'child' branches. You now have this situation on disk:

+    main/orders_100
    main/orders_100_200
+    main/orders_200
    main/orders_200_300
+    main/orders_300
    main/orders_300_400
+    main/orders_400
+    main/customers_100
    main/customers_100_200
+    main/customers_200
    child/orders_250_300
+    child/orders_300
    child/orders_300_400
+    child/orders_400

 Because the 'customers' table hasn't been modified on the child
 branch, there is no file for it there. If you request a page for it on
@@ -163,24 +180,34 @@ is linear, and the request's LSN identifies unambiguously which file
 you need to look at. For example, the history for the 'orders' table
 on the 'main' branch consists of these files:

+    main/orders_100
    main/orders_100_200
+    main/orders_200
    main/orders_200_300
+    main/orders_300
    main/orders_300_400
+    main/orders_400

 And from the 'child' branch's point of view, it consists of these
 files:

+    main/orders_100
    main/orders_100_200
+    main/orders_200
    main/orders_200_300
    child/orders_250_300
+    child/orders_300
    child/orders_300_400
+    child/orders_400

 The branch metadata includes the point where the child branch was
 created, LSN 250. If a page request comes with LSN 275, we read the
-page version from the 'child/orders_250_300' file. If the request LSN
-is 225, we read it from the 'main/orders_200_300' file instead.  The
-page versions between 250-300 in the 'main/orders_200_300' file are
-ignored when operating on the child branch.
+page version from the 'child/orders_250_300' file. We might also
+need to reconstruct the page version as it was at LSN 250, in order
+to replay the WAL up to LSN 275, using 'main/orders_200_300' and
+'main/orders_200'. The page versions between 250-300 in the
+'main/orders_200_300' file are ignored when operating on the child
+branch.

 Note: It doesn't make any difference if the child branch is created
 when the end of the main branch was at LSN 250, or later when the tip of
@@ -204,17 +231,37 @@ Let's look at the single branch scenario again. Imagine that the end
 of the branch is LSN 525, so that the GC horizon is currently at
 525-150 = 375

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
+	main/orders_300
 	main/orders_300_400
+	main/orders_400
 	main/orders_400_500
+	main/orders_500
+	main/customers_100
 	main/customers_100_200
+	main/customers_200

-We can remove files 'main/orders_100_200' and 'main/orders_200_300',
-because the end LSNs of those files are older than GC horizon 375, and
-there are more recent snapshot files for the table. 'main/orders_300_400'
-and 'main/orders_400_500' are still within the horizon, so they must be
-retained. 'main/customers_100_200' is old enough, but it cannot be
+We can remove the following files because the end LSNs of those files are
+older than GC horizon 375, and there are more recent snapshot files for the
+table:
+
+	main/orders_100       DELETE
+	main/orders_100_200   DELETE
+	main/orders_200       DELETE
+	main/orders_200_300   DELETE
+	main/orders_300       STILL NEEDED BY orders_300_400
+	main/orders_300_400   KEEP, NEWER THAN GC HORIZON
+	main/orders_400       .. 
+	main/orders_400_500   .. 
+	main/orders_500       .. 
+	main/customers_100      DELETE
+	main/customers_100_200  DELETE
+	main/customers_200      KEEP, NO NEWER VERSION
+
+'main/customers_100_200' is old enough, but it cannot be
 removed because there is no newer snapshot file for the table.

 Things get slightly more complicated with multiple branches. All of
@@ -223,41 +270,47 @@ retain older shapshot files that are still needed by child branches.
 For example, if child branch is created at LSN 150, and the 'customers'
 table is updated on the branch, you would have these files:

-	main/orders_100_200
-	main/orders_200_300
-	main/orders_300_400
-	main/orders_400_500
-	main/customers_100_200
-	child/customers_150_300
+	main/orders_100        KEEP, NEEDED BY child BRANCH
+	main/orders_100_200    KEEP, NEEDED BY child BRANCH
+	main/orders_200        DELETE
+	main/orders_200_300    DELETE
+	main/orders_300        KEEP, NEWER THAN GC HORIZON
+	main/orders_300_400    KEEP, NEWER THAN GC HORIZON
+	main/orders_400        KEEP, NEWER THAN GC HORIZON
+	main/orders_400_500    KEEP, NEWER THAN GC HORIZON
+	main/orders_500        KEEP, NEWER THAN GC HORIZON
+	main/customers_100       DELETE
+	main/customers_100_200   DELETE
+	main/customers_200       KEEP, NO NEWER VERSION
+	child/customers_150_300  DELETE
+	child/customers_300      KEEP, NO NEWER VERSION

-In this situation, the 'main/orders_100_200' file cannot be removed,
-even though it is older than the GC horizon, because it is still
-needed by the child branch.  'main/orders_200_300' can still be
-removed. So after garbage collection, these files would remain:
-
-	main/orders_100_200
-
-	main/orders_300_400
-	main/orders_400_500
-	main/customers_100_200
-	child/customers_150_300
+In this situation, 'main/orders_100' and 'main/orders_100_200' cannot
+be removed, even though they are older than the GC horizon, because
+they are still needed by the child branch. 'main/orders_200'
+and 'main/orders_200_300' can still be removed.

 If 'orders' is modified later on the 'child' branch, we will create a
-snapshot file for it on the child:
+new base image and delta file for it on the child:

+	main/orders_100
 	main/orders_100_200

+	main/orders_300
 	main/orders_300_400
+	main/orders_400
 	main/orders_400_500
-	main/customers_100_200
-	child/customers_150_300
+	main/orders_500
+	main/customers_200
+	child/customers_300
 	child/orders_150_400
+	child/orders_400

-After this, the 'main/orders_100_200' file can be removed. It is no
-longer needed by the child branch, because there is a newer snapshot
-file there. TODO: This optimization hasn't been implemented! The GC
-algorithm will currently keep the file on the 'main' branch anyway, for
-as long as the child branch exists.
+After this, the 'main/orders_100' and 'main/orders_100_200' file could
+be removed. It is no longer needed by the child branch, because there
+is a newer snapshot file there. TODO: This optimization hasn't been
+implemented! The GC algorithm will currently keep the file on the
+'main' branch anyway, for as long as the child branch exists.


 # TODO: On LSN ranges
@@ -265,21 +318,33 @@ as long as the child branch exists.
 In principle, each relation can be checkpointed separately, i.e. the
 LSN ranges of the files don't need to line up. So this would be legal:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
+	main/orders_300
 	main/orders_300_400
+	main/orders_400
+	main/customers_150
 	main/customers_150_250
+	main/customers_250
 	main/customers_250_500
+	main/customers_500

 However, the code currently always checkpoints all relations together.
 So that situation doesn't arise in practice.

 It would also be OK to have overlapping LSN ranges for the same relation:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
+	main/orders_300
 	main/orders_250_350
+	main/orders_350
 	main/orders_300_400
+	main/orders_400

 The code that reads the snapshot files should cope with this, but this
 situation doesn't arise either, because the checkpointing code never
@@ -287,12 +352,14 @@ does that.  It could be useful, however, as a transient state when
 garbage collecting around branch points, or explicit recovery
 points. For example, if we start with this:

+	main/orders_100
 	main/orders_100_200
+	main/orders_200
 	main/orders_200_300
-	main/orders_300_400
+	main/orders_300

 And there is a branch or explicit recovery point at LSN 150, we could
-replace 'main/orders_100_200' with 'main/orders_150_150' to keep a
+replace 'main/orders_100_200' with 'main/orders_150' to keep a
 snapshot only at that exact point that's still needed, removing the
 other page versions around it. But such compaction has not been
 implemented yet.
--- a/pageserver/src/layered_repository/blob.rs
+++ b/pageserver/src/layered_repository/blob.rs
@@ -0,0 +1,45 @@
+use std::{fs::File, io::Write};
+
+use anyhow::Result;
+use bookfile::{BookWriter, BoundedReader, ChapterId, ChapterWriter};
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+pub struct BlobRange {
+    offset: u64,
+    size: usize,
+}
+
+pub fn read_blob(reader: &BoundedReader<&'_ File>, range: &BlobRange) -> Result<Vec<u8>> {
+    let mut buf = vec![0u8; range.size];
+    reader.read_exact_at(&mut buf, range.offset)?;
+    Ok(buf)
+}
+
+pub struct BlobWriter {
+    writer: ChapterWriter<File>,
+    offset: u64,
+}
+
+impl BlobWriter {
+    // This function takes a BookWriter and creates a new chapter to ensure offset is 0.
+    pub fn new(book_writer: BookWriter<File>, chapter_id: impl Into<ChapterId>) -> Self {
+        let writer = book_writer.new_chapter(chapter_id);
+        Self { writer, offset: 0 }
+    }
+
+    pub fn write_blob(&mut self, blob: &[u8]) -> Result<BlobRange> {
+        self.writer.write_all(blob)?;
+
+        let range = BlobRange {
+            offset: self.offset,
+            size: blob.len(),
+        };
+        self.offset += blob.len() as u64;
+        Ok(range)
+    }
+
+    pub fn close(self) -> bookfile::Result<BookWriter<File>> {
+        self.writer.close()
+    }
+}
--- a/pageserver/src/layered_repository/delta_layer.rs
+++ b/pageserver/src/layered_repository/delta_layer.rs
@@ -0,0 +1,490 @@
+//!
+//! A DeltaLayer represents a collection of WAL records or page images in a range of
+//! LSNs, for one segment. It is stored on a file on disk.
+//!
+//! Usually a delta layer only contains differences - in the form of WAL records against
+//! a base LSN. However, if a segment is newly created, by creating a new relation or
+//! extending an old one, there might be no base image. In that case, all the entries in
+//! the delta layer must be page images or WAL records with the 'will_init' flag set, so
+//! that they can be replayed without referring to an older page version. Also in some
+//! circumstances, the predecessor layer might actually be another delta layer. That
+//! can happen when you create a new branch in the middle of a delta layer, and the WAL
+//! records on the new branch are put in a new delta layer.
+//!
+//! When a delta file needs to be accessed, we slurp the metadata and relsize chapters
+//! into memory, into the DeltaLayerInner struct. See load() and unload() functions.
+//! To access a page/WAL record, we search `page_version_metas` for the block # and LSN.
+//! The byte ranges in the metadata can be used to find the page/WAL record in
+//! PAGE_VERSIONS_CHAPTER.
+//!
+//! On disk, the delta files are stored in timelines/<timelineid> directory.
+//! Currently, there are no subdirectories, and each delta file is named like this:
+//!
+//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>_<end LSN>
+//!
+//! For example:
+//!
+//!    1663_13990_2609_0_5_000000000169C348_000000000169C349
+//!
+//! If a relation is dropped, we add a '_DROPPED' to the end of the filename to indicate that.
+//! So the above example would become:
+//!
+//!    1663_13990_2609_0_5_000000000169C348_000000000169C349_DROPPED
+//!
+//! The end LSN indicates when it was dropped in that case, we don't store it in the
+//! file contents in any way.
+//!
+//! A detlta file is constructed using the 'bookfile' crate. Each file consists of two
+//! parts: the page versions and the relation sizes. They are stored as separate chapters.
+//!
+use crate::layered_repository::blob::BlobWriter;
+use crate::layered_repository::filename::DeltaFileName;
+use crate::layered_repository::storage_layer::{
+    Layer, PageReconstructData, PageVersion, SegmentTag,
+};
+use crate::repository::WALRecord;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
+use anyhow::{bail, Result};
+use bytes::Bytes;
+use log::*;
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
+use std::fs;
+use std::fs::File;
+use std::io::Write;
+use std::ops::Bound::Included;
+use std::path::PathBuf;
+use std::sync::{Arc, Mutex, MutexGuard};
+
+use bookfile::{Book, BookWriter};
+
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
+
+use super::blob::{read_blob, BlobRange};
+
+// Magic constant to identify a Zenith delta file
+static DELTA_FILE_MAGIC: u32 = 0x5A616E01;
+
+/// Mapping from (block #, lsn) -> page/WAL record
+/// byte ranges in PAGE_VERSIONS_CHAPTER
+static PAGE_VERSION_METAS_CHAPTER: u64 = 1;
+/// Page/WAL bytes - cannot be interpreted
+/// without PAGE_VERSION_METAS_CHAPTER
+static PAGE_VERSIONS_CHAPTER: u64 = 2;
+static REL_SIZES_CHAPTER: u64 = 3;
+
+#[derive(Serialize, Deserialize)]
+struct PageVersionMeta {
+    page_image_range: Option<BlobRange>,
+    record_range: Option<BlobRange>,
+}
+
+///
+/// DeltaLayer is the in-memory data structure associated with an
+/// on-disk delta file.  We keep a DeltaLayer in memory for each
+/// file, in the LayerMap. If a layer is in "loaded" state, we have a
+/// copy of the file in memory, in 'inner'. Otherwise the struct is
+/// just a placeholder for a file that exists on disk, and it needs to
+/// be loaded before using it in queries.
+///
+pub struct DeltaLayer {
+    conf: &'static PageServerConf,
+    pub tenantid: ZTenantId,
+    pub timelineid: ZTimelineId,
+    pub seg: SegmentTag,
+
+    //
+    // This entry contains all the changes from 'start_lsn' to 'end_lsn'. The
+    // start is inclusive, and end is exclusive.
+    //
+    pub start_lsn: Lsn,
+    pub end_lsn: Lsn,
+
+    dropped: bool,
+
+    /// Base layer preceding this layer.
+    predecessor: Option<Arc<dyn Layer>>,
+
+    inner: Mutex<DeltaLayerInner>,
+}
+
+pub struct DeltaLayerInner {
+    /// If false, the 'page_version_metas' and 'relsizes' have not been
+    /// loaded into memory yet.
+    loaded: bool,
+
+    /// All versions of all pages in the file are are kept here.
+    /// Indexed by block number and LSN.
+    page_version_metas: BTreeMap<(u32, Lsn), PageVersionMeta>,
+
+    /// `relsizes` tracks the size of the relation at different points in time.
+    relsizes: BTreeMap<Lsn, u32>,
+}
+
+impl Layer for DeltaLayer {
+    fn get_timeline_id(&self) -> ZTimelineId {
+        return self.timelineid;
+    }
+
+    fn get_seg_tag(&self) -> SegmentTag {
+        return self.seg;
+    }
+
+    fn is_dropped(&self) -> bool {
+        return self.dropped;
+    }
+
+    fn get_start_lsn(&self) -> Lsn {
+        return self.start_lsn;
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
+        return self.end_lsn;
+    }
+
+    fn filename(&self) -> PathBuf {
+        PathBuf::from(
+            DeltaFileName {
+                seg: self.seg,
+                start_lsn: self.start_lsn,
+                end_lsn: self.end_lsn,
+                dropped: self.dropped,
+            }
+            .to_string(),
+        )
+    }
+
+    /// Look up given page in the cache.
+    fn get_page_reconstruct_data(
+        &self,
+        blknum: u32,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<Option<Lsn>> {
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let mut need_base_image_lsn: Option<Lsn> = Some(lsn);
+
+        // TODO: avoid opening the snapshot file for each read
+        let (_path, book) = self.open_book()?;
+        let page_version_reader = book.chapter_reader(PAGE_VERSIONS_CHAPTER)?;
+        {
+            let inner = self.load()?;
+            let minkey = (blknum, Lsn(0));
+            let maxkey = (blknum, lsn);
+            let mut iter = inner
+                .page_version_metas
+                .range((Included(&minkey), Included(&maxkey)));
+            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
+                if let Some(img_range) = &entry.page_image_range {
+                    let img = Bytes::from(read_blob(&page_version_reader, img_range)?);
+                    reconstruct_data.page_img = Some(img);
+                    need_base_image_lsn = None;
+                    break;
+                } else if let Some(rec_range) = &entry.record_range {
+                    let rec = WALRecord::des(&read_blob(&page_version_reader, rec_range)?)?;
+                    let will_init = rec.will_init;
+                    reconstruct_data.records.push(rec);
+                    if will_init {
+                        // This WAL record initializes the page, so no need to go further back
+                        need_base_image_lsn = None;
+                        break;
+                    } else {
+                        need_base_image_lsn = Some(*entry_lsn);
+                    }
+                } else {
+                    // No base image, and no WAL record. Huh?
+                    bail!("no page image or WAL record for requested page");
+                }
+            }
+
+            // Use the base image, if needed
+            if let Some(need_lsn) = need_base_image_lsn {
+                if let Some(predecessor) = &self.predecessor {
+                    need_base_image_lsn = predecessor.get_page_reconstruct_data(
+                        blknum,
+                        need_lsn,
+                        reconstruct_data,
+                    )?;
+                } else {
+                    bail!(
+                        "no base img found for {} at blk {} at LSN {}",
+                        self.seg,
+                        blknum,
+                        lsn
+                    );
+                }
+            }
+
+            // release lock on 'inner'
+        }
+
+        Ok(need_base_image_lsn)
+    }
+
+    /// Get size of the relation at given LSN
+    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
+        assert!(lsn >= self.start_lsn);
+
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let inner = self.load()?;
+        let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn)));
+
+        let result;
+        if let Some((_entry_lsn, entry)) = iter.next_back() {
+            result = *entry;
+        // Use the base image if needed
+        } else if let Some(predecessor) = &self.predecessor {
+            result = predecessor.get_seg_size(lsn)?;
+        } else {
+            result = 0;
+        }
+        Ok(result)
+    }
+
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
+        // Is the requested LSN after the rel was dropped?
+        if self.dropped && lsn >= self.end_lsn {
+            return Ok(false);
+        }
+
+        // Otherwise, it exists.
+        Ok(true)
+    }
+
+    ///
+    /// Release most of the memory used by this layer. If it's accessed again later,
+    /// it will need to be loaded back.
+    ///
+    fn unload(&self) -> Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.page_version_metas = BTreeMap::new();
+        inner.relsizes = BTreeMap::new();
+        inner.loaded = false;
+        Ok(())
+    }
+
+    fn delete(&self) -> Result<()> {
+        // delete underlying file
+        fs::remove_file(self.path())?;
+        Ok(())
+    }
+
+    fn is_incremental(&self) -> bool {
+        true
+    }
+}
+
+impl DeltaLayer {
+    fn path(&self) -> PathBuf {
+        Self::path_for(
+            self.conf,
+            self.timelineid,
+            self.tenantid,
+            &DeltaFileName {
+                seg: self.seg,
+                start_lsn: self.start_lsn,
+                end_lsn: self.end_lsn,
+                dropped: self.dropped,
+            },
+        )
+    }
+
+    fn path_for(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        fname: &DeltaFileName,
+    ) -> PathBuf {
+        conf.timeline_path(&timelineid, &tenantid)
+            .join(fname.to_string())
+    }
+
+    /// Create a new delta file, using the given btreemaps containing the page versions and
+    /// relsizes.
+    ///
+    /// This is used to write the in-memory layer to disk. The in-memory layer uses the same
+    /// data structure with two btreemaps as we do, so passing the btreemaps is currently
+    /// expedient.
+    pub fn create(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        seg: SegmentTag,
+        start_lsn: Lsn,
+        end_lsn: Lsn,
+        dropped: bool,
+        predecessor: Option<Arc<dyn Layer>>,
+        page_versions: BTreeMap<(u32, Lsn), PageVersion>,
+        relsizes: BTreeMap<Lsn, u32>,
+    ) -> Result<DeltaLayer> {
+        let delta_layer = DeltaLayer {
+            conf: conf,
+            timelineid: timelineid,
+            tenantid: tenantid,
+            seg: seg,
+            start_lsn: start_lsn,
+            end_lsn,
+            dropped,
+            inner: Mutex::new(DeltaLayerInner {
+                loaded: true,
+                page_version_metas: BTreeMap::new(),
+                relsizes: relsizes,
+            }),
+            predecessor,
+        };
+        let mut inner = delta_layer.inner.lock().unwrap();
+
+        // Write the in-memory btreemaps into a file
+        let path = delta_layer.path();
+
+        // Note: This overwrites any existing file. There shouldn't be any.
+        // FIXME: throw an error instead?
+        let file = File::create(&path)?;
+        let book = BookWriter::new(file, DELTA_FILE_MAGIC)?;
+
+        let mut page_version_writer = BlobWriter::new(book, PAGE_VERSIONS_CHAPTER);
+
+        for (key, page_version) in page_versions {
+            let page_image_range = page_version
+                .page_image
+                .map(|page_image| page_version_writer.write_blob(page_image.as_ref()))
+                .transpose()?;
+
+            let record_range = page_version
+                .record
+                .map(|record| {
+                    let buf = WALRecord::ser(&record)?;
+                    page_version_writer.write_blob(&buf)
+                })
+                .transpose()?;
+
+            let old = inner.page_version_metas.insert(
+                key,
+                PageVersionMeta {
+                    page_image_range,
+                    record_range,
+                },
+            );
+
+            assert!(old.is_none());
+        }
+
+        let book = page_version_writer.close()?;
+
+        // Write out page versions
+        let mut chapter = book.new_chapter(PAGE_VERSION_METAS_CHAPTER);
+        let buf = BTreeMap::ser(&inner.page_version_metas)?;
+        chapter.write_all(&buf)?;
+        let book = chapter.close()?;
+
+        // and relsizes to separate chapter
+        let mut chapter = book.new_chapter(REL_SIZES_CHAPTER);
+        let buf = BTreeMap::ser(&inner.relsizes)?;
+        chapter.write_all(&buf)?;
+        let book = chapter.close()?;
+
+        book.close()?;
+
+        trace!("saved {}", &path.display());
+
+        drop(inner);
+
+        Ok(delta_layer)
+    }
+
+    fn open_book(&self) -> Result<(PathBuf, Book<File>)> {
+        let path = Self::path_for(
+            self.conf,
+            self.timelineid,
+            self.tenantid,
+            &DeltaFileName {
+                seg: self.seg,
+                start_lsn: self.start_lsn,
+                end_lsn: self.end_lsn,
+                dropped: self.dropped,
+            },
+        );
+
+        let file = File::open(&path)?;
+        let book = Book::new(file)?;
+
+        Ok((path, book))
+    }
+
+    ///
+    /// Load the contents of the file into memory
+    ///
+    fn load(&self) -> Result<MutexGuard<DeltaLayerInner>> {
+        // quick exit if already loaded
+        let mut inner = self.inner.lock().unwrap();
+
+        if inner.loaded {
+            return Ok(inner);
+        }
+
+        let (path, book) = self.open_book()?;
+
+        let chapter = book.read_chapter(PAGE_VERSION_METAS_CHAPTER)?;
+        let page_version_metas = BTreeMap::des(&chapter)?;
+
+        let chapter = book.read_chapter(REL_SIZES_CHAPTER)?;
+        let relsizes = BTreeMap::des(&chapter)?;
+
+        debug!("loaded from {}", &path.display());
+
+        *inner = DeltaLayerInner {
+            loaded: true,
+            page_version_metas,
+            relsizes,
+        };
+
+        Ok(inner)
+    }
+
+    /// Create a DeltaLayer struct representing an existing file on disk.
+    pub fn new(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        filename: &DeltaFileName,
+        predecessor: Option<Arc<dyn Layer>>,
+    ) -> DeltaLayer {
+        DeltaLayer {
+            conf,
+            timelineid,
+            tenantid,
+            seg: filename.seg,
+            start_lsn: filename.start_lsn,
+            end_lsn: filename.end_lsn,
+            dropped: filename.dropped,
+            inner: Mutex::new(DeltaLayerInner {
+                loaded: false,
+                page_version_metas: BTreeMap::new(),
+                relsizes: BTreeMap::new(),
+            }),
+            predecessor,
+        }
+    }
+
+    /// debugging function to print out the contents of the layer
+    #[allow(unused)]
+    pub fn dump(&self) -> String {
+        let mut result = format!(
+            "----- snapshot layer for {} {}-{} ----\n",
+            self.seg, self.start_lsn, self.end_lsn
+        );
+
+        let inner = self.inner.lock().unwrap();
+        for (k, v) in inner.relsizes.iter() {
+            result += &format!("{}: {}\n", k, v);
+        }
+        //for (k, v) in inner.page_versions.iter() {
+        //    result += &format!("blk {} at {}: {}/{}\n", k.0, k.1, v.page_image.is_some(), v.record.is_some());
+        //}
+
+        result
+    }
+}
--- a/pageserver/src/layered_repository/filename.rs
+++ b/pageserver/src/layered_repository/filename.rs
@@ -0,0 +1,305 @@
+//!
+//! Helper functions for dealing with filenames of the image and delta layer files.
+//!
+use crate::layered_repository::storage_layer::SegmentTag;
+use crate::relish::*;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
+use std::fmt;
+use std::fs;
+
+use anyhow::Result;
+use log::*;
+use zenith_utils::lsn::Lsn;
+
+// Note: LayeredTimeline::load_layer_map() relies on this sort order
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+pub struct DeltaFileName {
+    pub seg: SegmentTag,
+    pub start_lsn: Lsn,
+    pub end_lsn: Lsn,
+    pub dropped: bool,
+}
+
+/// Represents the filename of a DeltaLayer
+///
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>
+///
+/// or if it was dropped:
+///
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>_DROPPED
+///
+impl DeltaFileName {
+    ///
+    /// Parse a string as a delta file name. Returns None if the filename does not
+    /// match the expected pattern.
+    ///
+    fn from_str(fname: &str) -> Option<Self> {
+        let rel;
+        let mut parts;
+        if let Some(rest) = fname.strip_prefix("rel_") {
+            parts = rest.split('_');
+            rel = RelishTag::Relation(RelTag {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+                relnode: parts.next()?.parse::<u32>().ok()?,
+                forknum: parts.next()?.parse::<u8>().ok()?,
+            });
+        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
+            parts = rest.split('_');
+            rel = RelishTag::FileNodeMap {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
+            parts = rest.split('_');
+            rel = RelishTag::TwoPhase {
+                xid: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
+            parts = rest.split('_');
+            rel = RelishTag::Checkpoint;
+        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
+            parts = rest.split('_');
+            rel = RelishTag::ControlFile;
+        } else {
+            return None;
+        }
+
+        let segno = parts.next()?.parse::<u32>().ok()?;
+
+        let seg = SegmentTag { rel, segno };
+
+        let start_lsn = Lsn::from_hex(parts.next()?).ok()?;
+        let end_lsn = Lsn::from_hex(parts.next()?).ok()?;
+
+        let mut dropped = false;
+        if let Some(suffix) = parts.next() {
+            if suffix == "DROPPED" {
+                dropped = true;
+            } else {
+                return None;
+            }
+        }
+        if parts.next().is_some() {
+            return None;
+        }
+
+        Some(DeltaFileName {
+            seg,
+            start_lsn,
+            end_lsn,
+            dropped,
+        })
+    }
+
+    fn to_string(&self) -> String {
+        let basename = match self.seg.rel {
+            RelishTag::Relation(reltag) => format!(
+                "rel_{}_{}_{}_{}",
+                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
+            ),
+            RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno,
+            } => format!("pg_xact_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno,
+            } => format!("pg_multixact_members_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno,
+            } => format!("pg_multixact_offsets_{:04X}", segno),
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
+            RelishTag::Checkpoint => format!("pg_control_checkpoint"),
+            RelishTag::ControlFile => format!("pg_control"),
+        };
+
+        format!(
+            "{}_{}_{:016X}_{:016X}{}",
+            basename,
+            self.seg.segno,
+            u64::from(self.start_lsn),
+            u64::from(self.end_lsn),
+            if self.dropped { "_DROPPED" } else { "" }
+        )
+    }
+}
+
+impl fmt::Display for DeltaFileName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.to_string())
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+pub struct ImageFileName {
+    pub seg: SegmentTag,
+    pub lsn: Lsn,
+}
+
+///
+/// Represents the filename of an ImageLayer
+///
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<LSN>
+///
+impl ImageFileName {
+    ///
+    /// Parse a string as an image file name. Returns None if the filename does not
+    /// match the expected pattern.
+    ///
+    fn from_str(fname: &str) -> Option<Self> {
+        let rel;
+        let mut parts;
+        if let Some(rest) = fname.strip_prefix("rel_") {
+            parts = rest.split('_');
+            rel = RelishTag::Relation(RelTag {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+                relnode: parts.next()?.parse::<u32>().ok()?,
+                forknum: parts.next()?.parse::<u8>().ok()?,
+            });
+        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
+            parts = rest.split('_');
+            rel = RelishTag::FileNodeMap {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
+            parts = rest.split('_');
+            rel = RelishTag::TwoPhase {
+                xid: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
+            parts = rest.split('_');
+            rel = RelishTag::Checkpoint;
+        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
+            parts = rest.split('_');
+            rel = RelishTag::ControlFile;
+        } else {
+            return None;
+        }
+
+        let segno = parts.next()?.parse::<u32>().ok()?;
+
+        let seg = SegmentTag { rel, segno };
+
+        let lsn = Lsn::from_hex(parts.next()?).ok()?;
+
+        if parts.next().is_some() {
+            return None;
+        }
+
+        Some(ImageFileName { seg, lsn })
+    }
+
+    fn to_string(&self) -> String {
+        let basename = match self.seg.rel {
+            RelishTag::Relation(reltag) => format!(
+                "rel_{}_{}_{}_{}",
+                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
+            ),
+            RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno,
+            } => format!("pg_xact_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno,
+            } => format!("pg_multixact_members_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno,
+            } => format!("pg_multixact_offsets_{:04X}", segno),
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
+            RelishTag::Checkpoint => format!("pg_control_checkpoint"),
+            RelishTag::ControlFile => format!("pg_control"),
+        };
+
+        format!(
+            "{}_{}_{:016X}",
+            basename,
+            self.seg.segno,
+            u64::from(self.lsn),
+        )
+    }
+}
+
+impl fmt::Display for ImageFileName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.to_string())
+    }
+}
+
+/// Scan timeline directory and create ImageFileName and DeltaFilename
+/// structs representing all files on disk
+///
+/// TODO: returning an Iterator would be more idiomatic
+pub fn list_files(
+    conf: &'static PageServerConf,
+    timelineid: ZTimelineId,
+    tenantid: ZTenantId,
+) -> Result<(Vec<ImageFileName>, Vec<DeltaFileName>)> {
+    let path = conf.timeline_path(&timelineid, &tenantid);
+
+    let mut deltafiles: Vec<DeltaFileName> = Vec::new();
+    let mut imgfiles: Vec<ImageFileName> = Vec::new();
+    for direntry in fs::read_dir(path)? {
+        let fname = direntry?.file_name();
+        let fname = fname.to_str().unwrap();
+
+        if let Some(deltafilename) = DeltaFileName::from_str(fname) {
+            deltafiles.push(deltafilename);
+        } else if let Some(imgfilename) = ImageFileName::from_str(fname) {
+            imgfiles.push(imgfilename);
+        } else if fname == "wal" || fname == "metadata" {
+            // ignore these
+        } else {
+            warn!("unrecognized filename in timeline dir: {}", fname);
+        }
+    }
+    return Ok((imgfiles, deltafiles));
+}
--- a/pageserver/src/layered_repository/image_layer.rs
+++ b/pageserver/src/layered_repository/image_layer.rs
@@ -0,0 +1,405 @@
+//! An ImageLayer represents an image or a snapshot of a segment at one particular LSN.
+//! It is stored in a file on disk.
+//!
+//! On disk, the image files are stored in timelines/<timelineid> directory.
+//! Currently, there are no subdirectories, and each snapshot file is named like this:
+//!
+//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<LSN>
+//!
+//! For example:
+//!
+//!    1663_13990_2609_0_5_000000000169C348
+//!
+//! An image file is constructed using the 'bookfile' crate.
+//!
+//! Only metadata is loaded into memory by the load function.
+//! When images are needed, they are read directly from disk.
+//!
+//! For blocky segments, the images are stored in BLOCKY_IMAGES_CHAPTER.
+//! All the images are required to be BLOCK_SIZE, which allows for random access.
+//!
+//! For non-blocky segments, the image can be found in NONBLOCKY_IMAGE_CHAPTER.
+//!
+use crate::layered_repository::filename::ImageFileName;
+use crate::layered_repository::storage_layer::{Layer, PageReconstructData, SegmentTag};
+use crate::layered_repository::LayeredTimeline;
+use crate::layered_repository::RELISH_SEG_SIZE;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
+use anyhow::{anyhow, bail, ensure, Result};
+use bytes::Bytes;
+use log::*;
+use std::convert::TryInto;
+use std::fs;
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+use std::sync::{Mutex, MutexGuard};
+
+use bookfile::{Book, BookWriter};
+
+use zenith_utils::lsn::Lsn;
+
+// Magic constant to identify a Zenith segment image file
+const IMAGE_FILE_MAGIC: u32 = 0x5A616E01 + 1;
+
+/// Contains each block in block # order
+const BLOCKY_IMAGES_CHAPTER: u64 = 1;
+const NONBLOCKY_IMAGE_CHAPTER: u64 = 2;
+
+const BLOCK_SIZE: usize = 8192;
+
+///
+/// ImageLayer is the in-memory data structure associated with an on-disk image
+/// file.  We keep an ImageLayer in memory for each file, in the LayerMap. If a
+/// layer is in "loaded" state, we have a copy of the file in memory, in 'inner'.
+/// Otherwise the struct is just a placeholder for a file that exists on disk,
+/// and it needs to be loaded before using it in queries.
+///
+pub struct ImageLayer {
+    conf: &'static PageServerConf,
+    pub tenantid: ZTenantId,
+    pub timelineid: ZTimelineId,
+    pub seg: SegmentTag,
+
+    // This entry contains an image of all pages as of this LSN
+    pub lsn: Lsn,
+
+    inner: Mutex<ImageLayerInner>,
+}
+
+#[derive(Clone)]
+enum ImageType {
+    Blocky { num_blocks: u32 },
+    NonBlocky,
+}
+
+pub struct ImageLayerInner {
+    /// If false, the 'image_type' has not been
+    /// loaded into memory yet.
+    loaded: bool,
+
+    /// Derived from filename and bookfile chapter metadata
+    image_type: ImageType,
+}
+
+impl Layer for ImageLayer {
+    fn filename(&self) -> PathBuf {
+        PathBuf::from(
+            ImageFileName {
+                seg: self.seg,
+                lsn: self.lsn,
+            }
+            .to_string(),
+        )
+    }
+
+    fn get_timeline_id(&self) -> ZTimelineId {
+        return self.timelineid;
+    }
+
+    fn get_seg_tag(&self) -> SegmentTag {
+        return self.seg;
+    }
+
+    fn is_dropped(&self) -> bool {
+        return false;
+    }
+
+    fn get_start_lsn(&self) -> Lsn {
+        return self.lsn;
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
+        return self.lsn;
+    }
+
+    /// Look up given page in the file
+    fn get_page_reconstruct_data(
+        &self,
+        blknum: u32,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<Option<Lsn>> {
+        assert!(lsn >= self.lsn);
+
+        let inner = self.load()?;
+
+        let base_blknum = blknum % RELISH_SEG_SIZE;
+
+        let (_path, book) = self.open_book()?;
+
+        let buf = match &inner.image_type {
+            ImageType::Blocky { num_blocks } => {
+                if base_blknum >= *num_blocks {
+                    bail!(
+                        "no base img found for {} at blk {} at LSN {}",
+                        self.seg,
+                        base_blknum,
+                        lsn
+                    );
+                }
+
+                let mut buf = vec![0u8; BLOCK_SIZE];
+                let offset = BLOCK_SIZE as u64 * base_blknum as u64;
+
+                let chapter = book.chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
+                chapter.read_exact_at(&mut buf, offset)?;
+
+                buf
+            }
+            ImageType::NonBlocky => {
+                ensure!(base_blknum == 0);
+                book.read_chapter(NONBLOCKY_IMAGE_CHAPTER)?.into_vec()
+            }
+        };
+
+        reconstruct_data.page_img = Some(Bytes::from(buf));
+        Ok(None)
+    }
+
+    /// Get size of the segment
+    fn get_seg_size(&self, _lsn: Lsn) -> Result<u32> {
+        let inner = self.load()?;
+        match inner.image_type {
+            ImageType::Blocky { num_blocks } => Ok(num_blocks),
+            ImageType::NonBlocky => Err(anyhow!("get_seg_size called for non-blocky segment")),
+        }
+    }
+
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, _lsn: Lsn) -> Result<bool> {
+        Ok(true)
+    }
+
+    ///
+    /// Release most of the memory used by this layer. If it's accessed again later,
+    /// it will need to be loaded back.
+    ///
+    fn unload(&self) -> Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.image_type = ImageType::Blocky { num_blocks: 0 };
+        inner.loaded = false;
+        Ok(())
+    }
+
+    fn delete(&self) -> Result<()> {
+        // delete underlying file
+        fs::remove_file(self.path())?;
+        Ok(())
+    }
+
+    fn is_incremental(&self) -> bool {
+        false
+    }
+}
+
+impl ImageLayer {
+    fn path(&self) -> PathBuf {
+        Self::path_for(
+            self.conf,
+            self.timelineid,
+            self.tenantid,
+            &ImageFileName {
+                seg: self.seg,
+                lsn: self.lsn,
+            },
+        )
+    }
+
+    fn path_for(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        fname: &ImageFileName,
+    ) -> PathBuf {
+        conf.timeline_path(&timelineid, &tenantid)
+            .join(fname.to_string())
+    }
+
+    /// Create a new image file, using the given array of pages.
+    fn create(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        seg: SegmentTag,
+        lsn: Lsn,
+        base_images: Vec<Bytes>,
+    ) -> Result<ImageLayer> {
+        let image_type = if seg.rel.is_blocky() {
+            let num_blocks: u32 = base_images.len().try_into()?;
+            ImageType::Blocky { num_blocks }
+        } else {
+            assert_eq!(base_images.len(), 1);
+            ImageType::NonBlocky
+        };
+
+        let layer = ImageLayer {
+            conf: conf,
+            timelineid: timelineid,
+            tenantid: tenantid,
+            seg: seg,
+            lsn: lsn,
+            inner: Mutex::new(ImageLayerInner {
+                loaded: true,
+                image_type: image_type.clone(),
+            }),
+        };
+        let inner = layer.inner.lock().unwrap();
+
+        // Write the images into a file
+        let path = layer.path();
+
+        // Note: This overwrites any existing file. There shouldn't be any.
+        // FIXME: throw an error instead?
+        let file = File::create(&path)?;
+        let book = BookWriter::new(file, IMAGE_FILE_MAGIC)?;
+
+        let book = match &image_type {
+            ImageType::Blocky { .. } => {
+                let mut chapter = book.new_chapter(BLOCKY_IMAGES_CHAPTER);
+                for block_bytes in base_images {
+                    assert_eq!(block_bytes.len(), BLOCK_SIZE);
+                    chapter.write_all(&block_bytes)?;
+                }
+                chapter.close()?
+            }
+            ImageType::NonBlocky => {
+                let mut chapter = book.new_chapter(NONBLOCKY_IMAGE_CHAPTER);
+                chapter.write_all(&base_images[0])?;
+                chapter.close()?
+            }
+        };
+
+        book.close()?;
+
+        trace!("saved {}", &path.display());
+
+        drop(inner);
+
+        Ok(layer)
+    }
+
+    // Create a new image file by materializing every page in a source layer
+    // at given LSN.
+    pub fn create_from_src(
+        conf: &'static PageServerConf,
+        timeline: &LayeredTimeline,
+        src: &dyn Layer,
+        lsn: Lsn,
+    ) -> Result<ImageLayer> {
+        let seg = src.get_seg_tag();
+        let timelineid = timeline.timelineid;
+
+        let startblk;
+        let size;
+        if seg.rel.is_blocky() {
+            size = src.get_seg_size(lsn)?;
+            startblk = seg.segno * RELISH_SEG_SIZE;
+        } else {
+            size = 1;
+            startblk = 0;
+        }
+
+        trace!(
+            "creating new ImageLayer for {} on timeline {} at {}",
+            seg,
+            timelineid,
+            lsn,
+        );
+
+        let mut base_images: Vec<Bytes> = Vec::new();
+        for blknum in startblk..(startblk + size) {
+            let img = timeline.materialize_page(seg, blknum, lsn, &*src)?;
+
+            base_images.push(img);
+        }
+
+        Self::create(conf, timelineid, timeline.tenantid, seg, lsn, base_images)
+    }
+
+    ///
+    /// Load the contents of the file into memory
+    ///
+    fn load(&self) -> Result<MutexGuard<ImageLayerInner>> {
+        // quick exit if already loaded
+        let mut inner = self.inner.lock().unwrap();
+
+        if inner.loaded {
+            return Ok(inner);
+        }
+
+        let (path, book) = self.open_book()?;
+
+        let image_type = if self.seg.rel.is_blocky() {
+            let chapter = book.chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
+            let images_len = chapter.len();
+            ensure!(images_len % BLOCK_SIZE as u64 == 0);
+            let num_blocks: u32 = (images_len / BLOCK_SIZE as u64).try_into()?;
+            ImageType::Blocky { num_blocks }
+        } else {
+            let _chapter = book.chapter_reader(NONBLOCKY_IMAGE_CHAPTER)?;
+            ImageType::NonBlocky
+        };
+
+        debug!("loaded from {}", &path.display());
+
+        *inner = ImageLayerInner {
+            loaded: true,
+            image_type,
+        };
+
+        Ok(inner)
+    }
+
+    fn open_book(&self) -> Result<(PathBuf, Book<File>)> {
+        let path = Self::path_for(
+            self.conf,
+            self.timelineid,
+            self.tenantid,
+            &ImageFileName {
+                seg: self.seg,
+                lsn: self.lsn,
+            },
+        );
+
+        let file = File::open(&path)?;
+        let book = Book::new(file)?;
+
+        Ok((path, book))
+    }
+
+    /// Create an ImageLayer struct representing an existing file on disk
+    pub fn new(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        filename: &ImageFileName,
+    ) -> ImageLayer {
+        ImageLayer {
+            conf,
+            timelineid,
+            tenantid,
+            seg: filename.seg,
+            lsn: filename.lsn,
+            inner: Mutex::new(ImageLayerInner {
+                loaded: false,
+                image_type: ImageType::Blocky { num_blocks: 0 },
+            }),
+        }
+    }
+
+    /// debugging function to print out the contents of the layer
+    #[allow(unused)]
+    pub fn dump(&self) -> String {
+        let mut result = format!("----- image layer for {} at {} ----\n", self.seg, self.lsn);
+
+        //let inner = self.inner.lock().unwrap();
+
+        //for (k, v) in inner.page_versions.iter() {
+        //    result += &format!("blk {} at {}: {}/{}\n", k.0, k.1, v.page_image.is_some(), v.record.is_some());
+        //}
+
+        result
+    }
+}
--- a/pageserver/src/layered_repository/inmemory_layer.rs
+++ b/pageserver/src/layered_repository/inmemory_layer.rs
@@ -2,19 +2,21 @@
 //! An in-memory layer stores recently received page versions in memory. The page versions
 //! are held in a BTreeMap, and there's another BTreeMap to track the size of the relation.
 //!
-use crate::layered_repository::page_history::PageHistory;
+use crate::layered_repository::filename::DeltaFileName;
 use crate::layered_repository::storage_layer::{
    Layer, PageReconstructData, PageVersion, SegmentTag, RELISH_SEG_SIZE,
 };
-use crate::layered_repository::{LayeredTimeline, SnapshotLayer};
+use crate::layered_repository::LayeredTimeline;
+use crate::layered_repository::{DeltaLayer, ImageLayer};
 use crate::repository::WALRecord;
 use crate::PageServerConf;
 use crate::{ZTenantId, ZTimelineId};
-use anyhow::{anyhow, bail, Result};
+use anyhow::{bail, Result};
 use bytes::Bytes;
 use log::*;
 use std::collections::BTreeMap;
 use std::ops::Bound::Included;
+use std::path::PathBuf;
 use std::sync::{Arc, Mutex};

 use zenith_utils::lsn::Lsn;
@@ -32,9 +34,15 @@ pub struct InMemoryLayer {
    ///
    start_lsn: Lsn,

+    /// LSN of the oldest page version stored in this layer
+    oldest_pending_lsn: Lsn,
+
    /// The above fields never change. The parts that do change are in 'inner',
    /// and protected by mutex.
    inner: Mutex<InMemoryLayerInner>,
+
+    /// Predecessor layer
+    img_layer: Option<Arc<dyn Layer>>,
 }

 pub struct InMemoryLayerInner {
@@ -43,9 +51,9 @@ pub struct InMemoryLayerInner {

    ///
    /// All versions of all pages in the layer are are kept here.
-    /// Indexed by block number.
+    /// Indexed by block number and LSN.
    ///
-    pages: BTreeMap<u32, PageHistory>,
+    page_versions: BTreeMap<(u32, Lsn), PageVersion>,

    ///
    /// `segsizes` tracks the size of the segment at different points in time.
@@ -53,7 +61,46 @@ pub struct InMemoryLayerInner {
    segsizes: BTreeMap<Lsn, u32>,
 }

+impl InMemoryLayerInner {
+    fn get_seg_size(&self, lsn: Lsn) -> u32 {
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let mut iter = self.segsizes.range((Included(&Lsn(0)), Included(&lsn)));
+
+        if let Some((_entry_lsn, entry)) = iter.next_back() {
+            *entry
+        } else {
+            0
+        }
+    }
+}
+
 impl Layer for InMemoryLayer {
+    // An in-memory layer doesn't really have a filename as it's not stored on disk,
+    // but we construct a filename as if it was a delta layer
+    fn filename(&self) -> PathBuf {
+        let inner = self.inner.lock().unwrap();
+
+        let end_lsn;
+        let dropped;
+        if let Some(drop_lsn) = inner.drop_lsn {
+            end_lsn = drop_lsn;
+            dropped = true;
+        } else {
+            end_lsn = Lsn(u64::MAX);
+            dropped = false;
+        }
+
+        let delta_filename = DeltaFileName {
+            seg: self.seg,
+            start_lsn: self.start_lsn,
+            end_lsn: end_lsn,
+            dropped: dropped,
+        }
+        .to_string();
+
+        PathBuf::from(format!("inmem-{}", delta_filename))
+    }
+
    fn get_timeline_id(&self) -> ZTimelineId {
        return self.timelineid;
    }
@@ -90,32 +137,29 @@ impl Layer for InMemoryLayer {
    ) -> Result<Option<Lsn>> {
        // Scan the BTreeMap backwards, starting from reconstruct_data.lsn.
        let mut need_base_image_lsn: Option<Lsn> = Some(lsn);
+
        assert!(self.seg.blknum_in_seg(blknum));

        {
            let inner = self.inner.lock().unwrap();
-            let pages = &inner.pages;
-
-            // FIXME: this assumes the latest page version is always the right answer.
-            // How should this work if the requested lsn is in the past? in the future?
-
-            let latest_version = pages
-                .get(&blknum)
-                .and_then(PageHistory::latest)
-                .ok_or_else(|| anyhow!("page not found"))?;
-
-            let (entry_lsn, entry) = latest_version;
-            if true {
+            let minkey = (blknum, Lsn(0));
+            let maxkey = (blknum, lsn);
+            let mut iter = inner
+                .page_versions
+                .range((Included(&minkey), Included(&maxkey)));
+            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
                if let Some(img) = &entry.page_image {
                    reconstruct_data.page_img = Some(img.clone());
                    need_base_image_lsn = None;
+                    break;
                } else if let Some(rec) = &entry.record {
                    reconstruct_data.records.push(rec.clone());
                    if rec.will_init {
                        // This WAL record initializes the page, so no need to go further back
                        need_base_image_lsn = None;
+                        break;
                    } else {
-                        need_base_image_lsn = Some(entry_lsn);
+                        need_base_image_lsn = Some(*entry_lsn);
                    }
                } else {
                    // No base image, and no WAL record. Huh?
@@ -123,7 +167,22 @@ impl Layer for InMemoryLayer {
                }
            }

-            // release lock on self.pages
+            // Use the base image, if needed
+            if let Some(need_lsn) = need_base_image_lsn {
+                if let Some(img_layer) = &self.img_layer {
+                    need_base_image_lsn =
+                        img_layer.get_page_reconstruct_data(blknum, need_lsn, reconstruct_data)?;
+                } else {
+                    bail!(
+                        "no base img found for {} at blk {} at LSN {}",
+                        self.seg,
+                        blknum,
+                        lsn
+                    );
+                }
+            }
+
+            // release lock on 'inner'
        }

        Ok(need_base_image_lsn)
@@ -131,18 +190,10 @@ impl Layer for InMemoryLayer {

    /// Get size of the relation at given LSN
    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let inner = self.inner.lock().unwrap();
-        let mut iter = inner.segsizes.range((Included(&Lsn(0)), Included(&lsn)));
+        assert!(lsn >= self.start_lsn);

-        if let Some((_entry_lsn, entry)) = iter.next_back() {
-            let result = *entry;
-            drop(inner);
-            trace!("get_seg_size: {} at {} -> {}", self.seg, lsn, result);
-            Ok(result)
-        } else {
-            bail!("No size found for {} at {} in memory", self.seg, lsn);
-        }
+        let inner = self.inner.lock().unwrap();
+        Ok(inner.get_seg_size(lsn))
    }

    /// Does this segment exist at given LSN?
@@ -159,9 +210,31 @@ impl Layer for InMemoryLayer {
        // Otherwise, it exists
        Ok(true)
    }
+
+    /// Cannot unload anything in an in-memory layer, since there's no backing
+    /// store. To release memory used by an in-memory layer, use 'freeze' to turn
+    /// it into an on-disk layer.
+    fn unload(&self) -> Result<()> {
+        Ok(())
+    }
+
+    /// Nothing to do here. When you drop the last reference to the layer, it will
+    /// be deallocated.
+    fn delete(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn is_incremental(&self) -> bool {
+        self.img_layer.is_some()
+    }
 }

 impl InMemoryLayer {
+    /// Return the oldest page version that's stored in this layer
+    pub fn get_oldest_pending_lsn(&self) -> Lsn {
+        self.oldest_pending_lsn
+    }
+
    ///
    /// Create a new, empty, in-memory layer
    ///
@@ -171,6 +244,7 @@ impl InMemoryLayer {
        tenantid: ZTenantId,
        seg: SegmentTag,
        start_lsn: Lsn,
+        oldest_pending_lsn: Lsn,
    ) -> Result<InMemoryLayer> {
        trace!(
            "initializing new empty InMemoryLayer for writing {} on timeline {} at {}",
@@ -185,11 +259,13 @@ impl InMemoryLayer {
            tenantid,
            seg,
            start_lsn,
+            oldest_pending_lsn,
            inner: Mutex::new(InMemoryLayerInner {
                drop_lsn: None,
-                pages: BTreeMap::new(),
+                page_versions: BTreeMap::new(),
                segsizes: BTreeMap::new(),
            }),
+            img_layer: None,
        })
    }

@@ -233,25 +309,23 @@ impl InMemoryLayer {
        );
        let mut inner = self.inner.lock().unwrap();

-        let page_history = inner
-            .pages
-            .entry(blknum)
-            .or_insert_with(PageHistory::default);
-        page_history.push(lsn, pv);
+        let old = inner.page_versions.insert((blknum, lsn), pv);
+
+        if old.is_some() {
+            // We already had an entry for this LSN. That's odd..
+            warn!(
+                "Page version of rel {} blk {} at {} already exists",
+                self.seg.rel, blknum, lsn
+            );
+        }

        // Also update the relation size, if this extended the relation.
        if self.seg.rel.is_blocky() {
            let newsize = blknum - self.seg.segno * RELISH_SEG_SIZE + 1;

-            let mut iter = inner.segsizes.range((Included(&Lsn(0)), Included(&lsn)));
-
-            let oldsize;
-            if let Some((_entry_lsn, entry)) = iter.next_back() {
-                oldsize = *entry;
-            } else {
-                oldsize = 0;
-                //bail!("No old size found for {} at {}", self.tag, lsn);
-            }
+            // use inner get_seg_size, since calling self.get_seg_size will try to acquire self.inner.lock
+            // which we've just acquired above
+            let oldsize = inner.get_seg_size(lsn);
            if newsize > oldsize {
                trace!(
                    "enlarging segment {} from {} to {} blocks at {}",
@@ -296,57 +370,43 @@ impl InMemoryLayer {
    /// Initialize a new InMemoryLayer for, by copying the state at the given
    /// point in time from given existing layer.
    ///
-    pub fn copy_snapshot(
+    pub fn create_successor_layer(
        conf: &'static PageServerConf,
-        timeline: &LayeredTimeline,
-        src: &dyn Layer,
+        src: Arc<dyn Layer>,
        timelineid: ZTimelineId,
        tenantid: ZTenantId,
-        lsn: Lsn,
+        start_lsn: Lsn,
+        oldest_pending_lsn: Lsn,
    ) -> Result<InMemoryLayer> {
-        trace!(
-            "initializing new InMemoryLayer for writing {} on timeline {} at {}",
-            src.get_seg_tag(),
-            timelineid,
-            lsn
-        );
-        let mut pages = BTreeMap::new();
-        let mut segsizes = BTreeMap::new();
-
        let seg = src.get_seg_tag();

-        let startblk;
-        let size;
-        if seg.rel.is_blocky() {
-            size = src.get_seg_size(lsn)?;
-            segsizes.insert(lsn, size);
-            startblk = seg.segno * RELISH_SEG_SIZE;
-        } else {
-            size = 1;
-            startblk = 0;
-        }
+        trace!(
+            "initializing new InMemoryLayer for writing {} on timeline {} at {}",
+            seg,
+            timelineid,
+            start_lsn,
+        );

-        for blknum in startblk..(startblk + size) {
-            let img = timeline.materialize_page(seg, blknum, lsn, src)?;
-            let pv = PageVersion {
-                page_image: Some(img),
-                record: None,
-            };
-            let page_history = PageHistory::from_image(lsn, pv);
-            pages.insert(blknum, page_history);
+        // For convenience, copy the segment size from the predecessor layer
+        let mut segsizes = BTreeMap::new();
+        if seg.rel.is_blocky() {
+            let size = src.get_seg_size(start_lsn)?;
+            segsizes.insert(start_lsn, size);
        }

        Ok(InMemoryLayer {
            conf,
            timelineid,
            tenantid,
-            seg: src.get_seg_tag(),
-            start_lsn: lsn,
+            seg,
+            start_lsn,
+            oldest_pending_lsn,
            inner: Mutex::new(InMemoryLayerInner {
                drop_lsn: None,
-                pages,
-                segsizes,
+                page_versions: BTreeMap::new(),
+                segsizes: segsizes,
            }),
+            img_layer: Some(src),
        })
    }

@@ -355,18 +415,21 @@ impl InMemoryLayer {
    ///
    /// The cutoff point for the layer that's written to disk is 'end_lsn'.
    ///
-    /// Returns new layers that replace this one. Always returns a
-    /// SnapshotLayer containing the page versions that were written to disk,
-    /// but if there were page versions newer than 'end_lsn', also return a new
-    /// in-memory layer containing those page versions. The caller replaces
-    /// this layer with the returned layers in the layer map.
+    /// Returns new layers that replace this one. Always returns a new image
+    /// layer containing the page versions at the cutoff LSN, that were written
+    /// to disk, and usually also a DeltaLayer that includes all the WAL records
+    /// between start LSN and the cutoff. (The delta layer is not needed when
+    /// a new relish is created with a single LSN, so that the start and end LSN
+    /// are the same.) If there were page versions newer than 'end_lsn', also
+    /// returns a new in-memory layer containing those page versions. The caller
+    /// replaces this layer with the returned layers in the layer map.
    ///
    pub fn freeze(
        &self,
        cutoff_lsn: Lsn,
        // This is needed just to call materialize_page()
        timeline: &LayeredTimeline,
-    ) -> Result<(Option<Arc<SnapshotLayer>>, Option<Arc<InMemoryLayer>>)> {
+    ) -> Result<(Vec<Arc<dyn Layer>>, Option<Arc<InMemoryLayer>>)> {
        info!(
            "freezing in memory layer for {} on timeline {} at {}",
            self.seg, self.timelineid, cutoff_lsn
@@ -388,11 +451,10 @@ impl InMemoryLayer {
        };

        // Divide all the page versions into old and new at the 'end_lsn' cutoff point.
-        let mut before_pages = BTreeMap::new();
+        let mut before_page_versions;
        let mut before_segsizes;
+        let mut after_page_versions;
        let mut after_segsizes;
-        let mut after_pages = BTreeMap::new();
-
        if !dropped {
            before_segsizes = BTreeMap::new();
            after_segsizes = BTreeMap::new();
@@ -404,60 +466,86 @@ impl InMemoryLayer {
                }
            }

-            for (blknum, page_history) in inner.pages.iter() {
-                let (old, new) = page_history.clone().split_at(end_lsn);
-                before_pages.insert(*blknum, old);
-                after_pages.insert(*blknum, new);
+            before_page_versions = BTreeMap::new();
+            after_page_versions = BTreeMap::new();
+            for ((blknum, lsn), pv) in inner.page_versions.iter() {
+                if *lsn == end_lsn {
+                    // Page versions at the cutoff LSN will be stored in the
+                    // materialized image layer.
+                } else if *lsn > end_lsn {
+                    after_page_versions.insert((*blknum, *lsn), pv.clone());
+                } else {
+                    before_page_versions.insert((*blknum, *lsn), pv.clone());
+                }
            }
        } else {
-            before_pages = inner.pages.clone();
+            before_page_versions = inner.page_versions.clone();
            before_segsizes = inner.segsizes.clone();
            after_segsizes = BTreeMap::new();
-            after_pages = BTreeMap::new();
+            after_page_versions = BTreeMap::new();
        }

        // we can release the lock now.
        drop(inner);

-        // Write the page versions before the cutoff to disk.
-        let snapfile = SnapshotLayer::create(
-            self.conf,
-            self.timelineid,
-            self.tenantid,
-            self.seg,
-            self.start_lsn,
-            end_lsn,
-            dropped,
-            before_pages,
-            before_segsizes,
-        )?;
+        let mut frozen_layers: Vec<Arc<dyn Layer>> = Vec::new();

-        // If there were any "new" page versions, initialize a new in-memory layer to hold
-        // them
-        let new_open = if !after_segsizes.is_empty() || !after_pages.is_empty() {
-            info!("created new in-mem layer for {} {}-", self.seg, end_lsn);
-
-            let new_open = Self::copy_snapshot(
+        if self.start_lsn != end_lsn {
+            // Write the page versions before the cutoff to disk.
+            let delta_layer = DeltaLayer::create(
                self.conf,
-                timeline,
-                &snapfile,
                self.timelineid,
                self.tenantid,
+                self.seg,
+                self.start_lsn,
                end_lsn,
+                dropped,
+                self.img_layer.clone(),
+                before_page_versions,
+                before_segsizes,
            )?;
-            let mut new_inner = new_open.inner.lock().unwrap();
-            new_inner.pages.append(&mut after_pages);
-            new_inner.segsizes.append(&mut after_segsizes);
-            drop(new_inner);
-
-            Some(Arc::new(new_open))
+            let delta_layer_rc: Arc<dyn Layer> = Arc::new(delta_layer);
+            frozen_layers.push(delta_layer_rc);
+            trace!(
+                "freeze: created delta layer {} {}-{}",
+                self.seg,
+                self.start_lsn,
+                end_lsn
+            );
        } else {
-            None
-        };
+            assert!(before_page_versions.is_empty());
+        }

-        let new_historic = Some(Arc::new(snapfile));
+        let mut new_open_rc = None;
+        if !dropped {
+            // Write a new base image layer at the cutoff point
+            let imgfile = ImageLayer::create_from_src(self.conf, timeline, self, end_lsn)?;
+            let imgfile_rc: Arc<dyn Layer> = Arc::new(imgfile);
+            frozen_layers.push(Arc::clone(&imgfile_rc));
+            trace!("freeze: created image layer {} at {}", self.seg, end_lsn);

-        Ok((new_historic, new_open))
+            // If there were any page versions newer than the cutoff, initialize a new in-memory
+            // layer to hold them
+            if !after_segsizes.is_empty() || !after_page_versions.is_empty() {
+                let new_open = Self::create_successor_layer(
+                    self.conf,
+                    imgfile_rc,
+                    self.timelineid,
+                    self.tenantid,
+                    end_lsn,
+                    end_lsn,
+                )?;
+                let mut new_inner = new_open.inner.lock().unwrap();
+                new_inner.page_versions.append(&mut after_page_versions);
+                new_inner.segsizes.append(&mut after_segsizes);
+                drop(new_inner);
+                trace!("freeze: created new in-mem layer {} {}-", self.seg, end_lsn);
+
+                new_open_rc = Some(Arc::new(new_open))
+            }
+        }
+
+        Ok((frozen_layers, new_open_rc))
    }

    /// debugging function to print out the contents of the layer
@@ -473,16 +561,14 @@ impl InMemoryLayer {
        for (k, v) in inner.segsizes.iter() {
            result += &format!("{}: {}\n", k, v);
        }
-        for (page_num, page_history) in inner.pages.iter() {
-            for (lsn, image) in page_history.iter() {
-                result += &format!(
-                    "blk {} at {}: {}/{}\n",
-                    page_num,
-                    lsn,
-                    image.page_image.is_some(),
-                    image.record.is_some()
-                );
-            }
+        for (k, v) in inner.page_versions.iter() {
+            result += &format!(
+                "blk {} at {}: {}/{}\n",
+                k.0,
+                k.1,
+                v.page_image.is_some(),
+                v.record.is_some()
+            );
        }

        result
--- a/pageserver/src/layered_repository/layer_map.rs
+++ b/pageserver/src/layered_repository/layer_map.rs
@@ -1,44 +1,95 @@
 //!
 //! The layer map tracks what layers exist for all the relations in a timeline.
 //!
-//! When the timeline is first accessed, the server lists of all snapshot files
+//! When the timeline is first accessed, the server lists of all layer files
 //! in the timelines/<timelineid> directory, and populates this map with
-//! SnapshotLayers corresponding to each file. When new WAL is received,
-//! we create InMemoryLayers to hold the incoming records. Now and then,
-//! in the checkpoint() function, the in-memory layers are frozen, forming
-//! new snapshot layers and corresponding files are written to disk.
+//! ImageLayer and DeltaLayer structs corresponding to each file. When new WAL
+//! is received, we create InMemoryLayers to hold the incoming records. Now and
+//! then, in the checkpoint() function, the in-memory layers are frozen, forming
+//! new image and delta layers and corresponding files are written to disk.
 //!

 use crate::layered_repository::storage_layer::{Layer, SegmentTag};
-use crate::layered_repository::{InMemoryLayer, SnapshotLayer};
+use crate::layered_repository::InMemoryLayer;
 use crate::relish::*;
 use anyhow::Result;
+use lazy_static::lazy_static;
 use log::*;
+use std::cmp::Ordering;
 use std::collections::HashSet;
-use std::collections::{BTreeMap, HashMap};
+use std::collections::{BTreeMap, BinaryHeap, HashMap};
 use std::ops::Bound::Included;
 use std::sync::Arc;
+use zenith_metrics::{register_int_gauge, IntGauge};
 use zenith_utils::lsn::Lsn;

-///
-/// LayerMap tracks what layers exist or a timeline. The last layer that is
-/// open for writes is always an InMemoryLayer, and is tracked separately
-/// because there can be only one for each segment. The older layers,
-/// stored on disk, are kept in a BTreeMap keyed by the layer's start LSN.
-///
-pub struct LayerMap {
-    segs: HashMap<SegmentTag, SegEntry>,
+lazy_static! {
+    static ref NUM_INMEMORY_LAYERS: IntGauge =
+        register_int_gauge!("pageserver_inmemory_layers", "Number of layers in memory")
+            .expect("failed to define a metric");
+    static ref NUM_ONDISK_LAYERS: IntGauge =
+        register_int_gauge!("pageserver_ondisk_layers", "Number of layers on-disk")
+            .expect("failed to define a metric");
 }

+///
+/// LayerMap tracks what layers exist on a timeline.
+///
+pub struct LayerMap {
+    /// All the layers keyed by segment tag
+    segs: HashMap<SegmentTag, SegEntry>,
+
+    /// All in-memory layers, ordered by 'oldest_pending_lsn' of each layer.
+    /// This allows easy access to the in-memory layer that contains the
+    /// oldest WAL record.
+    open_segs: BinaryHeap<OpenSegEntry>,
+}
+
+///
+/// Per-segment entry in the LayerMap.segs hash map
+///
+/// The last layer that is open for writes is always an InMemoryLayer,
+/// and is kept in a separate field, because there can be only one for
+/// each segment. The older layers, stored on disk, are kept in a
+/// BTreeMap keyed by the layer's start LSN.
 struct SegEntry {
    pub open: Option<Arc<InMemoryLayer>>,
-    pub historic: BTreeMap<Lsn, Arc<SnapshotLayer>>,
+    pub historic: BTreeMap<Lsn, Arc<dyn Layer>>,
 }

+/// Entry held LayerMap.open_segs, with boilerplate comparison
+/// routines to implement a min-heap ordered by 'oldest_pending_lsn'
+struct OpenSegEntry {
+    pub oldest_pending_lsn: Lsn,
+    pub layer: Arc<InMemoryLayer>,
+}
+impl Ord for OpenSegEntry {
+    fn cmp(&self, other: &Self) -> Ordering {
+        // BinaryHeap is a max-heap, and we want a min-heap. Reverse the ordering here
+        // to get that.
+        other.oldest_pending_lsn.cmp(&self.oldest_pending_lsn)
+    }
+}
+impl PartialOrd for OpenSegEntry {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        // BinaryHeap is a max-heap, and we want a min-heap. Reverse the ordering here
+        // to get that.
+        other
+            .oldest_pending_lsn
+            .partial_cmp(&self.oldest_pending_lsn)
+    }
+}
+impl PartialEq for OpenSegEntry {
+    fn eq(&self, other: &Self) -> bool {
+        self.oldest_pending_lsn.eq(&other.oldest_pending_lsn)
+    }
+}
+impl Eq for OpenSegEntry {}
+
 impl LayerMap {
    ///
-    /// Look up using the given segment tag and LSN. This differs from a plain
-    /// key-value lookup in that if there is any layer that covers the
+    /// Look up a layer using the given segment tag and LSN. This differs from a
+    /// plain key-value lookup in that if there is any layer that covers the
    /// given LSN, or precedes the given LSN, it is returned. In other words,
    /// you don't need to know the exact start LSN of the layer.
    ///
@@ -88,20 +139,38 @@ impl LayerMap {
            if let Some(_old) = &segentry.open {
                // FIXME: shouldn't exist, but check
            }
-            segentry.open = Some(layer);
+            segentry.open = Some(Arc::clone(&layer));
        } else {
            let segentry = SegEntry {
-                open: Some(layer),
+                open: Some(Arc::clone(&layer)),
                historic: BTreeMap::new(),
            };
            self.segs.insert(tag, segentry);
        }
+
+        let opensegentry = OpenSegEntry {
+            oldest_pending_lsn: layer.get_oldest_pending_lsn(),
+            layer: layer,
+        };
+        self.open_segs.push(opensegentry);
+
+        NUM_INMEMORY_LAYERS.inc();
+    }
+
+    /// Remove the oldest in-memory layer
+    pub fn pop_oldest_open(&mut self) {
+        let opensegentry = self.open_segs.pop().unwrap();
+        let segtag = opensegentry.layer.get_seg_tag();
+
+        let mut segentry = self.segs.get_mut(&segtag).unwrap();
+        segentry.open = None;
+        NUM_INMEMORY_LAYERS.dec();
    }

    ///
    /// Insert an on-disk layer
    ///
-    pub fn insert_historic(&mut self, layer: Arc<SnapshotLayer>) {
+    pub fn insert_historic(&mut self, layer: Arc<dyn Layer>) {
        let tag = layer.get_seg_tag();
        let start_lsn = layer.get_start_lsn();

@@ -117,6 +186,7 @@ impl LayerMap {
            };
            self.segs.insert(tag, segentry);
        }
+        NUM_ONDISK_LAYERS.inc();
    }

    ///
@@ -124,59 +194,83 @@ impl LayerMap {
    ///
    /// This should be called when the corresponding file on disk has been deleted.
    ///
-    pub fn remove_historic(&mut self, layer: &SnapshotLayer) {
+    pub fn remove_historic(&mut self, layer: &dyn Layer) {
        let tag = layer.get_seg_tag();
        let start_lsn = layer.get_start_lsn();

        if let Some(segentry) = self.segs.get_mut(&tag) {
            segentry.historic.remove(&start_lsn);
        }
+        NUM_ONDISK_LAYERS.dec();
    }

-    pub fn list_rels(&self, spcnode: u32, dbnode: u32) -> Result<HashSet<RelTag>> {
+    // List relations that exist at the lsn
+    pub fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashSet<RelTag>> {
        let mut rels: HashSet<RelTag> = HashSet::new();

-        for (seg, _entry) in self.segs.iter() {
+        for (seg, segentry) in self.segs.iter() {
            if let RelishTag::Relation(reltag) = seg.rel {
-                // FIXME: skip if it was dropped before the requested LSN. But there is no
-                // LSN argument
-
                if (spcnode == 0 || reltag.spcnode == spcnode)
                    && (dbnode == 0 || reltag.dbnode == dbnode)
                {
-                    rels.insert(reltag);
+                    // Add only if it exists at the requested LSN.
+                    if let Some(open) = &segentry.open {
+                        if open.get_end_lsn() > lsn {
+                            rels.insert(reltag);
+                        }
+                    } else if let Some((_k, _v)) = segentry
+                        .historic
+                        .range((Included(Lsn(0)), Included(lsn)))
+                        .next_back()
+                    {
+                        rels.insert(reltag);
+                    }
                }
            }
        }
        Ok(rels)
    }

-    pub fn list_nonrels(&self, _lsn: Lsn) -> Result<HashSet<RelishTag>> {
+    // List non-relation relishes that exist at the lsn
+    pub fn list_nonrels(&self, lsn: Lsn) -> Result<HashSet<RelishTag>> {
        let mut rels: HashSet<RelishTag> = HashSet::new();

        // Scan the timeline directory to get all rels in this timeline.
-        for (seg, _entry) in self.segs.iter() {
-            // FIXME: skip if it was dropped before the requested LSN.
-
+        for (seg, segentry) in self.segs.iter() {
            if let RelishTag::Relation(_) = seg.rel {
            } else {
-                rels.insert(seg.rel);
+                // Add only if it exists at the requested LSN.
+                if let Some(open) = &segentry.open {
+                    if open.get_end_lsn() > lsn {
+                        rels.insert(seg.rel);
+                    }
+                } else if let Some((_k, _v)) = segentry
+                    .historic
+                    .range((Included(Lsn(0)), Included(lsn)))
+                    .next_back()
+                {
+                    rels.insert(seg.rel);
+                }
            }
        }
        Ok(rels)
    }

-    /// Is there a newer layer for given segment?
-    pub fn newer_layer_exists(&self, seg: SegmentTag, lsn: Lsn) -> bool {
+    /// Is there a newer image layer for given segment?
+    ///
+    /// This is used for garbage collection, to determine if an old layer can
+    /// be deleted. We ignore in-memory layers because they are not durable
+    /// on disk, and delta layers because they depend on an older layer.
+    pub fn newer_image_layer_exists(&self, seg: SegmentTag, lsn: Lsn) -> bool {
        if let Some(segentry) = self.segs.get(&seg) {
-            if let Some(_open) = &segentry.open {
-                return true;
-            }
-
            for (newer_lsn, layer) in segentry
                .historic
                .range((Included(lsn), Included(Lsn(u64::MAX))))
            {
+                // Ignore delta layers.
+                if layer.is_incremental() {
+                    continue;
+                }
                if layer.get_end_lsn() > lsn {
                    trace!(
                        "found later layer for {}, {} {}-{}",
@@ -196,10 +290,12 @@ impl LayerMap {
        false
    }

-    pub fn iter_open_layers(&mut self) -> OpenLayerIter {
-        OpenLayerIter {
-            last: None,
-            segiter: self.segs.iter_mut(),
+    /// Return the oldest in-memory layer.
+    pub fn peek_oldest_open(&self) -> Option<Arc<InMemoryLayer>> {
+        if let Some(opensegentry) = self.open_segs.peek() {
+            Some(Arc::clone(&opensegentry.layer))
+        } else {
+            None
        }
    }

@@ -215,53 +311,18 @@ impl Default for LayerMap {
    fn default() -> Self {
        LayerMap {
            segs: HashMap::new(),
+            open_segs: BinaryHeap::new(),
        }
    }
 }

-pub struct OpenLayerIter<'a> {
-    last: Option<&'a mut SegEntry>,
-
-    segiter: std::collections::hash_map::IterMut<'a, SegmentTag, SegEntry>,
-}
-
-impl<'a> OpenLayerIter<'a> {
-    pub fn replace(&mut self, replacement: Option<Arc<InMemoryLayer>>) {
-        let segentry = self.last.as_mut().unwrap();
-        segentry.open = replacement;
-    }
-
-    pub fn insert_historic(&mut self, new_layer: Arc<SnapshotLayer>) {
-        let start_lsn = new_layer.get_start_lsn();
-
-        let segentry = self.last.as_mut().unwrap();
-        segentry.historic.insert(start_lsn, new_layer);
-    }
-}
-
-impl<'a> Iterator for OpenLayerIter<'a> {
-    type Item = Arc<InMemoryLayer>;
-
-    fn next(&mut self) -> std::option::Option<<Self as std::iter::Iterator>::Item> {
-        while let Some((_seg, entry)) = self.segiter.next() {
-            if let Some(open) = &entry.open {
-                let op = Arc::clone(&open);
-                self.last = Some(entry);
-                return Some(op);
-            }
-        }
-        self.last = None;
-        None
-    }
-}
-
 pub struct HistoricLayerIter<'a> {
    segiter: std::collections::hash_map::Iter<'a, SegmentTag, SegEntry>,
-    iter: Option<std::collections::btree_map::Iter<'a, Lsn, Arc<SnapshotLayer>>>,
+    iter: Option<std::collections::btree_map::Iter<'a, Lsn, Arc<dyn Layer>>>,
 }

 impl<'a> Iterator for HistoricLayerIter<'a> {
-    type Item = Arc<SnapshotLayer>;
+    type Item = Arc<dyn Layer>;

    fn next(&mut self) -> std::option::Option<<Self as std::iter::Iterator>::Item> {
        loop {
--- a/pageserver/src/layered_repository/page_history.rs
+++ b/pageserver/src/layered_repository/page_history.rs
@@ -1,94 +0,0 @@
-use super::storage_layer::PageVersion;
-use std::collections::VecDeque;
-use zenith_utils::lsn::Lsn;
-
-/// A data structure that holds one or more versions of a particular page number.
-//
-#[derive(Default, Clone)]
-pub struct PageHistory {
-    /// Pages stored in order, from oldest to newest.
-    pages: VecDeque<(Lsn, PageVersion)>,
-}
-
-impl PageHistory {
-    /// Create a new PageHistory containing a single image.
-    pub fn from_image(lsn: Lsn, image: PageVersion) -> Self {
-        let mut pages = VecDeque::new();
-        pages.push_back((lsn, image));
-        PageHistory { pages }
-    }
-
-    /// Push a newer page image.
-    pub fn push(&mut self, lsn: Lsn, page: PageVersion) {
-        if let Some((back_lsn, _)) = self.pages.back() {
-            debug_assert_ne!(
-                back_lsn, &lsn,
-                "push page at lsn {:?} but one already exists",
-                lsn
-            );
-            debug_assert!(back_lsn < &lsn, "pushed page is older than latest lsn");
-        }
-        self.pages.push_back((lsn, page));
-    }
-
-    pub fn latest(&self) -> Option<(Lsn, &PageVersion)> {
-        self.pages.back().map(|(lsn, page)| (*lsn, page))
-    }
-
-    /// Split a page history at a particular LSN.
-    ///
-    /// This consumes this PageHistory and returns two new ones.
-    /// Any changes exactly matching the split LSN will be in the
-    /// "old" history.
-    //
-    // FIXME: Is this necessary? There is some debate whether "splitting"
-    // layers is the best design.
-    //
-    pub fn split_at(self, split_lsn: Lsn) -> (PageHistory, PageHistory) {
-        let mut old = PageHistory::default();
-        let mut new = PageHistory::default();
-        for (lsn, page) in self.pages {
-            if lsn > split_lsn {
-                new.push(lsn, page)
-            } else {
-                old.push(lsn, page);
-            }
-        }
-        (old, new)
-    }
-
-    pub fn iter(&self) -> impl Iterator<Item = &(Lsn, PageVersion)> {
-        self.pages.iter()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn page_history() {
-        fn make_page(b: u8) -> PageVersion {
-            let image = vec![b; 8192].into();
-            PageVersion {
-                page_image: Some(image),
-                record: None,
-            }
-        }
-
-        let mut ph = PageHistory::default();
-        ph.push(10.into(), make_page(1));
-        ph.push(20.into(), make_page(2));
-        ph.push(30.into(), make_page(3));
-
-        let (latest_lsn, latest_image) = ph.latest().unwrap();
-        assert_eq!(latest_lsn, 30.into());
-        assert!(matches!(latest_image, PageVersion { page_image: Some(im), .. } if im[0] == 3));
-
-        let mut it = ph.iter();
-        assert_eq!(it.next().unwrap().0, 10.into());
-        assert_eq!(it.next().unwrap().0, 20.into());
-        assert_eq!(it.next().unwrap().0, 30.into());
-        assert!(it.next().is_none());
-    }
-}
--- a/pageserver/src/layered_repository/snapshot_layer.rs
+++ b/pageserver/src/layered_repository/snapshot_layer.rs
@@ -1,567 +0,0 @@
-//!
-//! A SnapshotLayer represents one snapshot file on disk. One file holds all page
-//! version and size information of one relation, in a range of LSN.
-//! The name "snapshot file" is a bit of a misnomer because a snapshot file doesn't
-//! contain a snapshot at a specific LSN, but rather all the page versions in a range
-//! of LSNs.
-//!
-//! Currently, a snapshot file contains full information needed to reconstruct any
-//! page version in the LSN range, without consulting any other snapshot files. When
-//! a new snapshot file is created for writing, the full contents of relation are
-//! materialized as it is at the beginning of the LSN range. That can be very expensive,
-//! we should find a way to store differential files. But this keeps the read-side
-//! of things simple. You can find the correct snapshot file based on RelishTag and
-//! timeline+LSN, and once you've located it, you have all the data you need to in that
-//! file.
-//!
-//! When a snapshot file needs to be accessed, we slurp the whole file into memory, into
-//! the SnapshotLayer struct. See load() and unload() functions.
-//!
-//! On disk, the snapshot files are stored in timelines/<timelineid> directory.
-//! Currently, there are no subdirectories, and each snapshot file is named like this:
-//!
-//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<start LSN>_<end LSN>
-//!
-//! For example:
-//!
-//!    1663_13990_2609_0_000000000169C348_000000000169C349
-//!
-//! If a relation is dropped, we add a '_DROPPED' to the end of the filename to indicate that.
-//! So the above example would become:
-//!
-//!    1663_13990_2609_0_000000000169C348_000000000169C349_DROPPED
-//!
-//! The end LSN indicates when it was dropped in that case, we don't store it in the
-//! file contents in any way.
-//!
-//! A snapshot file is constructed using the 'bookfile' crate. Each file consists of two
-//! parts: the page versions and the relation sizes. They are stored as separate chapters.
-//!
-use crate::layered_repository::page_history::PageHistory;
-use crate::layered_repository::storage_layer::{
-    Layer, PageReconstructData, PageVersion, SegmentTag,
-};
-use crate::relish::*;
-use crate::PageServerConf;
-use crate::{ZTenantId, ZTimelineId};
-use anyhow::{bail, Result};
-use log::*;
-use std::collections::BTreeMap;
-use std::fmt;
-use std::fs;
-use std::fs::File;
-use std::io::Write;
-use std::ops::Bound::Included;
-use std::path::PathBuf;
-use std::sync::{Arc, Mutex, MutexGuard};
-
-use bookfile::{Book, BookWriter};
-
-use zenith_utils::bin_ser::BeSer;
-use zenith_utils::lsn::Lsn;
-
-// Magic constant to identify a Zenith snapshot file
-static SNAPSHOT_FILE_MAGIC: u32 = 0x5A616E01;
-
-static PAGE_VERSIONS_CHAPTER: u64 = 1;
-static REL_SIZES_CHAPTER: u64 = 2;
-
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
-struct SnapshotFileName {
-    seg: SegmentTag,
-    start_lsn: Lsn,
-    end_lsn: Lsn,
-    dropped: bool,
-}
-
-impl SnapshotFileName {
-    fn from_str(fname: &str) -> Option<Self> {
-        // Split the filename into parts
-        //
-        //    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>
-        //
-        // or if it was dropped:
-        //
-        //    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>_DROPPED
-        //
-        let rel;
-        let mut parts;
-        if let Some(rest) = fname.strip_prefix("rel_") {
-            parts = rest.split('_');
-            rel = RelishTag::Relation(RelTag {
-                spcnode: parts.next()?.parse::<u32>().ok()?,
-                dbnode: parts.next()?.parse::<u32>().ok()?,
-                relnode: parts.next()?.parse::<u32>().ok()?,
-                forknum: parts.next()?.parse::<u8>().ok()?,
-            });
-        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::Clog,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::MultiXactMembers,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::MultiXactOffsets,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
-            parts = rest.split('_');
-            rel = RelishTag::FileNodeMap {
-                spcnode: parts.next()?.parse::<u32>().ok()?,
-                dbnode: parts.next()?.parse::<u32>().ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
-            parts = rest.split('_');
-            rel = RelishTag::TwoPhase {
-                xid: parts.next()?.parse::<u32>().ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
-            parts = rest.split('_');
-            rel = RelishTag::Checkpoint;
-        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
-            parts = rest.split('_');
-            rel = RelishTag::ControlFile;
-        } else {
-            return None;
-        }
-
-        let segno = parts.next()?.parse::<u32>().ok()?;
-
-        let seg = SegmentTag { rel, segno };
-
-        let start_lsn = Lsn::from_hex(parts.next()?).ok()?;
-        let end_lsn = Lsn::from_hex(parts.next()?).ok()?;
-
-        let mut dropped = false;
-        if let Some(suffix) = parts.next() {
-            if suffix == "DROPPED" {
-                dropped = true;
-            } else {
-                warn!("unrecognized filename in timeline dir: {}", fname);
-                return None;
-            }
-        }
-        if parts.next().is_some() {
-            warn!("unrecognized filename in timeline dir: {}", fname);
-            return None;
-        }
-
-        Some(SnapshotFileName {
-            seg,
-            start_lsn,
-            end_lsn,
-            dropped,
-        })
-    }
-
-    fn to_string(&self) -> String {
-        let basename = match self.seg.rel {
-            RelishTag::Relation(reltag) => format!(
-                "rel_{}_{}_{}_{}",
-                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
-            ),
-            RelishTag::Slru {
-                slru: SlruKind::Clog,
-                segno,
-            } => format!("pg_xact_{:04X}", segno),
-            RelishTag::Slru {
-                slru: SlruKind::MultiXactMembers,
-                segno,
-            } => format!("pg_multixact_members_{:04X}", segno),
-            RelishTag::Slru {
-                slru: SlruKind::MultiXactOffsets,
-                segno,
-            } => format!("pg_multixact_offsets_{:04X}", segno),
-            RelishTag::FileNodeMap { spcnode, dbnode } => {
-                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
-            }
-            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
-            RelishTag::Checkpoint => format!("pg_control_checkpoint"),
-            RelishTag::ControlFile => format!("pg_control"),
-        };
-
-        format!(
-            "{}_{}_{:016X}_{:016X}{}",
-            basename,
-            self.seg.segno,
-            u64::from(self.start_lsn),
-            u64::from(self.end_lsn),
-            if self.dropped { "_DROPPED" } else { "" }
-        )
-    }
-}
-
-impl fmt::Display for SnapshotFileName {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}", self.to_string())
-    }
-}
-
-///
-/// SnapshotLayer is the in-memory data structure associated with an
-/// on-disk snapshot file.  We keep a SnapshotLayer in memory for each
-/// file, in the LayerMap. If a layer is in "loaded" state, we have a
-/// copy of the file in memory, in 'inner'. Otherwise the struct is
-/// just a placeholder for a file that exists on disk, and it needs to
-/// be loaded before using it in queries.
-///
-pub struct SnapshotLayer {
-    conf: &'static PageServerConf,
-    pub tenantid: ZTenantId,
-    pub timelineid: ZTimelineId,
-    pub seg: SegmentTag,
-
-    //
-    // This entry contains all the changes from 'start_lsn' to 'end_lsn'. The
-    // start is inclusive, and end is exclusive.
-    pub start_lsn: Lsn,
-    pub end_lsn: Lsn,
-
-    dropped: bool,
-
-    inner: Mutex<SnapshotLayerInner>,
-}
-
-pub struct SnapshotLayerInner {
-    /// If false, the 'page_versions' and 'relsizes' have not been
-    /// loaded into memory yet.
-    loaded: bool,
-
-    /// All versions of all pages in the file are are kept here.
-    /// Indexed by block number and LSN.
-    pages: BTreeMap<u32, PageHistory>,
-
-    /// `relsizes` tracks the size of the relation at different points in time.
-    relsizes: BTreeMap<Lsn, u32>,
-}
-
-impl Layer for SnapshotLayer {
-    fn get_timeline_id(&self) -> ZTimelineId {
-        return self.timelineid;
-    }
-
-    fn get_seg_tag(&self) -> SegmentTag {
-        return self.seg;
-    }
-
-    fn is_dropped(&self) -> bool {
-        return self.dropped;
-    }
-
-    fn get_start_lsn(&self) -> Lsn {
-        return self.start_lsn;
-    }
-
-    fn get_end_lsn(&self) -> Lsn {
-        return self.end_lsn;
-    }
-
-    /// Look up given page in the cache.
-    fn get_page_reconstruct_data(
-        &self,
-        blknum: u32,
-        lsn: Lsn,
-        reconstruct_data: &mut PageReconstructData,
-    ) -> Result<Option<Lsn>> {
-        /*
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let mut need_base_image_lsn: Option<Lsn> = Some(lsn);
-        {
-            let inner = self.load()?;
-            let minkey = (blknum, Lsn(0));
-            let maxkey = (blknum, lsn);
-            let mut iter = inner
-                .page_versions
-                .range((Included(&minkey), Included(&maxkey)));
-            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
-                if let Some(img) = &entry.page_image {
-                    reconstruct_data.page_img = Some(img.clone());
-                    need_base_image_lsn = None;
-                    break;
-                } else if let Some(rec) = &entry.record {
-                    reconstruct_data.records.push(rec.clone());
-                    if rec.will_init {
-                        // This WAL record initializes the page, so no need to go further back
-                        need_base_image_lsn = None;
-                        break;
-                    } else {
-                        need_base_image_lsn = Some(*entry_lsn);
-                    }
-                } else {
-                    // No base image, and no WAL record. Huh?
-                    bail!("no page image or WAL record for requested page");
-                }
-            }
-
-            // release lock on 'inner'
-        }
-
-        Ok(need_base_image_lsn)
-
-        */
-        todo!()
-    }
-
-    /// Get size of the relation at given LSN
-    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let inner = self.load()?;
-        let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn)));
-
-        if let Some((_entry_lsn, entry)) = iter.next_back() {
-            let result = *entry;
-            drop(inner);
-            trace!("get_seg_size: {} at {} -> {}", self.seg, lsn, result);
-            Ok(result)
-        } else {
-            error!(
-                "No size found for {} at {} in snapshot layer {} {}-{}",
-                self.seg, lsn, self.seg, self.start_lsn, self.end_lsn
-            );
-            bail!(
-                "No size found for {} at {} in snapshot layer",
-                self.seg,
-                lsn
-            );
-        }
-    }
-
-    /// Does this segment exist at given LSN?
-    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
-        // Is the requested LSN after the rel was dropped?
-        if self.dropped && lsn >= self.end_lsn {
-            return Ok(false);
-        }
-
-        // Otherwise, it exists.
-        Ok(true)
-    }
-}
-
-impl SnapshotLayer {
-    fn path(&self) -> PathBuf {
-        Self::path_for(
-            self.conf,
-            self.timelineid,
-            self.tenantid,
-            &SnapshotFileName {
-                seg: self.seg,
-                start_lsn: self.start_lsn,
-                end_lsn: self.end_lsn,
-                dropped: self.dropped,
-            },
-        )
-    }
-
-    fn path_for(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        fname: &SnapshotFileName,
-    ) -> PathBuf {
-        conf.timeline_path(&timelineid, &tenantid)
-            .join(fname.to_string())
-    }
-
-    /// Create a new snapshot file, using the given btreemaps containing the page versions and
-    /// relsizes.
-    ///
-    /// This is used to write the in-memory layer to disk. The in-memory layer uses the same
-    /// data structure with two btreemaps as we do, so passing the btreemaps is currently
-    /// expedient.
-    pub fn create(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        seg: SegmentTag,
-        start_lsn: Lsn,
-        end_lsn: Lsn,
-        dropped: bool,
-        pages: BTreeMap<u32, PageHistory>,
-        relsizes: BTreeMap<Lsn, u32>,
-    ) -> Result<SnapshotLayer> {
-        let snapfile = SnapshotLayer {
-            conf: conf,
-            timelineid: timelineid,
-            tenantid: tenantid,
-            seg: seg,
-            start_lsn: start_lsn,
-            end_lsn,
-            dropped,
-            inner: Mutex::new(SnapshotLayerInner {
-                loaded: true,
-                pages,
-                relsizes,
-            }),
-        };
-
-        /*
-        let inner = snapfile.inner.lock().unwrap();
-
-        // Write the in-memory btreemaps into a file
-        let path = snapfile.path();
-
-        // Note: This overwrites any existing file. There shouldn't be any.
-        // FIXME: throw an error instead?
-        let file = File::create(&path)?;
-        let book = BookWriter::new(file, SNAPSHOT_FILE_MAGIC)?;
-
-        // Write out page versions
-        let mut chapter = book.new_chapter(PAGE_VERSIONS_CHAPTER);
-        let buf = BTreeMap::ser(&inner.page_versions)?;
-        chapter.write_all(&buf)?;
-        let book = chapter.close()?;
-
-        // and relsizes to separate chapter
-        let mut chapter = book.new_chapter(REL_SIZES_CHAPTER);
-        let buf = BTreeMap::ser(&inner.relsizes)?;
-        chapter.write_all(&buf)?;
-        let book = chapter.close()?;
-
-        book.close()?;
-
-        trace!("saved {}", &path.display());
-
-        drop(inner);
-
-        Ok(snapfile)
-        */
-
-        todo!()
-    }
-
-    ///
-    /// Load the contents of the file into memory
-    ///
-    fn load(&self) -> Result<MutexGuard<SnapshotLayerInner>> {
-        /*
-        // quick exit if already loaded
-        let mut inner = self.inner.lock().unwrap();
-
-        if inner.loaded {
-            return Ok(inner);
-        }
-
-        let path = Self::path_for(
-            self.conf,
-            self.timelineid,
-            self.tenantid,
-            &SnapshotFileName {
-                seg: self.seg,
-                start_lsn: self.start_lsn,
-                end_lsn: self.end_lsn,
-                dropped: self.dropped,
-            },
-        );
-
-        let file = File::open(&path)?;
-        let book = Book::new(file)?;
-
-        let chapter = book.read_chapter(PAGE_VERSIONS_CHAPTER)?;
-        let page_versions = BTreeMap::des(&chapter)?;
-
-        let chapter = book.read_chapter(REL_SIZES_CHAPTER)?;
-        let relsizes = BTreeMap::des(&chapter)?;
-
-        debug!("loaded from {}", &path.display());
-
-        *inner = SnapshotLayerInner {
-            loaded: true,
-            page_versions,
-            relsizes,
-        };
-
-        Ok(inner)
-        */
-
-        todo!()
-    }
-
-    /// Create SnapshotLayers representing all files on disk
-    ///
-    // TODO: returning an Iterator would be more idiomatic
-    pub fn list_snapshot_files(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-    ) -> Result<Vec<Arc<SnapshotLayer>>> {
-        /*
-        let path = conf.timeline_path(&timelineid, &tenantid);
-
-        let mut snapfiles: Vec<Arc<SnapshotLayer>> = Vec::new();
-        for direntry in fs::read_dir(path)? {
-            let fname = direntry?.file_name();
-            let fname = fname.to_str().unwrap();
-
-            if let Some(snapfilename) = SnapshotFileName::from_str(fname) {
-                let snapfile = SnapshotLayer {
-                    conf,
-                    timelineid,
-                    tenantid,
-                    seg: snapfilename.seg,
-                    start_lsn: snapfilename.start_lsn,
-                    end_lsn: snapfilename.end_lsn,
-                    dropped: snapfilename.dropped,
-                    inner: Mutex::new(SnapshotLayerInner {
-                        loaded: false,
-                        page_versions: BTreeMap::new(),
-                        relsizes: BTreeMap::new(),
-                    }),
-                };
-
-                snapfiles.push(Arc::new(snapfile));
-            }
-        }
-        return Ok(snapfiles);
-        */
-        todo!()
-    }
-
-    pub fn delete(&self) -> Result<()> {
-        // delete underlying file
-        fs::remove_file(self.path())?;
-        Ok(())
-    }
-
-    ///
-    /// Release most of the memory used by this layer. If it's accessed again later,
-    /// it will need to be loaded back.
-    ///
-    pub fn unload(&self) -> Result<()> {
-        /*
-        let mut inner = self.inner.lock().unwrap();
-        inner.page_versions = BTreeMap::new();
-        inner.relsizes = BTreeMap::new();
-        inner.loaded = false;
-        Ok(())
-        */
-        todo!()
-    }
-
-    /// debugging function to print out the contents of the layer
-    #[allow(unused)]
-    pub fn dump(&self) -> String {
-        let mut result = format!(
-            "----- snapshot layer for {} {}-{} ----\n",
-            self.seg, self.start_lsn, self.end_lsn
-        );
-
-        let inner = self.inner.lock().unwrap();
-        for (k, v) in inner.relsizes.iter() {
-            result += &format!("{}: {}\n", k, v);
-        }
-        //for (k, v) in inner.page_versions.iter() {
-        //    result += &format!("blk {} at {}: {}/{}\n", k.0, k.1, v.page_image.is_some(), v.record.is_some());
-        //}
-
-        result
-    }
-}
--- a/pageserver/src/layered_repository/storage_layer.rs
+++ b/pageserver/src/layered_repository/storage_layer.rs
@@ -9,6 +9,7 @@ use anyhow::Result;
 use bytes::Bytes;
 use serde::{Deserialize, Serialize};
 use std::fmt;
+use std::path::PathBuf;

 use zenith_utils::lsn::Lsn;

@@ -102,6 +103,11 @@ pub trait Layer: Send + Sync {
    fn get_end_lsn(&self) -> Lsn;
    fn is_dropped(&self) -> bool;

+    /// Filename used to store this layer on disk. (Even in-memory layers
+    /// implement this, to print a handy unique identifier for the layer for
+    /// log messages, even though they're never not on disk.)
+    fn filename(&self) -> PathBuf;
+
    ///
    /// Return data needed to reconstruct given page at LSN.
    ///
@@ -121,8 +127,22 @@ pub trait Layer: Send + Sync {
        reconstruct_data: &mut PageReconstructData,
    ) -> Result<Option<Lsn>>;

-    // Functions that correspond to the Timeline trait functions.
+    /// Return size of the segment at given LSN. (Only for blocky relations.)
    fn get_seg_size(&self, lsn: Lsn) -> Result<u32>;

+    /// Does the segment exist at given LSN? Or was it dropped before it.
    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool>;
+
+    /// Does this layer only contain some data for the segment (incremental),
+    /// or does it contain a version of every page? This is important to know
+    /// for garbage collecting old layers: an incremental layer depends on
+    /// the previous non-incremental layer.
+    fn is_incremental(&self) -> bool;
+
+    /// Release memory used by this layer. There is no corresponding 'load'
+    /// function, that's done implicitly when you call one of the get-functions.
+    fn unload(&self) -> Result<()>;
+
+    /// Permanently remove this layer from disk.
+    fn delete(&self) -> Result<()>;
 }
--- a/pageserver/src/lib.rs
+++ b/pageserver/src/lib.rs
@@ -9,17 +9,14 @@ use zenith_metrics::{register_int_gauge_vec, IntGaugeVec};

 pub mod basebackup;
 pub mod branches;
+pub mod http;
 pub mod layered_repository;
 pub mod logger;
-pub mod object_key;
-pub mod object_repository;
-pub mod object_store;
-pub mod page_cache;
 pub mod page_service;
 pub mod relish;
 pub mod repository;
 pub mod restore_local_repo;
-pub mod rocksdb_storage;
+pub mod tenant_mgr;
 pub mod waldecoder;
 pub mod walreceiver;
 pub mod walredo;
@@ -55,14 +52,6 @@ pub struct PageServerConf {
    pub auth_type: AuthType,

    pub auth_validation_public_key_path: Option<PathBuf>,
-
-    pub repository_format: RepositoryFormat,
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub enum RepositoryFormat {
-    Layered,
-    RocksDb,
 }

 impl PageServerConf {
--- a/pageserver/src/object_key.rs
+++ b/pageserver/src/object_key.rs
@@ -1,49 +0,0 @@
-//!
-//! Common structs shared by object_repository.rs and object_store.rs.
-//!
-
-use crate::relish::RelishTag;
-use serde::{Deserialize, Serialize};
-use zenith_utils::zid::ZTimelineId;
-
-///
-/// ObjectKey is the key type used to identify objects stored in an object
-/// repository. It is shared between object_repository.rs and object_store.rs.
-/// It is mostly opaque to ObjectStore, it just stores and retrieves objects
-/// using the key given by the caller.
-///
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ObjectKey {
-    pub timeline: ZTimelineId,
-    pub tag: ObjectTag,
-}
-
-///
-/// ObjectTag is a part of ObjectKey that is specific to the type of
-/// the stored object.
-///
-/// NB: the order of the enum values is significant!  In particular,
-/// rocksdb_storage.rs assumes that TimelineMetadataTag is first
-///
-/// Buffer is the kind of object that is accessible by the public
-/// get_page_at_lsn() / put_page_image() / put_wal_record() functions in
-/// the repository.rs interface. The rest are internal objects stored in
-/// the key-value store, to store various metadata. They're not directly
-/// accessible outside object_repository.rs
-///
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
-pub enum ObjectTag {
-    // dummy tag preceeding all other keys
-    FirstTag,
-
-    // Metadata about a timeline. Not versioned.
-    TimelineMetadataTag,
-
-    // These objects store metadata about one relish. Currently it's used
-    // just to track the relish's size. It's not used for non-blocky relishes
-    // at all.
-    RelationMetadata(RelishTag),
-
-    // These are the pages exposed in the public Repository/Timeline interface.
-    Buffer(RelishTag, u32),
-}
--- a/pageserver/src/object_repository.rs
+++ b/pageserver/src/object_repository.rs
--- a/pageserver/src/object_store.rs
+++ b/pageserver/src/object_store.rs
@@ -1,92 +0,0 @@
-//! Low-level key-value storage abstraction.
-//!
-use crate::object_key::*;
-use crate::relish::*;
-use anyhow::Result;
-use std::collections::HashSet;
-use std::iter::Iterator;
-use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::ZTimelineId;
-
-///
-/// Low-level storage abstraction.
-///
-/// All the data in the repository is stored in a key-value store. This trait
-/// abstracts the details of the key-value store.
-///
-/// A simple key-value store would support just GET and PUT operations with
-/// a key, but the upper layer needs slightly complicated read operations
-///
-/// The most frequently used function is 'object_versions'. It is used
-/// to look up a page version. It is LSN aware, in that the caller
-/// specifies an LSN, and the function returns all values for that
-/// block with the same or older LSN.
-///
-pub trait ObjectStore: Send + Sync {
-    ///
-    /// Store a value with given key.
-    ///
-    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()>;
-
-    /// Read entry with the exact given key.
-    ///
-    /// This is used for retrieving metadata with special key that doesn't
-    /// correspond to any real relation.
-    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>>;
-
-    /// Read key greater or equal than specified
-    fn get_next_key(&self, key: &ObjectKey) -> Result<Option<ObjectKey>>;
-
-    /// Iterate through all page versions of one object.
-    ///
-    /// Returns all page versions in descending LSN order, along with the LSN
-    /// of each page version.
-    fn object_versions<'a>(
-        &'a self,
-        key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>>;
-
-    /// Iterate through versions of all objects in a timeline.
-    ///
-    /// Returns objects in increasing key-version order.
-    /// Returns all versions up to and including the specified LSN.
-    fn objects<'a>(
-        &'a self,
-        timeline: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = Result<(ObjectTag, Lsn, Vec<u8>)>> + 'a>>;
-
-    /// Iterate through all keys with given tablespace and database ID, and LSN <= 'lsn'.
-    /// Both dbnode and spcnode can be InvalidId (0) which means get all relations in tablespace/cluster
-    ///
-    /// This is used to implement 'create database'
-    fn list_rels(
-        &self,
-        timelineid: ZTimelineId,
-        spcnode: u32,
-        dbnode: u32,
-        lsn: Lsn,
-    ) -> Result<HashSet<RelTag>>;
-
-    /// Iterate through non-rel relishes
-    ///
-    /// This is used to prepare tarball for new node startup.
-    fn list_nonrels<'a>(&'a self, timelineid: ZTimelineId, lsn: Lsn) -> Result<HashSet<RelishTag>>;
-
-    /// Iterate through objects tags. If nonrel_only, then only non-relationa data is iterated.
-    ///
-    /// This is used to implement GC and preparing tarball for new node startup
-    /// Returns objects in increasing key-version order.
-    fn list_objects<'a>(
-        &'a self,
-        timelineid: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = ObjectTag> + 'a>>;
-
-    /// Unlink object (used by GC). This mehod may actually delete object or just mark it for deletion.
-    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()>;
-
-    // Compact storage and remove versions marged for deletion
-    fn compact(&self);
-}
--- a/pageserver/src/page_service.rs
+++ b/pageserver/src/page_service.rs
@@ -10,12 +10,11 @@
 //     *callmemaybe <zenith timelineid> $url* -- ask pageserver to start walreceiver on $url
 //

-use anyhow::{anyhow, bail, ensure, Result};
+use anyhow::{anyhow, bail, ensure, Context, Result};
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use lazy_static::lazy_static;
 use log::*;
 use regex::Regex;
-use std::io::Write;
 use std::net::TcpListener;
 use std::str;
 use std::str::FromStr;
@@ -23,21 +22,20 @@ use std::sync::Arc;
 use std::thread;
 use std::{io, net::TcpStream};
 use zenith_metrics::{register_histogram_vec, HistogramVec};
-use zenith_utils::auth::JwtAuth;
+use zenith_utils::auth::{self, JwtAuth};
 use zenith_utils::auth::{Claims, Scope};
+use zenith_utils::lsn::Lsn;
 use zenith_utils::postgres_backend::PostgresBackend;
 use zenith_utils::postgres_backend::{self, AuthType};
 use zenith_utils::pq_proto::{
    BeMessage, FeMessage, RowDescriptor, HELLO_WORLD_ROW, SINGLE_COL_ROWDESC,
 };
 use zenith_utils::zid::{ZTenantId, ZTimelineId};
-use zenith_utils::{bin_ser::BeSer, lsn::Lsn};

 use crate::basebackup;
 use crate::branches;
-use crate::page_cache;
 use crate::relish::*;
-use crate::repository::Modification;
+use crate::tenant_mgr;
 use crate::walreceiver;
 use crate::PageServerConf;

@@ -145,7 +143,7 @@ impl PagestreamBeMessage {
 ///
 pub fn thread_main(
    conf: &'static PageServerConf,
-    auth: Arc<Option<JwtAuth>>,
+    auth: Option<Arc<JwtAuth>>,
    listener: TcpListener,
    auth_type: AuthType,
 ) -> anyhow::Result<()> {
@@ -153,7 +151,7 @@ pub fn thread_main(
        let (socket, peer_addr) = listener.accept()?;
        debug!("accepted connection from {}", peer_addr);
        socket.set_nodelay(true).unwrap();
-        let local_auth = Arc::clone(&auth);
+        let local_auth = auth.clone();
        thread::spawn(move || {
            if let Err(err) = page_service_conn_main(conf, local_auth, socket, auth_type) {
                error!("error: {}", err);
@@ -164,7 +162,7 @@ pub fn thread_main(

 fn page_service_conn_main(
    conf: &'static PageServerConf,
-    auth: Arc<Option<JwtAuth>>,
+    auth: Option<Arc<JwtAuth>>,
    socket: TcpStream,
    auth_type: AuthType,
 ) -> anyhow::Result<()> {
@@ -185,7 +183,7 @@ fn page_service_conn_main(
 #[derive(Debug)]
 struct PageServerHandler {
    conf: &'static PageServerConf,
-    auth: Arc<Option<JwtAuth>>,
+    auth: Option<Arc<JwtAuth>>,
    claims: Option<Claims>,
 }

@@ -208,7 +206,7 @@ lazy_static! {
 }

 impl PageServerHandler {
-    pub fn new(conf: &'static PageServerConf, auth: Arc<Option<JwtAuth>>) -> Self {
+    pub fn new(conf: &'static PageServerConf, auth: Option<Arc<JwtAuth>>) -> Self {
        PageServerHandler {
            conf,
            auth,
@@ -231,13 +229,10 @@ impl PageServerHandler {
        tenantid: ZTenantId,
    ) -> anyhow::Result<()> {
        // Check that the timeline exists
-        let repository = page_cache::get_repository_for_tenant(&tenantid)?;
-        let timeline = repository.get_timeline(timelineid).map_err(|_| {
-            anyhow!(
-                "client requested pagestream on timeline {} which does not exist in page server",
-                timelineid
-            )
-        })?;
+        let repository = tenant_mgr::get_repository_for_tenant(&tenantid)?;
+        let timeline = repository
+            .get_timeline(timelineid)
+            .context(format!("error fetching timeline {}", timelineid))?;

        /* switch client to COPYBOTH */
        pgb.write_message(&BeMessage::CopyBothResponse)?;
@@ -343,14 +338,10 @@ impl PageServerHandler {
        tenantid: ZTenantId,
    ) -> anyhow::Result<()> {
        // check that the timeline exists
-        let repository = page_cache::get_repository_for_tenant(&tenantid)?;
-        let timeline = repository.get_timeline(timelineid).map_err(|e| {
-            error!("error fetching timeline: {:?}", e);
-            anyhow!(
-                "client requested basebackup on timeline {} which does not exist in page server",
-                timelineid
-            )
-        })?;
+        let repository = tenant_mgr::get_repository_for_tenant(&tenantid)?;
+        let timeline = repository
+            .get_timeline(timelineid)
+            .context(format!("error fetching timeline {}", timelineid))?;
        /* switch client to COPYOUT */
        pgb.write_message(&BeMessage::CopyOutResponse)?;
        info!("sent CopyOut");
@@ -389,19 +380,7 @@ impl PageServerHandler {
            .claims
            .as_ref()
            .expect("claims presence already checked");
-        match (&claims.scope, tenantid) {
-            (Scope::Tenant, None) => {
-                bail!("Attempt to access management api with tenant scope. Permission denied")
-            }
-            (Scope::Tenant, Some(tenantid)) => {
-                if claims.tenant_id.unwrap() != tenantid {
-                    bail!("Tenant id mismatch. Permission denied")
-                }
-                Ok(())
-            }
-            (Scope::PageServerApi, None) => Ok(()), // access to management api for PageServerApi scope
-            (Scope::PageServerApi, Some(_)) => Ok(()), // access to tenant api using PageServerApi scope
-        }
+        Ok(auth::check_permission(claims, tenantid)?)
    }
 }

@@ -507,10 +486,10 @@ impl postgres_backend::Handler for PageServerHandler {
            self.check_permission(Some(tenantid))?;

            // Check that the timeline exists
-            let repository = page_cache::get_repository_for_tenant(&tenantid)?;
-            if repository.get_timeline(timelineid).is_err() {
-                bail!("client requested callmemaybe on timeline {} which does not exist in page server", timelineid);
-            }
+            let repository = tenant_mgr::get_repository_for_tenant(&tenantid)?;
+            repository
+                .get_timeline(timelineid)
+                .context(format!("error fetching timeline {}", timelineid))?;

            walreceiver::launch_wal_receiver(&self.conf, timelineid, &connstr, tenantid.to_owned());

@@ -538,80 +517,6 @@ impl postgres_backend::Handler for PageServerHandler {
            pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
                .write_message_noflush(&BeMessage::DataRow(&[Some(&branch)]))?
                .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
-        } else if query_string.starts_with("push ") {
-            // push <zenith tenantid as hex string> <zenith timelineid as hex string>
-            let re = Regex::new(r"^push ([[:xdigit:]]+) ([[:xdigit:]]+)$").unwrap();
-
-            let caps = re
-                .captures(query_string)
-                .ok_or_else(|| anyhow!("invalid push: '{}'", query_string))?;
-
-            let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;
-            let timelineid = ZTimelineId::from_str(caps.get(2).unwrap().as_str())?;
-
-            self.check_permission(Some(tenantid))?;
-
-            let start_lsn = Lsn(0); // TODO this needs to come from the repo
-            let timeline = page_cache::get_repository_for_tenant(&tenantid)?
-                .create_empty_timeline(timelineid, start_lsn)?;
-
-            pgb.write_message(&BeMessage::CopyInResponse)?;
-
-            let mut last_lsn = Lsn(0);
-
-            while let Some(msg) = pgb.read_message()? {
-                match msg {
-                    FeMessage::CopyData(bytes) => {
-                        let modification = Modification::des(&bytes)?;
-
-                        last_lsn = modification.lsn;
-                        timeline.put_raw_data(
-                            modification.tag,
-                            modification.lsn,
-                            &modification.data,
-                        )?;
-                    }
-                    FeMessage::CopyDone => {
-                        timeline.advance_last_valid_lsn(last_lsn);
-                        break;
-                    }
-                    FeMessage::Sync => {}
-                    _ => bail!("unexpected message {:?}", msg),
-                }
-            }
-
-            pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
-        } else if query_string.starts_with("request_push ") {
-            // request_push <zenith tenantid as hex string> <zenith timelineid as hex string> <postgres_connection_uri>
-            let re = Regex::new(r"^request_push ([[:xdigit:]]+) ([[:xdigit:]]+) (.*)$").unwrap();
-
-            let caps = re
-                .captures(query_string)
-                .ok_or_else(|| anyhow!("invalid request_push: '{}'", query_string))?;
-
-            let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;
-            let timelineid = ZTimelineId::from_str(caps.get(2).unwrap().as_str())?;
-            let postgres_connection_uri = caps.get(3).unwrap().as_str();
-
-            self.check_permission(Some(tenantid))?;
-
-            let timeline =
-                page_cache::get_repository_for_tenant(&tenantid)?.get_timeline(timelineid)?;
-
-            let mut conn = postgres::Client::connect(postgres_connection_uri, postgres::NoTls)?;
-            let mut copy_in = conn.copy_in(format!("push {}", timelineid.to_string()).as_str())?;
-
-            let history = timeline.history()?;
-            for update_res in history {
-                let update = update_res?;
-                let update_bytes = update.ser()?;
-                copy_in.write_all(&update_bytes)?;
-                copy_in.flush()?; // ensure that messages are sent inside individual CopyData packets
-            }
-
-            copy_in.finish()?;
-
-            pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
        } else if query_string.starts_with("branch_list ") {
            // branch_list <zenith tenantid as hex string>
            let re = Regex::new(r"^branch_list ([[:xdigit:]]+)$").unwrap();
@@ -645,7 +550,7 @@ impl postgres_backend::Handler for PageServerHandler {

            let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;

-            page_cache::create_repository_for_tenant(&self.conf, tenantid)?;
+            tenant_mgr::create_repository_for_tenant(&self.conf, tenantid)?;

            pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
                .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
@@ -679,20 +584,11 @@ impl postgres_backend::Handler for PageServerHandler {
                .map(|h| h.as_str().parse())
                .unwrap_or(Ok(self.conf.gc_horizon))?;

-            let repo = page_cache::get_repository_for_tenant(&tenantid)?;
+            let repo = tenant_mgr::get_repository_for_tenant(&tenantid)?;

            let result = repo.gc_iteration(Some(timelineid), gc_horizon, true)?;

            pgb.write_message_noflush(&BeMessage::RowDescription(&[
-                RowDescriptor::int8_col(b"n_relations"),
-                RowDescriptor::int8_col(b"truncated"),
-                RowDescriptor::int8_col(b"deleted"),
-                RowDescriptor::int8_col(b"prep_deleted"),
-                RowDescriptor::int8_col(b"slru_deleted"),
-                RowDescriptor::int8_col(b"chkp_deleted"),
-                RowDescriptor::int8_col(b"control_deleted"),
-                RowDescriptor::int8_col(b"filenodemap_deleted"),
-                RowDescriptor::int8_col(b"dropped"),
                RowDescriptor::int8_col(b"snapshot_relfiles_total"),
                RowDescriptor::int8_col(b"snapshot_relfiles_needed_by_cutoff"),
                RowDescriptor::int8_col(b"snapshot_relfiles_needed_by_branches"),
@@ -708,52 +604,38 @@ impl postgres_backend::Handler for PageServerHandler {
                RowDescriptor::int8_col(b"elapsed"),
            ]))?
            .write_message_noflush(&BeMessage::DataRow(&[
-                Some(&result.n_relations.to_string().as_bytes()),
-                Some(&result.truncated.to_string().as_bytes()),
-                Some(&result.deleted.to_string().as_bytes()),
-                Some(&result.prep_deleted.to_string().as_bytes()),
-                Some(&result.slru_deleted.to_string().as_bytes()),
-                Some(&result.chkp_deleted.to_string().as_bytes()),
-                Some(&result.control_deleted.to_string().as_bytes()),
-                Some(&result.filenodemap_deleted.to_string().as_bytes()),
-                Some(&result.dropped.to_string().as_bytes()),
-                Some(&result.snapshot_relfiles_total.to_string().as_bytes()),
+                Some(&result.ondisk_relfiles_total.to_string().as_bytes()),
                Some(
                    &result
-                        .snapshot_relfiles_needed_by_cutoff
+                        .ondisk_relfiles_needed_by_cutoff
                        .to_string()
                        .as_bytes(),
                ),
                Some(
                    &result
-                        .snapshot_relfiles_needed_by_branches
+                        .ondisk_relfiles_needed_by_branches
                        .to_string()
                        .as_bytes(),
                ),
-                Some(&result.snapshot_relfiles_not_updated.to_string().as_bytes()),
-                Some(&result.snapshot_relfiles_removed.to_string().as_bytes()),
-                Some(&result.snapshot_relfiles_dropped.to_string().as_bytes()),
-                Some(&result.snapshot_nonrelfiles_total.to_string().as_bytes()),
+                Some(&result.ondisk_relfiles_not_updated.to_string().as_bytes()),
+                Some(&result.ondisk_relfiles_removed.to_string().as_bytes()),
+                Some(&result.ondisk_relfiles_dropped.to_string().as_bytes()),
+                Some(&result.ondisk_nonrelfiles_total.to_string().as_bytes()),
                Some(
                    &result
-                        .snapshot_nonrelfiles_needed_by_cutoff
+                        .ondisk_nonrelfiles_needed_by_cutoff
                        .to_string()
                        .as_bytes(),
                ),
                Some(
                    &result
-                        .snapshot_nonrelfiles_needed_by_branches
+                        .ondisk_nonrelfiles_needed_by_branches
                        .to_string()
                        .as_bytes(),
                ),
-                Some(
-                    &result
-                        .snapshot_nonrelfiles_not_updated
-                        .to_string()
-                        .as_bytes(),
-                ),
-                Some(&result.snapshot_nonrelfiles_removed.to_string().as_bytes()),
-                Some(&result.snapshot_nonrelfiles_dropped.to_string().as_bytes()),
+                Some(&result.ondisk_nonrelfiles_not_updated.to_string().as_bytes()),
+                Some(&result.ondisk_nonrelfiles_removed.to_string().as_bytes()),
+                Some(&result.ondisk_nonrelfiles_dropped.to_string().as_bytes()),
                Some(&result.elapsed.as_millis().to_string().as_bytes()),
            ]))?
            .write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;
--- a/pageserver/src/repository.rs
+++ b/pageserver/src/repository.rs
@@ -1,10 +1,8 @@
-use crate::object_key::*;
 use crate::relish::*;
 use anyhow::Result;
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
-use std::iter::Iterator;
 use std::ops::AddAssign;
 use std::sync::Arc;
 use std::time::Duration;
@@ -57,58 +55,38 @@ pub trait Repository: Send + Sync {
 ///
 #[derive(Default)]
 pub struct GcResult {
-    // FIXME: These counters make sense for the ObjectRepository. They are not used
-    // by the LayeredRepository.
-    pub n_relations: u64,
-    pub inspected: u64,
-    pub truncated: u64,
-    pub deleted: u64,
-    pub prep_deleted: u64,        // RelishTag::Twophase
-    pub slru_deleted: u64,        // RelishTag::Slru
-    pub chkp_deleted: u64,        // RelishTag::Checkpoint
-    pub control_deleted: u64,     // RelishTag::ControlFile
-    pub filenodemap_deleted: u64, // RelishTag::FileNodeMap
-    pub dropped: u64,
+    pub ondisk_relfiles_total: u64,
+    pub ondisk_relfiles_needed_by_cutoff: u64,
+    pub ondisk_relfiles_needed_by_branches: u64,
+    pub ondisk_relfiles_not_updated: u64,
+    pub ondisk_relfiles_removed: u64, // # of layer files removed because they have been made obsolete by newer ondisk files.
+    pub ondisk_relfiles_dropped: u64, // # of layer files removed because the relation was dropped

-    // These are used for the LayeredRepository instead
-    pub snapshot_relfiles_total: u64,
-    pub snapshot_relfiles_needed_by_cutoff: u64,
-    pub snapshot_relfiles_needed_by_branches: u64,
-    pub snapshot_relfiles_not_updated: u64,
-    pub snapshot_relfiles_removed: u64, // # of snapshot files removed because they have been made obsolete by newer snapshot files.
-    pub snapshot_relfiles_dropped: u64, // # of snapshot files removed because the relation was dropped
-
-    pub snapshot_nonrelfiles_total: u64,
-    pub snapshot_nonrelfiles_needed_by_cutoff: u64,
-    pub snapshot_nonrelfiles_needed_by_branches: u64,
-    pub snapshot_nonrelfiles_not_updated: u64,
-    pub snapshot_nonrelfiles_removed: u64, // # of snapshot files removed because they have been made obsolete by newer snapshot files.
-    pub snapshot_nonrelfiles_dropped: u64, // # of snapshot files removed because the relation was dropped
+    pub ondisk_nonrelfiles_total: u64,
+    pub ondisk_nonrelfiles_needed_by_cutoff: u64,
+    pub ondisk_nonrelfiles_needed_by_branches: u64,
+    pub ondisk_nonrelfiles_not_updated: u64,
+    pub ondisk_nonrelfiles_removed: u64, // # of layer files removed because they have been made obsolete by newer ondisk files.
+    pub ondisk_nonrelfiles_dropped: u64, // # of layer files removed because the relation was dropped

    pub elapsed: Duration,
 }

 impl AddAssign for GcResult {
    fn add_assign(&mut self, other: Self) {
-        self.n_relations += other.n_relations;
-        self.truncated += other.truncated;
-        self.deleted += other.deleted;
-        self.dropped += other.dropped;
+        self.ondisk_relfiles_total += other.ondisk_relfiles_total;
+        self.ondisk_relfiles_needed_by_cutoff += other.ondisk_relfiles_needed_by_cutoff;
+        self.ondisk_relfiles_needed_by_branches += other.ondisk_relfiles_needed_by_branches;
+        self.ondisk_relfiles_not_updated += other.ondisk_relfiles_not_updated;
+        self.ondisk_relfiles_removed += other.ondisk_relfiles_removed;
+        self.ondisk_relfiles_dropped += other.ondisk_relfiles_dropped;

-        self.snapshot_relfiles_total += other.snapshot_relfiles_total;
-        self.snapshot_relfiles_needed_by_cutoff += other.snapshot_relfiles_needed_by_cutoff;
-        self.snapshot_relfiles_needed_by_branches += other.snapshot_relfiles_needed_by_branches;
-        self.snapshot_relfiles_not_updated += other.snapshot_relfiles_not_updated;
-        self.snapshot_relfiles_removed += other.snapshot_relfiles_removed;
-        self.snapshot_relfiles_dropped += other.snapshot_relfiles_dropped;
-
-        self.snapshot_nonrelfiles_total += other.snapshot_nonrelfiles_total;
-        self.snapshot_nonrelfiles_needed_by_cutoff += other.snapshot_nonrelfiles_needed_by_cutoff;
-        self.snapshot_nonrelfiles_needed_by_branches +=
-            other.snapshot_nonrelfiles_needed_by_branches;
-        self.snapshot_nonrelfiles_not_updated += other.snapshot_nonrelfiles_not_updated;
-        self.snapshot_nonrelfiles_removed += other.snapshot_nonrelfiles_removed;
-        self.snapshot_nonrelfiles_dropped += other.snapshot_nonrelfiles_dropped;
+        self.ondisk_nonrelfiles_total += other.ondisk_nonrelfiles_total;
+        self.ondisk_nonrelfiles_needed_by_cutoff += other.ondisk_nonrelfiles_needed_by_cutoff;
+        self.ondisk_nonrelfiles_needed_by_branches += other.ondisk_nonrelfiles_needed_by_branches;
+        self.ondisk_nonrelfiles_not_updated += other.ondisk_nonrelfiles_not_updated;
+        self.ondisk_nonrelfiles_removed += other.ondisk_nonrelfiles_removed;
+        self.ondisk_nonrelfiles_dropped += other.ondisk_nonrelfiles_dropped;

        self.elapsed += other.elapsed;
    }
@@ -150,14 +128,7 @@ pub trait Timeline: Send + Sync {
    fn put_wal_record(&self, tag: RelishTag, blknum: u32, rec: WALRecord) -> Result<()>;

    /// Like put_wal_record, but with ready-made image of the page.
-    fn put_page_image(
-        &self,
-        tag: RelishTag,
-        blknum: u32,
-        lsn: Lsn,
-        img: Bytes,
-        update_meta: bool,
-    ) -> Result<()>;
+    fn put_page_image(&self, tag: RelishTag, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()>;

    /// Truncate relation
    fn put_truncation(&self, rel: RelishTag, lsn: Lsn, nblocks: u32) -> Result<()>;
@@ -166,9 +137,6 @@ pub trait Timeline: Send + Sync {
    /// This method is used for marking dropped relations and truncated SLRU segments
    fn put_unlink(&self, tag: RelishTag, lsn: Lsn) -> Result<()>;

-    /// Put raw data
-    fn put_raw_data(&self, tag: ObjectTag, lsn: Lsn, data: &[u8]) -> Result<()>;
-
    /// Remember the all WAL before the given LSN has been processed.
    ///
    /// The WAL receiver calls this after the put_* functions, to indicate that
@@ -195,40 +163,6 @@ pub trait Timeline: Send + Sync {
    /// NOTE: This has nothing to do with checkpoint in PostgreSQL. We don't
    /// know anything about them here in the repository.
    fn checkpoint(&self) -> Result<()>;
-
-    /// Events for all relations in the timeline.
-    /// Contains updates from start up to the last valid LSN
-    /// at time of history() call. This lsn can be read via the lsn() function.
-    ///
-    /// Relation size is increased implicitly and decreased with Truncate updates.
-    // TODO ordering guarantee?
-    fn history<'a>(&'a self) -> Result<Box<dyn History + 'a>>;
-}
-
-pub trait History: Iterator<Item = Result<Modification>> {
-    /// The last_valid_lsn at the time of history() call.
-    fn lsn(&self) -> Lsn;
-}
-
-//
-// Structure representing any update operation of object storage.
-// It is used to copy object storage content in PUSH method.
-//
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Modification {
-    pub tag: ObjectTag,
-    pub lsn: Lsn,
-    pub data: Vec<u8>,
-}
-
-impl Modification {
-    pub fn new(entry: (ObjectTag, Lsn, Vec<u8>)) -> Modification {
-        Modification {
-            tag: entry.0,
-            lsn: entry.1,
-            data: entry.2,
-        }
-    }
 }

 #[derive(Clone)]
@@ -280,17 +214,13 @@ impl WALRecord {
 mod tests {
    use super::*;
    use crate::layered_repository::LayeredRepository;
-    use crate::object_repository::ObjectRepository;
-    use crate::object_repository::{ObjectValue, PageEntry, RelationSizeEntry};
-    use crate::rocksdb_storage::RocksObjectStore;
    use crate::walredo::{WalRedoError, WalRedoManager};
-    use crate::{PageServerConf, RepositoryFormat};
+    use crate::PageServerConf;
    use postgres_ffi::pg_constants;
    use std::fs;
    use std::path::PathBuf;
    use std::str::FromStr;
    use std::time::Duration;
-    use zenith_utils::bin_ser::BeSer;
    use zenith_utils::postgres_backend::AuthType;
    use zenith_utils::zid::ZTenantId;

@@ -320,10 +250,7 @@ mod tests {

    static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);

-    fn get_test_repo(
-        test_name: &str,
-        repository_format: RepositoryFormat,
-    ) -> Result<Box<dyn Repository>> {
+    fn get_test_repo(test_name: &str) -> Result<Box<dyn Repository>> {
        let repo_dir = PathBuf::from(format!("../tmp_check/test_{}", test_name));
        let _ = fs::remove_dir_all(&repo_dir);
        fs::create_dir_all(&repo_dir)?;
@@ -340,7 +267,6 @@ mod tests {
            pg_distrib_dir: "".into(),
            auth_type: AuthType::Trust,
            auth_validation_public_key_path: None,
-            repository_format,
        };
        // Make a static copy of the config. This can never be free'd, but that's
        // OK in a test.
@@ -350,41 +276,18 @@ mod tests {

        let walredo_mgr = TestRedoManager {};

-        let repo: Box<dyn Repository + Sync + Send> = match conf.repository_format {
-            RepositoryFormat::Layered => Box::new(LayeredRepository::new(
-                conf,
-                Arc::new(walredo_mgr),
-                tenantid,
-            )),
-            RepositoryFormat::RocksDb => {
-                let obj_store = RocksObjectStore::create(conf, &tenantid)?;
-
-                Box::new(ObjectRepository::new(
-                    conf,
-                    Arc::new(obj_store),
-                    Arc::new(walredo_mgr),
-                    tenantid,
-                ))
-            }
-        };
+        let repo = Box::new(LayeredRepository::new(
+            conf,
+            Arc::new(walredo_mgr),
+            tenantid,
+        ));

        Ok(repo)
    }

-    /// Test get_relsize() and truncation.
    #[test]
-    fn test_relsize_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_relsize_rocksdb", RepositoryFormat::RocksDb)?;
-        test_relsize(&*repo)
-    }
-
-    #[test]
-    fn test_relsize_layered() -> Result<()> {
-        let repo = get_test_repo("test_relsize_layered", RepositoryFormat::Layered)?;
-        test_relsize(&*repo)
-    }
-
-    fn test_relsize(repo: &dyn Repository) -> Result<()> {
+    fn test_relsize() -> Result<()> {
+        let repo = get_test_repo("test_relsize")?;
        // get_timeline() with non-existent timeline id should fail
        //repo.get_timeline("11223344556677881122334455667788");

@@ -393,11 +296,11 @@ mod tests {
        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;

        tline.init_valid_lsn(Lsn(1));
-        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"), true)?;
-        tline.put_page_image(TESTREL_A, 1, Lsn(4), TEST_IMG("foo blk 1 at 4"), true)?;
-        tline.put_page_image(TESTREL_A, 2, Lsn(5), TEST_IMG("foo blk 2 at 5"), true)?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"))?;
+        tline.put_page_image(TESTREL_A, 1, Lsn(4), TEST_IMG("foo blk 1 at 4"))?;
+        tline.put_page_image(TESTREL_A, 2, Lsn(5), TEST_IMG("foo blk 2 at 5"))?;

        tline.advance_last_valid_lsn(Lsn(5));

@@ -469,22 +372,9 @@ mod tests {

    /// Test get_relsize() and truncation with a file larger than 1 GB, so that it's
    /// split into multiple 1 GB segments in Postgres.
-    ///
-    /// This isn't very interesting with the RocksDb implementation, as we don't pay
-    /// any attention to Postgres segment boundaries there.
    #[test]
-    fn test_large_rel_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_large_rel_rocksdb", RepositoryFormat::RocksDb)?;
-        test_large_rel(&*repo)
-    }
-
-    #[test]
-    fn test_large_rel_layered() -> Result<()> {
-        let repo = get_test_repo("test_large_rel_layered", RepositoryFormat::Layered)?;
-        test_large_rel(&*repo)
-    }
-
-    fn test_large_rel(repo: &dyn Repository) -> Result<()> {
+    fn test_large_rel() -> Result<()> {
+        let repo = get_test_repo("test_large_rel")?;
        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;

@@ -494,7 +384,7 @@ mod tests {
        for blknum in 0..pg_constants::RELSEG_SIZE + 1 {
            let img = TEST_IMG(&format!("foo blk {} at {}", blknum, Lsn(lsn)));
            lsn += 1;
-            tline.put_page_image(TESTREL_A, blknum as u32, Lsn(lsn), img, true)?;
+            tline.put_page_image(TESTREL_A, blknum as u32, Lsn(lsn), img)?;
        }
        tline.advance_last_valid_lsn(Lsn(lsn));

@@ -539,49 +429,27 @@ mod tests {
        Ok(())
    }

-    fn skip_nonrel_objects<'a>(
-        snapshot: Box<dyn History + 'a>,
-    ) -> Result<impl Iterator<Item = <dyn History as Iterator>::Item> + 'a> {
-        Ok(snapshot.skip_while(|r| match r {
-            Ok(m) => match m.tag {
-                ObjectTag::RelationMetadata(_) => false,
-                _ => true,
-            },
-            _ => panic!("Iteration error"),
-        }))
-    }
-
-    #[test]
-    fn test_branch_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_branch_rocksdb", RepositoryFormat::RocksDb)?;
-        test_branch(&*repo)
-    }
-
-    #[test]
-    fn test_branch_layered() -> Result<()> {
-        let repo = get_test_repo("test_branch_layered", RepositoryFormat::Layered)?;
-        test_branch(&*repo)
-    }
-
    ///
    /// Test branch creation
    ///
-    fn test_branch(repo: &dyn Repository) -> Result<()> {
+    #[test]
+    fn test_branch() -> Result<()> {
+        let repo = get_test_repo("test_branch")?;
        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;

        // Import initial dummy checkpoint record, otherwise the get_timeline() call
        // after branching fails below
-        tline.put_page_image(RelishTag::Checkpoint, 0, Lsn(1), ZERO_PAGE.clone(), false)?;
+        tline.put_page_image(RelishTag::Checkpoint, 0, Lsn(1), ZERO_PAGE.clone())?;

        // Create a relation on the timeline
        tline.init_valid_lsn(Lsn(1));
-        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("foo blk 0 at 4"), true)?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("foo blk 0 at 4"))?;

        // Create another relation
-        tline.put_page_image(TESTREL_B, 0, Lsn(2), TEST_IMG("foobar blk 0 at 2"), true)?;
+        tline.put_page_image(TESTREL_B, 0, Lsn(2), TEST_IMG("foobar blk 0 at 2"))?;

        tline.advance_last_valid_lsn(Lsn(4));

@@ -590,7 +458,7 @@ mod tests {
        repo.branch_timeline(timelineid, newtimelineid, Lsn(3))?;
        let newtline = repo.get_timeline(newtimelineid)?;

-        newtline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("bar blk 0 at 4"), true)?;
+        newtline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("bar blk 0 at 4"))?;
        newtline.advance_last_valid_lsn(Lsn(4));

        // Check page contents on both branches
@@ -614,92 +482,6 @@ mod tests {
        Ok(())
    }

-    #[test]
-    fn test_history_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_history_rocksdb", RepositoryFormat::RocksDb)?;
-        test_history(&*repo)
-    }
-    #[test]
-    // TODO: This doesn't work with the layered storage, the functions needed for push/pull
-    // functionality haven't been implemented yet.
-    #[ignore]
-    fn test_history_layered() -> Result<()> {
-        let repo = get_test_repo("test_history_layered", RepositoryFormat::Layered)?;
-        test_history(&*repo)
-    }
-    fn test_history(repo: &dyn Repository) -> Result<()> {
-        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
-
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(0));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        // add a page and advance the last valid LSN
-        let rel = TESTREL_A;
-        tline.put_page_image(rel, 1, Lsn(1), TEST_IMG("blk 1 @ lsn 1"), true)?;
-        tline.advance_last_valid_lsn(Lsn(1));
-
-        let expected_page = Modification {
-            tag: ObjectTag::Buffer(rel, 1),
-            lsn: Lsn(1),
-            data: ObjectValue::ser(&ObjectValue::Page(PageEntry::Page(TEST_IMG(
-                "blk 1 @ lsn 1",
-            ))))?,
-        };
-        let expected_init_size = Modification {
-            tag: ObjectTag::RelationMetadata(rel),
-            lsn: Lsn(1),
-            data: ObjectValue::ser(&ObjectValue::RelationSize(RelationSizeEntry::Size(2)))?,
-        };
-        let expected_trunc_size = Modification {
-            tag: ObjectTag::RelationMetadata(rel),
-            lsn: Lsn(2),
-            data: ObjectValue::ser(&ObjectValue::RelationSize(RelationSizeEntry::Size(0)))?,
-        };
-
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(1));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(
-            Some(&expected_init_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        // truncate to zero, but don't advance the last valid LSN
-        tline.put_truncation(rel, Lsn(2), 0)?;
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(1));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(
-            Some(&expected_init_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        // advance the last valid LSN and the truncation should be observable
-        tline.advance_last_valid_lsn(Lsn(2));
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(2));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(
-            Some(&expected_init_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(
-            Some(&expected_trunc_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        Ok(())
-    }
-
    // Mock WAL redo manager that doesn't do much
    struct TestRedoManager {}

--- a/pageserver/src/restore_local_repo.rs
+++ b/pageserver/src/restore_local_repo.rs
@@ -50,7 +50,7 @@ pub fn import_timeline_from_postgres_datadir(
                    timeline.get_page_at_lsn_nowait(RelishTag::ControlFile, 0, lsn)?;
                let pg_control = ControlFileData::decode(&pg_control_bytes)?;
                let checkpoint_bytes = pg_control.checkPointCopy.encode();
-                timeline.put_page_image(RelishTag::Checkpoint, 0, lsn, checkpoint_bytes, false)?;
+                timeline.put_page_image(RelishTag::Checkpoint, 0, lsn, checkpoint_bytes)?;
            }
            Some("pg_filenode.map") => import_nonrel_file(
                timeline,
@@ -170,7 +170,7 @@ fn import_relfile(
                    forknum,
                };
                let tag = RelishTag::Relation(rel);
-                timeline.put_page_image(tag, blknum, lsn, Bytes::copy_from_slice(&buf), true)?;
+                timeline.put_page_image(tag, blknum, lsn, Bytes::copy_from_slice(&buf))?;
            }

            // TODO: UnexpectedEof is expected
@@ -211,7 +211,7 @@ fn import_nonrel_file(

    info!("importing non-rel file {}", path.display());

-    timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buffer[..]), false)?;
+    timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buffer[..]))?;
    Ok(())
 }

@@ -236,7 +236,6 @@ fn import_slru_file(timeline: &dyn Timeline, lsn: Lsn, slru: SlruKind, path: &Pa
                    rpageno,
                    lsn,
                    Bytes::copy_from_slice(&buf),
-                    true,
                )?;
            }

@@ -334,7 +333,7 @@ pub fn import_timeline_wal(walpath: &Path, timeline: &dyn Timeline, startpoint:

    info!("reached end of WAL at {}", last_lsn);
    let checkpoint_bytes = checkpoint.encode();
-    timeline.put_page_image(RelishTag::Checkpoint, 0, last_lsn, checkpoint_bytes, false)?;
+    timeline.put_page_image(RelishTag::Checkpoint, 0, last_lsn, checkpoint_bytes)?;

    timeline.advance_last_valid_lsn(last_lsn);
    timeline.checkpoint()?;
@@ -352,7 +351,9 @@ pub fn save_decoded_record(
    recdata: Bytes,
    lsn: Lsn,
 ) -> Result<()> {
-    checkpoint.update_next_xid(decoded.xl_xid);
+    if decoded.xl_xid > 0 {
+        checkpoint.update_next_xid(decoded.xl_xid);
+    }

    // Iterate through all the blocks that the record modifies, and
    // "put" a separate copy of the record for each block.
@@ -387,9 +388,30 @@ pub fn save_decoded_record(
        if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) == pg_constants::XLOG_DBASE_CREATE {
            let createdb = XlCreateDatabase::decode(&mut buf);
            save_xlog_dbase_create(timeline, lsn, &createdb)?;
-        } else {
-            // TODO
-            trace!("XLOG_DBASE_DROP is not handled yet");
+        } else if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK)
+            == pg_constants::XLOG_DBASE_DROP
+        {
+            let dropdb = XlDropDatabase::decode(&mut buf);
+
+            for tablespace_id in dropdb.tablespace_ids {
+                let rels = timeline.list_rels(tablespace_id, dropdb.db_id, lsn)?;
+                for rel in rels {
+                    timeline.put_unlink(RelishTag::Relation(rel), lsn)?;
+                }
+                trace!(
+                    "Unlink FileNodeMap {}, {} at lsn {}",
+                    tablespace_id,
+                    dropdb.db_id,
+                    lsn
+                );
+                timeline.put_unlink(
+                    RelishTag::FileNodeMap {
+                        spcnode: tablespace_id,
+                        dbnode: dropdb.db_id,
+                    },
+                    lsn,
+                )?;
+            }
        }
    } else if decoded.xl_rmid == pg_constants::RM_TBLSPC_ID {
        trace!("XLOG_TBLSPC_CREATE/DROP is not handled yet");
@@ -407,7 +429,6 @@ pub fn save_decoded_record(
                rpageno,
                lsn,
                ZERO_PAGE,
-                true,
            )?;
        } else {
            assert!(info == pg_constants::CLOG_TRUNCATE);
@@ -425,9 +446,11 @@ pub fn save_decoded_record(
            let parsed_xact = XlXactParsedRecord::decode(&mut buf, decoded.xl_xid, decoded.xl_info);
            save_xact_record(timeline, lsn, &parsed_xact, decoded)?;
            // Remove twophase file. see RemoveTwoPhaseFile() in postgres code
-            info!(
+            trace!(
                "unlink twophaseFile for xid {} parsed_xact.xid {} here at {}",
-                decoded.xl_xid, parsed_xact.xid, lsn
+                decoded.xl_xid,
+                parsed_xact.xid,
+                lsn
            );
            timeline.put_unlink(
                RelishTag::TwoPhase {
@@ -446,7 +469,6 @@ pub fn save_decoded_record(
                0,
                lsn,
                Bytes::copy_from_slice(&buf[..]),
-                true,
            )?;
        }
    } else if decoded.xl_rmid == pg_constants::RM_MULTIXACT_ID {
@@ -464,7 +486,6 @@ pub fn save_decoded_record(
                rpageno,
                lsn,
                ZERO_PAGE,
-                true,
            )?;
        } else if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE {
            let pageno = buf.get_u32_le();
@@ -478,7 +499,6 @@ pub fn save_decoded_record(
                rpageno,
                lsn,
                ZERO_PAGE,
-                true,
            )?;
        } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
            let xlrec = XlMultiXactCreate::decode(&mut buf);
@@ -489,7 +509,7 @@ pub fn save_decoded_record(
        }
    } else if decoded.xl_rmid == pg_constants::RM_RELMAP_ID {
        let xlrec = XlRelmapUpdate::decode(&mut buf);
-        save_relmap_record(timeline, lsn, &xlrec, decoded)?;
+        save_relmap_page(timeline, lsn, &xlrec, decoded)?;
    } else if decoded.xl_rmid == pg_constants::RM_XLOG_ID {
        let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
        if info == pg_constants::XLOG_NEXTOID {
@@ -560,7 +580,7 @@ fn save_xlog_dbase_create(timeline: &dyn Timeline, lsn: Lsn, rec: &XlCreateDatab

            debug!("copying block {} from {} to {}", blknum, src_rel, dst_rel);

-            timeline.put_page_image(RelishTag::Relation(dst_rel), blknum, lsn, content, true)?;
+            timeline.put_page_image(RelishTag::Relation(dst_rel), blknum, lsn, content)?;
            num_blocks_copied += 1;
        }

@@ -584,7 +604,7 @@ fn save_xlog_dbase_create(timeline: &dyn Timeline, lsn: Lsn, rec: &XlCreateDatab
                        spcnode: tablespace_id,
                        dbnode: db_id,
                    };
-                    timeline.put_page_image(new_tag, 0, lsn, img, false)?;
+                    timeline.put_page_image(new_tag, 0, lsn, img)?;
                    break;
                }
            }
@@ -946,22 +966,23 @@ fn save_multixact_truncate_record(
    Ok(())
 }

-fn save_relmap_record(
+fn save_relmap_page(
    timeline: &dyn Timeline,
    lsn: Lsn,
    xlrec: &XlRelmapUpdate,
    decoded: &DecodedWALRecord,
 ) -> Result<()> {
-    let rec = WALRecord {
-        lsn,
-        will_init: true,
-        rec: decoded.record.clone(),
-        main_data_offset: decoded.main_data_offset as u32,
-    };
    let tag = RelishTag::FileNodeMap {
        spcnode: xlrec.tsid,
        dbnode: xlrec.dbid,
    };
-    timeline.put_wal_record(tag, 0, rec)?;
+
+    let mut buf = decoded.record.clone();
+    buf.advance(decoded.main_data_offset);
+    // skip xl_relmap_update
+    buf.advance(12);
+
+    timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buf[..]))?;
+
    Ok(())
 }
--- a/pageserver/src/rocksdb_storage.rs
+++ b/pageserver/src/rocksdb_storage.rs
@@ -1,475 +0,0 @@
-//!
-//! An implementation of the ObjectStore interface, backed by RocksDB
-//!
-use crate::object_key::*;
-use crate::object_store::ObjectStore;
-use crate::relish::*;
-use crate::PageServerConf;
-use anyhow::{bail, Result};
-use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
-use std::sync::{Arc, Mutex};
-use zenith_utils::bin_ser::BeSer;
-use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::ZTenantId;
-use zenith_utils::zid::ZTimelineId;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-struct StorageKey {
-    obj_key: ObjectKey,
-    lsn: Lsn,
-}
-
-impl StorageKey {
-    /// The first key for a given timeline
-    fn timeline_start(timeline: ZTimelineId) -> Self {
-        Self {
-            obj_key: ObjectKey {
-                timeline,
-                tag: ObjectTag::TimelineMetadataTag,
-            },
-            lsn: Lsn(0),
-        }
-    }
-}
-
-///
-/// RocksDB very inefficiently delete random record. Instead of it we have to use merge
-/// filter, which allows to throw away records at LSM merge phase.
-/// Unfortunately, it is hard (if ever possible)  to determine whether version can be removed
-/// at merge time. Version ca be removed if:
-/// 1. It is above PITR horizon (we need to get current LSN and gc_horizon from config)
-/// 2. Page is reconstructed at horizon (all WAL records above horizon are applied and can be removed)
-///
-/// So we have GC process which reconstructs pages at horizon and mark deteriorated WAL record
-/// for deletion. To mark object for deletion we can either set some flag in object itself.
-/// But it is complicated with new object value format, because RocksDB storage knows nothing about
-/// this format. Also updating whole record just to set one bit seems to be inefficient in any case.
-/// This is why we keep keys of marked for deletion versions in HashSet in memory.
-/// When LSM merge filter found key in this map, it removes it from the set preventing memory overflow.
-///
-struct GarbageCollector {
-    garbage: Mutex<HashSet<Vec<u8>>>,
-}
-
-impl GarbageCollector {
-    fn new() -> GarbageCollector {
-        GarbageCollector {
-            garbage: Mutex::new(HashSet::new()),
-        }
-    }
-
-    /// Called by GC to mark version as delete
-    fn mark_for_deletion(&self, key: &[u8]) {
-        let mut garbage = self.garbage.lock().unwrap();
-        garbage.insert(key.to_vec());
-    }
-
-    /// Called by LSM merge filter. If it finds key in the set, then
-    /// it doesn't merge it and removes from this set.
-    fn was_deleted(&self, key: &[u8]) -> bool {
-        let key = key.to_vec();
-        let mut garbage = self.garbage.lock().unwrap();
-        garbage.remove(&key)
-    }
-}
-
-pub struct RocksObjectStore {
-    _conf: &'static PageServerConf,
-
-    // RocksDB handle
-    db: rocksdb::DB,
-    gc: Arc<GarbageCollector>,
-}
-
-impl ObjectStore for RocksObjectStore {
-    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>> {
-        let val = self.db.get(StorageKey::ser(&StorageKey {
-            obj_key: key.clone(),
-            lsn,
-        })?)?;
-        if let Some(val) = val {
-            Ok(val)
-        } else {
-            bail!("could not find page {:?}", key);
-        }
-    }
-
-    fn get_next_key(&self, key: &ObjectKey) -> Result<Option<ObjectKey>> {
-        let mut iter = self.db.raw_iterator();
-        let search_key = StorageKey {
-            obj_key: key.clone(),
-            lsn: Lsn(0),
-        };
-        iter.seek(search_key.ser()?);
-        if !iter.valid() {
-            Ok(None)
-        } else {
-            let key = StorageKey::des(iter.key().unwrap())?;
-            Ok(Some(key.obj_key.clone()))
-        }
-    }
-
-    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()> {
-        self.db.put(
-            StorageKey::ser(&StorageKey {
-                obj_key: key.clone(),
-                lsn,
-            })?,
-            value,
-        )?;
-        Ok(())
-    }
-
-    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()> {
-        self.gc.mark_for_deletion(&StorageKey::ser(&StorageKey {
-            obj_key: key.clone(),
-            lsn,
-        })?);
-        Ok(())
-    }
-
-    /// Iterate through page versions of given page, starting from the given LSN.
-    /// The versions are walked in descending LSN order.
-    fn object_versions<'a>(
-        &'a self,
-        key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>> {
-        let iter = RocksObjectVersionIter::new(&self.db, key, lsn)?;
-        Ok(Box::new(iter))
-    }
-
-    /// Iterate through all timeline objects
-    fn list_objects<'a>(
-        &'a self,
-        timeline: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = ObjectTag> + 'a>> {
-        let iter = RocksObjectIter::new(&self.db, timeline, lsn)?;
-        Ok(Box::new(iter))
-    }
-
-    /// Get a list of all distinct relations in given tablespace and database.
-    ///
-    /// TODO: This implementation is very inefficient, it scans
-    /// through all entries in the given database. In practice, this
-    /// is used for CREATE DATABASE, and usually the template database is small.
-    /// But if it's not, this will be slow.
-    fn list_rels(
-        &self,
-        timelineid: ZTimelineId,
-        spcnode: u32,
-        dbnode: u32,
-        lsn: Lsn,
-    ) -> Result<HashSet<RelTag>> {
-        // FIXME: This scans everything. Very slow
-
-        let mut rels: HashSet<RelTag> = HashSet::new();
-
-        let mut search_rel_tag = RelTag {
-            spcnode,
-            dbnode,
-            relnode: 0,
-            forknum: 0u8,
-        };
-        let mut iter = self.db.raw_iterator();
-        loop {
-            let search_key = StorageKey {
-                obj_key: ObjectKey {
-                    timeline: timelineid,
-                    tag: ObjectTag::RelationMetadata(RelishTag::Relation(search_rel_tag)),
-                },
-                lsn: Lsn(0),
-            };
-            iter.seek(search_key.ser()?);
-            if !iter.valid() {
-                break;
-            }
-            let key = StorageKey::des(iter.key().unwrap())?;
-
-            if let ObjectTag::RelationMetadata(RelishTag::Relation(rel_tag)) = key.obj_key.tag {
-                if spcnode != 0 && rel_tag.spcnode != spcnode
-                    || dbnode != 0 && rel_tag.dbnode != dbnode
-                {
-                    break;
-                }
-                if key.lsn <= lsn {
-                    // visible in this snapshot
-                    rels.insert(rel_tag);
-                }
-                search_rel_tag = rel_tag;
-                // skip to next relation
-                // FIXME: What if relnode is u32::MAX ?
-                search_rel_tag.relnode += 1;
-            } else {
-                // no more relation metadata entries
-                break;
-            }
-        }
-
-        Ok(rels)
-    }
-
-    /// Get a list of all distinct NON-relations in timeline
-    /// that are visible at given lsn.
-    ///
-    /// TODO: This implementation is very inefficient, it scans
-    /// through all non-rel page versions in the system. In practice, this
-    /// is used when initializing a new compute node, and the non-rel files
-    /// are never very large nor change very frequently, so this will do for now.
-    fn list_nonrels(&self, timelineid: ZTimelineId, lsn: Lsn) -> Result<HashSet<RelishTag>> {
-        let mut rels: HashSet<RelishTag> = HashSet::new();
-
-        let search_key = StorageKey {
-            obj_key: ObjectKey {
-                timeline: timelineid,
-                tag: ObjectTag::Buffer(FIRST_NONREL_RELISH_TAG, 0),
-            },
-            lsn: Lsn(0),
-        };
-
-        let mut iter = self.db.raw_iterator();
-        iter.seek(search_key.ser()?);
-        while iter.valid() {
-            let key = StorageKey::des(iter.key().unwrap())?;
-
-            if key.obj_key.timeline != timelineid {
-                // reached end of this timeline in the store
-                break;
-            }
-
-            if let ObjectTag::Buffer(rel_tag, _blknum) = key.obj_key.tag {
-                if key.lsn <= lsn {
-                    // visible in this snapshot
-                    rels.insert(rel_tag);
-                }
-            }
-            // TODO: we could skip to next relation here like we do in list_rels(),
-            // but hopefully there are not that many SLRU segments or other non-rel
-            // entries for it to matter.
-            iter.next();
-        }
-
-        Ok(rels)
-    }
-
-    /// Iterate through versions of all objects in a timeline.
-    ///
-    /// Returns objects in increasing key-version order.
-    /// Returns all versions up to and including the specified LSN.
-    fn objects<'a>(
-        &'a self,
-        timeline: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = Result<(ObjectTag, Lsn, Vec<u8>)>> + 'a>> {
-        let start_key = StorageKey::timeline_start(timeline);
-        let start_key_bytes = StorageKey::ser(&start_key)?;
-        let iter = self.db.iterator(rocksdb::IteratorMode::From(
-            &start_key_bytes,
-            rocksdb::Direction::Forward,
-        ));
-
-        Ok(Box::new(RocksObjects {
-            iter,
-            timeline,
-            lsn,
-        }))
-    }
-
-    fn compact(&self) {
-        self.db.compact_range::<&[u8], &[u8]>(None, None);
-    }
-}
-
-impl RocksObjectStore {
-    /// Open a RocksDB database.
-    pub fn open(conf: &'static PageServerConf, tenantid: &ZTenantId) -> Result<RocksObjectStore> {
-        let opts = Self::get_rocksdb_opts();
-        let obj_store = Self::new(conf, opts, tenantid)?;
-        Ok(obj_store)
-    }
-
-    /// Create a new, empty RocksDB database.
-    pub fn create(conf: &'static PageServerConf, tenantid: &ZTenantId) -> Result<RocksObjectStore> {
-        let path = conf.tenant_path(&tenantid).join("rocksdb-storage");
-        std::fs::create_dir(&path)?;
-
-        let mut opts = Self::get_rocksdb_opts();
-        opts.create_if_missing(true);
-        opts.set_error_if_exists(true);
-        let obj_store = Self::new(conf, opts, tenantid)?;
-        Ok(obj_store)
-    }
-
-    fn new(
-        conf: &'static PageServerConf,
-        mut opts: rocksdb::Options,
-        tenantid: &ZTenantId,
-    ) -> Result<RocksObjectStore> {
-        let path = conf.tenant_path(&tenantid).join("rocksdb-storage");
-        let gc = Arc::new(GarbageCollector::new());
-        let gc_ref = gc.clone();
-        opts.set_compaction_filter("ttl", move |_level: u32, key: &[u8], _val: &[u8]| {
-            if gc_ref.was_deleted(key) {
-                rocksdb::compaction_filter::Decision::Remove
-            } else {
-                rocksdb::compaction_filter::Decision::Keep
-            }
-        });
-        let db = rocksdb::DB::open(&opts, &path)?;
-        let obj_store = RocksObjectStore {
-            _conf: conf,
-            db,
-            gc,
-        };
-        Ok(obj_store)
-    }
-
-    /// common options used by `open` and `create`
-    fn get_rocksdb_opts() -> rocksdb::Options {
-        let mut opts = rocksdb::Options::default();
-        opts.set_use_fsync(true);
-        opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
-        opts
-    }
-}
-
-///
-/// Iterator for `object_versions`. Returns all page versions of a given block, in
-/// reverse LSN order.
-///
-struct RocksObjectVersionIter<'a> {
-    obj_key: ObjectKey,
-    dbiter: rocksdb::DBRawIterator<'a>,
-    first_call: bool,
-}
-impl<'a> RocksObjectVersionIter<'a> {
-    fn new(
-        db: &'a rocksdb::DB,
-        obj_key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<RocksObjectVersionIter<'a>> {
-        let key = StorageKey {
-            obj_key: obj_key.clone(),
-            lsn,
-        };
-        let mut dbiter = db.raw_iterator();
-        dbiter.seek_for_prev(StorageKey::ser(&key)?); // locate last entry
-        Ok(RocksObjectVersionIter {
-            first_call: true,
-            obj_key: obj_key.clone(),
-            dbiter,
-        })
-    }
-}
-impl<'a> Iterator for RocksObjectVersionIter<'a> {
-    type Item = (Lsn, Vec<u8>);
-
-    fn next(&mut self) -> std::option::Option<Self::Item> {
-        if self.first_call {
-            self.first_call = false;
-        } else {
-            self.dbiter.prev(); // walk backwards
-        }
-
-        if !self.dbiter.valid() {
-            return None;
-        }
-        let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
-        if key.obj_key.tag != self.obj_key.tag {
-            return None;
-        }
-        let val = self.dbiter.value().unwrap();
-        let result = val.to_vec();
-
-        Some((key.lsn, result))
-    }
-}
-
-struct RocksObjects<'r> {
-    iter: rocksdb::DBIterator<'r>,
-    timeline: ZTimelineId,
-    lsn: Lsn,
-}
-
-impl<'r> Iterator for RocksObjects<'r> {
-    // TODO consider returning Box<[u8]>
-    type Item = Result<(ObjectTag, Lsn, Vec<u8>)>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.next_result().transpose()
-    }
-}
-
-impl<'r> RocksObjects<'r> {
-    fn next_result(&mut self) -> Result<Option<(ObjectTag, Lsn, Vec<u8>)>> {
-        for (key_bytes, v) in &mut self.iter {
-            let key = StorageKey::des(&key_bytes)?;
-
-            if key.obj_key.timeline != self.timeline {
-                return Ok(None);
-            }
-
-            if key.lsn > self.lsn {
-                // TODO can speed up by seeking iterator
-                continue;
-            }
-
-            return Ok(Some((key.obj_key.tag, key.lsn, v.to_vec())));
-        }
-
-        Ok(None)
-    }
-}
-
-///
-/// Iterator for `list_objects`. Returns all objects preceeding specified LSN
-///
-struct RocksObjectIter<'a> {
-    timeline: ZTimelineId,
-    key: StorageKey,
-    lsn: Lsn,
-    dbiter: rocksdb::DBRawIterator<'a>,
-}
-impl<'a> RocksObjectIter<'a> {
-    fn new(db: &'a rocksdb::DB, timeline: ZTimelineId, lsn: Lsn) -> Result<RocksObjectIter<'a>> {
-        let key = StorageKey {
-            obj_key: ObjectKey {
-                timeline,
-                tag: ObjectTag::FirstTag,
-            },
-            lsn: Lsn(0),
-        };
-        let dbiter = db.raw_iterator();
-        Ok(RocksObjectIter {
-            key,
-            timeline,
-            lsn,
-            dbiter,
-        })
-    }
-}
-impl<'a> Iterator for RocksObjectIter<'a> {
-    type Item = ObjectTag;
-
-    fn next(&mut self) -> std::option::Option<Self::Item> {
-        loop {
-            self.dbiter.seek(StorageKey::ser(&self.key).unwrap());
-            if !self.dbiter.valid() {
-                return None;
-            }
-            let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
-            if key.obj_key.timeline != self.timeline {
-                // End of this timeline
-                return None;
-            }
-            self.key = key.clone();
-            self.key.lsn = Lsn(u64::MAX); // next seek should skip all versions
-            if key.lsn <= self.lsn {
-                // visible in this snapshot
-                return Some(key.obj_key.tag);
-            }
-        }
-    }
-}
--- a/pageserver/src/tenant_mgr.rs
+++ b/pageserver/src/tenant_mgr.rs
@@ -3,11 +3,9 @@

 use crate::branches;
 use crate::layered_repository::LayeredRepository;
-use crate::object_repository::ObjectRepository;
 use crate::repository::Repository;
-use crate::rocksdb_storage::RocksObjectStore;
 use crate::walredo::PostgresRedoManager;
-use crate::{PageServerConf, RepositoryFormat};
+use crate::PageServerConf;
 use anyhow::{anyhow, bail, Result};
 use lazy_static::lazy_static;
 use log::info;
@@ -33,27 +31,12 @@ pub fn init(conf: &'static PageServerConf) {
        let walredo_mgr = PostgresRedoManager::new(conf, tenantid);

        // Set up an object repository, for actual data storage.
-        let repo: Arc<dyn Repository + Sync + Send> = match conf.repository_format {
-            RepositoryFormat::Layered => {
-                let repo = Arc::new(LayeredRepository::new(
-                    conf,
-                    Arc::new(walredo_mgr),
-                    tenantid,
-                ));
-                LayeredRepository::launch_checkpointer_thread(conf, repo.clone());
-                repo
-            }
-            RepositoryFormat::RocksDb => {
-                let obj_store = RocksObjectStore::open(conf, &tenantid).unwrap();
-
-                Arc::new(ObjectRepository::new(
-                    conf,
-                    Arc::new(obj_store),
-                    Arc::new(walredo_mgr),
-                    tenantid,
-                ))
-            }
-        };
+        let repo = Arc::new(LayeredRepository::new(
+            conf,
+            Arc::new(walredo_mgr),
+            tenantid,
+        ));
+        LayeredRepository::launch_checkpointer_thread(conf, repo.clone());

        info!("initialized storage for tenant: {}", &tenantid);
        m.insert(tenantid, repo);
--- a/pageserver/src/waldecoder.rs
+++ b/pageserver/src/waldecoder.rs
@@ -1,6 +1,6 @@
 //!
 //! WAL decoder. For each WAL record, it decodes the record to figure out which data blocks
-//! the record affects, to add the records to the page cache.
+//! the record affects, so that they can be stored in repository.
 //!
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use crc32c::*;
@@ -331,6 +331,31 @@ impl XlCreateDatabase {
    }
 }

+#[repr(C)]
+#[derive(Debug)]
+pub struct XlDropDatabase {
+    pub db_id: Oid,
+    pub n_tablespaces: Oid, /* number of tablespace IDs */
+    pub tablespace_ids: Vec<Oid>,
+}
+
+impl XlDropDatabase {
+    pub fn decode(buf: &mut Bytes) -> XlDropDatabase {
+        let mut rec = XlDropDatabase {
+            db_id: buf.get_u32_le(),
+            n_tablespaces: buf.get_u32_le(),
+            tablespace_ids: Vec::<Oid>::new(),
+        };
+
+        for _i in 0..rec.n_tablespaces {
+            let id = buf.get_u32_le();
+            rec.tablespace_ids.push(id);
+        }
+
+        rec
+    }
+}
+
 #[repr(C)]
 #[derive(Debug)]
 pub struct XlHeapInsert {
@@ -351,6 +376,7 @@ impl XlHeapInsert {
 #[derive(Debug)]
 pub struct XlHeapMultiInsert {
    pub flags: u8,
+    pub _padding: u8,
    pub ntuples: u16,
 }

@@ -358,6 +384,7 @@ impl XlHeapMultiInsert {
    pub fn decode(buf: &mut Bytes) -> XlHeapMultiInsert {
        XlHeapMultiInsert {
            flags: buf.get_u8(),
+            _padding: buf.get_u8(),
            ntuples: buf.get_u16_le(),
        }
    }
@@ -368,6 +395,8 @@ impl XlHeapMultiInsert {
 pub struct XlHeapDelete {
    pub xmax: TransactionId,
    pub offnum: OffsetNumber,
+    pub _padding: u16,
+    pub t_cid: u32,
    pub infobits_set: u8,
    pub flags: u8,
 }
@@ -377,6 +406,8 @@ impl XlHeapDelete {
        XlHeapDelete {
            xmax: buf.get_u32_le(),
            offnum: buf.get_u16_le(),
+            _padding: buf.get_u16_le(),
+            t_cid: buf.get_u32_le(),
            infobits_set: buf.get_u8(),
            flags: buf.get_u8(),
        }
@@ -390,6 +421,7 @@ pub struct XlHeapUpdate {
    pub old_offnum: OffsetNumber,
    pub old_infobits_set: u8,
    pub flags: u8,
+    pub t_cid: u32,
    pub new_xmax: TransactionId,
    pub new_offnum: OffsetNumber,
 }
@@ -401,6 +433,7 @@ impl XlHeapUpdate {
            old_offnum: buf.get_u16_le(),
            old_infobits_set: buf.get_u8(),
            flags: buf.get_u8(),
+            t_cid: buf.get_u32(),
            new_xmax: buf.get_u32_le(),
            new_offnum: buf.get_u16_le(),
        }
@@ -844,6 +877,7 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
        let blkno = blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32;
        if info == pg_constants::XLOG_HEAP_INSERT {
            let xlrec = XlHeapInsert::decode(&mut buf);
+            assert_eq!(0, buf.remaining());
            if (xlrec.flags
                & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
                    | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
@@ -859,6 +893,7 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
            }
        } else if info == pg_constants::XLOG_HEAP_DELETE {
            let xlrec = XlHeapDelete::decode(&mut buf);
+            assert_eq!(0, buf.remaining());
            if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 {
                let mut blk = DecodedBkpBlock::new();
                blk.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
@@ -872,6 +907,9 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
            || info == pg_constants::XLOG_HEAP_HOT_UPDATE
        {
            let xlrec = XlHeapUpdate::decode(&mut buf);
+            // the size of tuple data is inferred from the size of the record.
+            // we can't validate the remaining number of bytes without parsing
+            // the tuple data.
            if (xlrec.flags & pg_constants::XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0 {
                let mut blk = DecodedBkpBlock::new();
                blk.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
@@ -897,6 +935,15 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
        let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
        if info == pg_constants::XLOG_HEAP2_MULTI_INSERT {
            let xlrec = XlHeapMultiInsert::decode(&mut buf);
+
+            let offset_array_len = if xlogrec.xl_info & pg_constants::XLOG_HEAP_INIT_PAGE > 0 {
+                // the offsets array is omitted if XLOG_HEAP_INIT_PAGE is set
+                0
+            } else {
+                std::mem::size_of::<u16>() * xlrec.ntuples as usize
+            };
+            assert_eq!(offset_array_len, buf.remaining());
+
            if (xlrec.flags
                & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
                    | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
--- a/pageserver/src/walreceiver.rs
+++ b/pageserver/src/walreceiver.rs
@@ -1,14 +1,15 @@
 //!
-//! WAL receiver connects to the WAL safekeeper service,
-//! streams WAL, decodes records and saves them in page cache.
+//! WAL receiver connects to the WAL safekeeper service, streams WAL,
+//! decodes records and saves them in the repository for the correct
+//! timeline.
 //!
 //! We keep one WAL receiver active per timeline.

-use crate::page_cache;
 use crate::relish::*;
 use crate::restore_local_repo;
+use crate::tenant_mgr;
 use crate::waldecoder::*;
-use crate::{PageServerConf, RepositoryFormat};
+use crate::PageServerConf;
 use anyhow::{Error, Result};
 use lazy_static::lazy_static;
 use log::*;
@@ -145,7 +146,7 @@ fn walreceiver_main(
    let end_of_wal = Lsn::from(u64::from(identify.xlogpos));
    let mut caught_up = false;

-    let repository = page_cache::get_repository_for_tenant(tenantid)?;
+    let repository = tenant_mgr::get_repository_for_tenant(tenantid)?;
    let timeline = repository.get_timeline(timelineid).unwrap();

    //
@@ -228,14 +229,13 @@ fn walreceiver_main(
                            0,
                            lsn,
                            new_checkpoint_bytes,
-                            false,
                        )?;
                    }
                }

-                // Update the last_valid LSN value in the page cache one more time. We updated
+                // Update the last_valid LSN value in the timeline one more time. We updated
                // it in the loop above, between each WAL record, but we might have received
-                // a partial record after the last completed record. Our page cache's value
+                // a partial record after the last completed record. Our timeline's value
                // better reflect that, because GetPage@LSN requests might also point in the
                // middle of a record, if the request LSN was taken from the server's current
                // flush ptr.
@@ -264,12 +264,6 @@ fn walreceiver_main(
                    )?;

                    if newest_segno - oldest_segno >= 10 {
-                        // FIXME: The layered repository performs checkpointing in a separate thread, so this
-                        // isn't needed anymore. Remove 'checkpoint' from the Timeline trait altogether?
-                        if conf.repository_format == RepositoryFormat::RocksDb {
-                            timeline.checkpoint()?;
-                        }
-
                        // TODO: This is where we could remove WAL older than last_rec_lsn.
                        //remove_wal_files(timelineid, pg_constants::WAL_SEGMENT_SIZE, last_rec_lsn)?;
                    }
--- a/pageserver/src/walredo.rs
+++ b/pageserver/src/walredo.rs
@@ -418,11 +418,6 @@ impl PostgresRedoManager {
                    } else {
                        panic!();
                    }
-                } else if xlogrec.xl_rmid == pg_constants::RM_RELMAP_ID {
-                    // Relation map file has size 512 bytes
-                    page.clear();
-                    page.extend_from_slice(&buf[12..]); // skip xl_relmap_update
-                    assert!(page.len() == 512); // size of pg_filenode.map
                }
            }

--- a/postgres_ffi/Cargo.toml
+++ b/postgres_ffi/Cargo.toml
@@ -19,8 +19,9 @@ lazy_static = "1.4"
 log = "0.4.14"
 memoffset = "0.6.2"
 thiserror = "1.0"
+serde = { version = "1.0", features = ["derive"] }
 workspace_hack = { path = "../workspace_hack" }
 zenith_utils = { path = "../zenith_utils" }

 [build-dependencies]
-bindgen = "0.57"
+bindgen = "0.59.1"
--- a/postgres_ffi/README
+++ b/postgres_ffi/README
@@ -13,11 +13,6 @@ in each major PostgreSQL version. Currently, this module is based on
 PostgreSQL v14, but in the future we will probably need a separate
 copy for each PostgreSQL version.

-To interact with the C structs, there is some unsafe code in this
-module. Do not copy-paste that to the rest of the codebase! Keep the
-amount of unsafe code to a minimum, and limited to this module only,
-and only where it's truly needed.
-
 TODO: Currently, there is also some code that deals with WAL records
 in pageserver/src/waldecoder.rs.  That should be moved into this
 module. The rest of the codebase should not have intimate knowledge of
--- a/postgres_ffi/build.rs
+++ b/postgres_ffi/build.rs
@@ -3,6 +3,44 @@ extern crate bindgen;
 use std::env;
 use std::path::PathBuf;

+use bindgen::callbacks::ParseCallbacks;
+
+#[derive(Debug)]
+struct PostgresFfiCallbacks;
+
+impl ParseCallbacks for PostgresFfiCallbacks {
+    fn include_file(&self, filename: &str) {
+        // This does the equivalent of passing bindgen::CargoCallbacks
+        // to the builder .parse_callbacks() method.
+        let cargo_callbacks = bindgen::CargoCallbacks;
+        cargo_callbacks.include_file(filename)
+    }
+
+    // Add any custom #[derive] attributes to the data structures that bindgen
+    // creates.
+    fn add_derives(&self, name: &str) -> Vec<String> {
+        // This is the list of data structures that we want to serialize/deserialize.
+        let serde_list = [
+            "XLogRecord",
+            "XLogPageHeaderData",
+            "XLogLongPageHeaderData",
+            "CheckPoint",
+            "FullTransactionId",
+            "ControlFileData",
+        ];
+
+        if serde_list.contains(&name) {
+            vec![
+                "Default".into(), // Default allows us to easily fill the padding fields with 0.
+                "Serialize".into(),
+                "Deserialize".into(),
+            ]
+        } else {
+            vec![]
+        }
+    }
+}
+
 fn main() {
    // Tell cargo to invalidate the built crate whenever the wrapper changes
    println!("cargo:rerun-if-changed=pg_control_ffi.h");
@@ -19,20 +57,23 @@ fn main() {
        // Tell cargo to invalidate the built crate whenever any of the
        // included header files changed.
        //
-        .parse_callbacks(Box::new(bindgen::CargoCallbacks))
+        .parse_callbacks(Box::new(PostgresFfiCallbacks))
        //
        // These are the types and constants that we want to generate bindings for
        //
-        .whitelist_type("ControlFileData")
-        .whitelist_type("CheckPoint")
-        .whitelist_type("FullTransactionId")
-        .whitelist_type("XLogRecord")
-        .whitelist_type("XLogPageHeaderData")
-        .whitelist_type("XLogLongPageHeaderData")
-        .whitelist_var("XLOG_PAGE_MAGIC")
-        .whitelist_var("PG_CONTROL_FILE_SIZE")
-        .whitelist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
-        .whitelist_type("DBState")
+        .allowlist_type("ControlFileData")
+        .allowlist_type("CheckPoint")
+        .allowlist_type("FullTransactionId")
+        .allowlist_type("XLogRecord")
+        .allowlist_type("XLogPageHeaderData")
+        .allowlist_type("XLogLongPageHeaderData")
+        .allowlist_var("XLOG_PAGE_MAGIC")
+        .allowlist_var("PG_CONTROL_FILE_SIZE")
+        .allowlist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
+        .allowlist_type("DBState")
+        // Because structs are used for serialization, tell bindgen to emit
+        // explicit padding fields.
+        .explicit_padding(true)
        //
        // Path the server include dir. It is in tmp_install/include/server, if you did
        // "configure --prefix=<path to tmp_install>". But if you used "configure --prefix=/",
--- a/postgres_ffi/src/controlfile_utils.rs
+++ b/postgres_ffi/src/controlfile_utils.rs
@@ -43,6 +43,8 @@ impl ControlFileData {
    /// Interpret a slice of bytes as a Postgres control file.
    ///
    pub fn decode(buf: &[u8]) -> Result<ControlFileData> {
+        use zenith_utils::bin_ser::LeSer;
+
        // Check that the slice has the expected size. The control file is
        // padded with zeros up to a 512 byte sector size, so accept a
        // larger size too, so that the caller can just the whole file
@@ -55,26 +57,8 @@ impl ControlFileData {
        let OFFSETOF_CRC = Self::pg_control_crc_offset();
        let expectedcrc = crc32c::crc32c(&buf[0..OFFSETOF_CRC]);

-        // Convert the slice into an array of the right size, and use `transmute` to
-        // reinterpret the raw bytes as a ControlFileData struct.
-        //
-        // NB: Ideally we would use 'zerocopy::FromBytes' for this, but bindgen doesn't
-        // derive FromBytes for us. The safety of this depends on the same constraints
-        // as for FromBytes, namely, all of its fields must implement FromBytes. That
-        // includes the primitive integer types, like `u8`, `u16`, `u32`, `u64` and their
-        // signed variants. But `bool` is not safe, because the contents of the high bits
-        // in a rust bool are undefined. In practice, PostgreSQL uses 1 to represent
-        // true and 0 for false, which is compatible with Rust bool, but let's try not to
-        // depend on it.
-        //
-        // FIXME: ControlFileData does contain 'bool's at the moment.
-        //
-        // See https://github.com/zenithdb/zenith/issues/207 for discussion on the safety
-        // of this.
-        let mut b: [u8; SIZEOF_CONTROLDATA] = [0u8; SIZEOF_CONTROLDATA];
-        b.copy_from_slice(&buf[0..SIZEOF_CONTROLDATA]);
-        let controlfile: ControlFileData =
-            unsafe { std::mem::transmute::<[u8; SIZEOF_CONTROLDATA], ControlFileData>(b) };
+        // Use serde to deserialize the input as a ControlFileData struct.
+        let controlfile = ControlFileData::des(buf)?;

        // Check the CRC
        if expectedcrc != controlfile.crc {
@@ -93,21 +77,10 @@ impl ControlFileData {
    ///
    /// The CRC is recomputed to match the contents of the fields.
    pub fn encode(&self) -> Bytes {
-        //
-        // Use `transmute` to reinterpret struct as raw bytes.
-        //
-        // FIXME: This triggers undefined behavior, because the contents
-        // of the padding bytes are undefined, and this leaks those
-        // undefined bytes into the resulting array. The Rust code won't
-        // care what's in those bytes, and PostgreSQL doesn't care
-        // either. HOWEVER, it is a potential security issue, because the
-        // bytes can contain arbitrary pieces of memory from the page
-        // server. In the worst case, that could be private keys or
-        // another tenant's data.
-        //
-        // See https://github.com/zenithdb/zenith/issues/207 for discussion.
-        let b: [u8; SIZEOF_CONTROLDATA] =
-            unsafe { std::mem::transmute::<ControlFileData, [u8; SIZEOF_CONTROLDATA]>(*self) };
+        use zenith_utils::bin_ser::LeSer;
+
+        // Serialize into a new buffer.
+        let b = self.ser().unwrap();

        // Recompute the CRC
        let OFFSETOF_CRC = Self::pg_control_crc_offset();
--- a/postgres_ffi/src/lib.rs
+++ b/postgres_ffi/src/lib.rs
@@ -4,6 +4,9 @@
 // suppress warnings on rust 1.53 due to bindgen unit tests.
 // https://github.com/rust-lang/rust-bindgen/issues/1651
 #![allow(deref_nullptr)]
+
+use serde::{Deserialize, Serialize};
+
 include!(concat!(env!("OUT_DIR"), "/bindings.rs"));

 pub mod controlfile_utils;
--- a/postgres_ffi/src/pg_constants.rs
+++ b/postgres_ffi/src/pg_constants.rs
@@ -116,6 +116,7 @@ pub const XLOG_HEAP_INSERT: u8 = 0x00;
 pub const XLOG_HEAP_DELETE: u8 = 0x10;
 pub const XLOG_HEAP_UPDATE: u8 = 0x20;
 pub const XLOG_HEAP_HOT_UPDATE: u8 = 0x40;
+pub const XLOG_HEAP_INIT_PAGE: u8 = 0x80;
 pub const XLOG_HEAP2_VISIBLE: u8 = 0x40;
 pub const XLOG_HEAP2_MULTI_INSERT: u8 = 0x50;
 pub const XLH_INSERT_ALL_FROZEN_SET: u8 = (1 << 5) as u8;
--- a/postgres_ffi/src/xlog_utils.rs
+++ b/postgres_ffi/src/xlog_utils.rs
@@ -44,8 +44,6 @@ pub type TimeLineID = u32;
 pub type TimestampTz = i64;
 pub type XLogSegNo = u64;

-const XID_CHECKPOINT_INTERVAL: u32 = 1024;
-
 #[allow(non_snake_case)]
 pub fn XLogSegmentsPerXLogId(wal_segsz_bytes: usize) -> XLogSegNo {
    (0x100000000u64 / wal_segsz_bytes as u64) as XLogSegNo
@@ -121,6 +119,7 @@ fn find_end_of_wal_segment(
    let mut rec_hdr = [0u8; XLOG_RECORD_CRC_OFFS];

    while offs < wal_seg_size {
+        // we are at the beginning of the page; read it in
        if offs % XLOG_BLCKSZ == 0 {
            if let Ok(bytes_read) = file.read(&mut buf) {
                if bytes_read != buf.len() {
@@ -144,11 +143,12 @@ fn find_end_of_wal_segment(
            } else {
                offs += XLOG_SIZE_OF_XLOG_SHORT_PHD;
            }
+        // beginning of the next record
        } else if contlen == 0 {
            let page_offs = offs % XLOG_BLCKSZ;
            let xl_tot_len = LittleEndian::read_u32(&buf[page_offs..page_offs + 4]) as usize;
            if xl_tot_len == 0 {
-                break;
+                break; // zeros, reached the end
            }
            last_valid_rec_pos = offs;
            offs += 4;
@@ -156,12 +156,13 @@ fn find_end_of_wal_segment(
            contlen = xl_tot_len - 4;
            rec_hdr[0..4].copy_from_slice(&buf[page_offs..page_offs + 4]);
        } else {
-            let page_offs = offs % XLOG_BLCKSZ;
            // we're continuing a record, possibly from previous page.
+            let page_offs = offs % XLOG_BLCKSZ;
            let pageleft = XLOG_BLCKSZ - page_offs;

            // read the rest of the record, or as much as fits on this page.
            let n = min(contlen, pageleft);
+            // fill rec_hdr (header up to (but not including) xl_crc field)
            if rec_offs < XLOG_RECORD_CRC_OFFS {
                let len = min(XLOG_RECORD_CRC_OFFS - rec_offs, n);
                rec_hdr[rec_offs..rec_offs + len].copy_from_slice(&buf[page_offs..page_offs + len]);
@@ -185,6 +186,8 @@ fn find_end_of_wal_segment(
                crc = crc32c_append(crc, &rec_hdr);
                offs = (offs + 7) & !7; // pad on 8 bytes boundary */
                if crc == wal_crc {
+                    // record is valid, advance the result to its end (with
+                    // alignment to the next record taken into account)
                    last_valid_rec_pos = offs;
                } else {
                    info!(
@@ -201,6 +204,9 @@ fn find_end_of_wal_segment(

 ///
 /// Scan a directory that contains PostgreSQL WAL files, for the end of WAL.
+/// If precise, returns end LSN (next insertion point, basically);
+/// otherwise, start of the last segment.
+/// Returns (0, 0) if there is no WAL.
 ///
 pub fn find_end_of_wal(
    data_dir: &Path,
@@ -271,23 +277,13 @@ pub fn main() {

 impl XLogRecord {
    pub fn from_bytes(buf: &mut Bytes) -> XLogRecord {
-        XLogRecord {
-            xl_tot_len: buf.get_u32_le(),
-            xl_xid: buf.get_u32_le(),
-            xl_prev: buf.get_u64_le(),
-            xl_info: buf.get_u8(),
-            xl_rmid: buf.get_u8(),
-            xl_crc: {
-                buf.advance(2);
-                buf.get_u32_le()
-            },
-        }
+        use zenith_utils::bin_ser::LeSer;
+        XLogRecord::des_from(&mut buf.reader()).unwrap()
    }

    pub fn encode(&self) -> Bytes {
-        let b: [u8; XLOG_SIZE_OF_XLOG_RECORD];
-        b = unsafe { std::mem::transmute::<XLogRecord, [u8; XLOG_SIZE_OF_XLOG_RECORD]>(*self) };
-        Bytes::copy_from_slice(&b[..])
+        use zenith_utils::bin_ser::LeSer;
+        self.ser().unwrap().into()
    }

    // Is this record an XLOG_SWITCH record? They need some special processing,
@@ -298,34 +294,20 @@ impl XLogRecord {

 impl XLogPageHeaderData {
    pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogPageHeaderData {
-        let hdr: XLogPageHeaderData = XLogPageHeaderData {
-            xlp_magic: buf.get_u16_le(),
-            xlp_info: buf.get_u16_le(),
-            xlp_tli: buf.get_u32_le(),
-            xlp_pageaddr: buf.get_u64_le(),
-            xlp_rem_len: buf.get_u32_le(),
-        };
-        buf.get_u32_le(); //padding
-        hdr
+        use zenith_utils::bin_ser::LeSer;
+        XLogPageHeaderData::des_from(&mut buf.reader()).unwrap()
    }
 }

 impl XLogLongPageHeaderData {
    pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogLongPageHeaderData {
-        XLogLongPageHeaderData {
-            std: XLogPageHeaderData::from_bytes(buf),
-            xlp_sysid: buf.get_u64_le(),
-            xlp_seg_size: buf.get_u32_le(),
-            xlp_xlog_blcksz: buf.get_u32_le(),
-        }
+        use zenith_utils::bin_ser::LeSer;
+        XLogLongPageHeaderData::des_from(&mut buf.reader()).unwrap()
    }

    pub fn encode(&self) -> Bytes {
-        let b: [u8; XLOG_SIZE_OF_XLOG_LONG_PHD];
-        b = unsafe {
-            std::mem::transmute::<XLogLongPageHeaderData, [u8; XLOG_SIZE_OF_XLOG_LONG_PHD]>(*self)
-        };
-        Bytes::copy_from_slice(&b[..])
+        use zenith_utils::bin_ser::LeSer;
+        self.ser().unwrap().into()
    }
 }

@@ -333,26 +315,24 @@ pub const SIZEOF_CHECKPOINT: usize = std::mem::size_of::<CheckPoint>();

 impl CheckPoint {
    pub fn encode(&self) -> Bytes {
-        let b: [u8; SIZEOF_CHECKPOINT];
-        b = unsafe { std::mem::transmute::<CheckPoint, [u8; SIZEOF_CHECKPOINT]>(*self) };
-        Bytes::copy_from_slice(&b[..])
+        use zenith_utils::bin_ser::LeSer;
+        self.ser().unwrap().into()
    }

    pub fn decode(buf: &[u8]) -> Result<CheckPoint, anyhow::Error> {
-        let mut b = [0u8; SIZEOF_CHECKPOINT];
-        b.copy_from_slice(&buf[0..SIZEOF_CHECKPOINT]);
-        let checkpoint: CheckPoint;
-        checkpoint = unsafe { std::mem::transmute::<[u8; SIZEOF_CHECKPOINT], CheckPoint>(b) };
-        Ok(checkpoint)
+        use zenith_utils::bin_ser::LeSer;
+        Ok(CheckPoint::des(buf)?)
    }

    // Update next XID based on provided new_xid and stored epoch.
    // Next XID should be greater than new_xid.
    // Also take in account 32-bit wrap-around.
    pub fn update_next_xid(&mut self, xid: u32) {
-        let xid = xid.wrapping_add(XID_CHECKPOINT_INTERVAL - 1) & !(XID_CHECKPOINT_INTERVAL - 1);
        let full_xid = self.nextXid.value;
-        let new_xid = std::cmp::max(xid + 1, pg_constants::FIRST_NORMAL_TRANSACTION_ID);
+        let new_xid = std::cmp::max(
+            xid.wrapping_add(1),
+            pg_constants::FIRST_NORMAL_TRANSACTION_ID,
+        );
        let old_xid = full_xid as u32;
        if new_xid.wrapping_sub(old_xid) as i32 > 0 {
            let mut epoch = full_xid >> 32;
@@ -385,6 +365,7 @@ pub fn generate_wal_segment(pg_control: &ControlFileData) -> Bytes {
                xlp_tli: 1, // FIXME: always use Postgres timeline 1
                xlp_pageaddr: pg_control.checkPoint - XLOG_SIZE_OF_XLOG_LONG_PHD as u64,
                xlp_rem_len: 0,
+                ..Default::default() // Put 0 in padding fields.
            }
        },
        xlp_sysid: pg_control.system_identifier,
@@ -404,6 +385,7 @@ pub fn generate_wal_segment(pg_control: &ControlFileData) -> Bytes {
        xl_info: pg_constants::XLOG_CHECKPOINT_SHUTDOWN,
        xl_rmid: pg_constants::RM_XLOG_ID,
        xl_crc: 0,
+        ..Default::default() // Put 0 in padding fields.
    };

    let mut rec_shord_hdr_bytes = BytesMut::new();
--- a/pre-commit.py
+++ b/pre-commit.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+from typing import List
+import subprocess
+import sys
+import enum
+import argparse
+import os
+
+
+@enum.unique
+class Color(enum.Enum):
+    RED = "\033[0;31m"
+    GREEN = "\033[0;33m"
+    CYAN = "\033[0;36m"
+
+
+NC = "\033[0m"  # No Color
+
+
+def colorify(
+    s: str,
+    color: Color,
+    no_color: bool = False,
+):
+    if no_color:
+        return s
+    return f"{color.value}{s}{NC}"
+
+
+def rustfmt(fix_inplace: bool = False, no_color: bool = False) -> str:
+    cmd = "rustfmt --edition=2018"
+    if not fix_inplace:
+        cmd += " --check"
+    if no_color:
+        cmd += " --color=never"
+    return cmd
+
+
+def get_commit_files() -> List[str]:
+    files = subprocess.check_output(
+        "git diff --cached --name-only --diff-filter=ACM".split()
+    )
+    return files.decode().splitlines()
+
+
+def check(
+    name: str, suffix: str, cmd: str, changed_files: List[str], no_color: bool = False
+):
+    print(f"Checking: {name} ", end="")
+    applicable_files = list(
+        filter(lambda fname: fname.strip().endswith(suffix), changed_files)
+    )
+    if not applicable_files:
+        print(colorify("[NOT APPLICABLE]", Color.CYAN, no_color))
+        return
+
+    cmd = f'{cmd} {" ".join(applicable_files)}'
+    res = subprocess.run(cmd.split(), capture_output=True)
+    if res.returncode != 0:
+        print(colorify("[FAILED]", Color.RED, no_color))
+        print("Please inspect the output below and run make fmt to fix automatically\n")
+        print(res.stdout.decode())
+        exit(1)
+
+    print(colorify("[OK]", Color.GREEN, no_color))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--fix-inplace", action="store_true", help="apply fixes inplace"
+    )
+    parser.add_argument(
+        "--no-color", action="store_true", help="disable colored output", default=not sys.stdout.isatty()
+    )
+    args = parser.parse_args()
+
+    files = get_commit_files()
+    # we use rustfmt here because cargo fmt does not accept list of files
+    # it internally gathers project files and feeds them to rustfmt
+    # so because we want to check only files included in the commit we use rustfmt directly
+    check(
+        name="rustfmt",
+        suffix=".rs",
+        cmd=rustfmt(fix_inplace=args.fix_inplace, no_color=args.no_color),
+        changed_files=files,
+        no_color=args.no_color,
+    )
--- a/proxy/src/proxy.rs
+++ b/proxy/src/proxy.rs
@@ -182,14 +182,14 @@ impl ProxyConnection {

        let hello_message = format!("☀️  Welcome to Zenith!

-To proceed with database creation open following link:
+To proceed with database creation, open the following link:

-    {}{}
+    {redirect_uri}{sess_id}

-It needed to be done once and we will send you '.pgpass' file which will allow you to access or create
+It needs to be done once and we will send you '.pgpass' file, which will allow you to access or create
 databases without opening the browser.

-", self.state.conf.redirect_uri,self.psql_session_id);
+", redirect_uri = self.state.conf.redirect_uri, sess_id = self.psql_session_id);

        self.pgb
            .write_message_noflush(&BeMessage::AuthenticationOk)?;
--- a/snapfile/Cargo.toml
+++ b/snapfile/Cargo.toml
@@ -1,21 +0,0 @@
-[package]
-name = "snapfile"
-version = "0.1.0"
-edition = "2018"
-
-[[bin]]
-name = "snaptool"
-path = "snaptool/main.rs"
-
-[dependencies]
-anyhow = "1.0"
-aversion = "0.2"
-bookfile = "0.3"
-serde = { version = "1.0", features = ["derive"] }
-rand = "0.8.3"
-structopt = "0.3"
-zenith_utils = { path = "../zenith_utils" }
-hex = "0.4.3"
-
-[dev-dependencies]
-tempfile = "3.2"
--- a/snapfile/snaptool/main.rs
+++ b/snapfile/snaptool/main.rs
@@ -1,64 +0,0 @@
-use anyhow::{Context, Result};
-use snapfile::{squash, SnapFile};
-use std::env::current_dir;
-use std::path::PathBuf;
-use structopt::StructOpt;
-
-#[derive(StructOpt)]
-#[structopt(about = "A tool for manipulating snapshot files")]
-enum Params {
-    Squash(Squash),
-    Describe(Describe),
-}
-
-#[derive(StructOpt)]
-struct Squash {
-    older: PathBuf,
-    newer: PathBuf,
-}
-
-#[derive(StructOpt)]
-struct Describe {
-    file: PathBuf,
-}
-
-fn print_errors(error: anyhow::Error) {
-    let formatted: Vec<_> = error.chain().map(ToString::to_string).collect();
-    eprintln!("{}", formatted.join(": "));
-}
-fn main() {
-    let res = snaptool_main();
-    if let Err(e) = res {
-        print_errors(e);
-    }
-}
-
-fn snaptool_main() -> Result<()> {
-    let params = Params::from_args();
-    match &params {
-        Params::Squash(squash_params) => {
-            let out_dir = current_dir()?;
-            squash(&squash_params.older, &squash_params.newer, &out_dir).with_context(|| {
-                format!(
-                    "squash {} {}",
-                    squash_params.older.to_string_lossy(),
-                    squash_params.newer.to_string_lossy()
-                )
-            })?;
-        }
-        Params::Describe(describe_params) => {
-            describe(describe_params)
-                .with_context(|| format!("describe {}", describe_params.file.to_string_lossy()))?;
-        }
-    }
-    Ok(())
-}
-
-fn describe(params: &Describe) -> Result<()> {
-    let mut snap = SnapFile::new(&params.file)?;
-    let meta = snap.read_meta()?;
-
-    println!("{:?}: {:#?}", params.file, meta);
-
-    Ok(())
-}
--- a/snapfile/src/lib.rs
+++ b/snapfile/src/lib.rs
@@ -1,339 +0,0 @@
-//! A file format for storage a snapshot of pages.
-
-#![warn(missing_docs)]
-#![forbid(unsafe_code)]
-#![warn(clippy::cast_possible_truncation)]
-
-mod page;
-mod squash;
-mod versioned;
-
-#[doc(inline)]
-pub use page::Page;
-
-#[doc(inline)]
-pub use squash::squash;
-
-use anyhow::{bail, Context, Result};
-use aversion::group::{DataSink, DataSourceExt};
-use aversion::util::cbor::CborData;
-use bookfile::{Book, BookWriter, ChapterWriter};
-use std::ffi::OsString;
-use std::fs::File;
-use std::io::Write;
-use std::ops::AddAssign;
-use std::path::{Path, PathBuf};
-pub use versioned::{PageIndex, PageLocation, Predecessor, SnapFileMeta};
-use zenith_utils::lsn::Lsn;
-
-impl SnapFileMeta {
-    pub fn new(previous: Option<SnapFileMeta>, timeline: [u8; 16], lsn: Lsn) -> Self {
-        // Store the metadata of the predecessor snapshot, if there is one.
-        let predecessor = previous.map(|prev| Predecessor {
-            timeline: prev.timeline,
-            lsn: prev.lsn,
-        });
-
-        SnapFileMeta {
-            timeline,
-            predecessor,
-            lsn: lsn.into(),
-        }
-    }
-
-    fn to_filename(&self) -> OsString {
-        let timeline_string = hex::encode(self.timeline);
-        let pred_lsn = match &self.predecessor {
-            None => 0,
-            Some(pred) => pred.lsn,
-        };
-        format!("{}_{:x}_{:x}.zdb", timeline_string, pred_lsn, self.lsn).into()
-    }
-}
-
-impl PageIndex {
-    /// Retrieve the page offset from the index.
-    ///
-    /// If the page is not in the index, returns `None`.
-    fn get_page_location(&self, page_num: u64) -> Option<PageLocation> {
-        self.map.get(&page_num).copied()
-    }
-
-    fn page_count(&self) -> usize {
-        self.map.len()
-    }
-}
-
-impl PageLocation {
-    fn to_offset(&self) -> u64 {
-        // Counts in units of one page.
-        self.0 * 8192
-    }
-}
-
-impl AddAssign<u64> for PageLocation {
-    fn add_assign(&mut self, rhs: u64) {
-        self.0 += rhs;
-    }
-}
-
-/// A read-only snapshot file.
-pub struct SnapFile {
-    book: Book<File>,
-    page_index: PageIndex,
-}
-
-impl SnapFile {
-    /// Open a new `SnapFile` for reading.
-    ///
-    /// This call will validate some of the file's format and read the file's
-    /// metadata; it may return an error if the file format is invalid.
-    pub fn new(path: &Path) -> Result<Self> {
-        let file =
-            File::open(path).with_context(|| format!("snapfile {}", path.to_string_lossy()))?;
-        let book = Book::new(file)?;
-        if book.magic() != versioned::SNAPFILE_MAGIC {
-            bail!("bad magic number");
-        }
-
-        // Read the page index into memory.
-        let chapter_reader = book
-            .chapter_reader(versioned::CHAPTER_PAGE_INDEX)
-            .context("snapfile missing index chapter")?;
-        let mut source = CborData::new(chapter_reader);
-        let page_index: PageIndex = source.expect_message()?;
-        Ok(SnapFile { book, page_index })
-    }
-
-    /// Read the snapshot metadata.
-    pub fn read_meta(&mut self) -> Result<SnapFileMeta> {
-        let chapter_reader = self
-            .book
-            .chapter_reader(versioned::CHAPTER_SNAP_META)
-            .context("snapfile missing meta")?;
-        let mut source = CborData::new(chapter_reader);
-        let meta: SnapFileMeta = source.expect_message()?;
-        Ok(meta)
-    }
-
-    /// Return the number of pages stored in this snapshot.
-    pub fn page_count(&self) -> usize {
-        self.page_index.page_count()
-    }
-
-    /// Check if a page exists in this snapshot's index.
-    ///
-    /// Returns `true` if the given page is stored in this snapshot file,
-    /// `false` if not.
-    pub fn has_page(&self, page_num: u64) -> bool {
-        self.page_index.get_page_location(page_num).is_some()
-    }
-
-    /// Read a page.
-    ///
-    /// If this returns Ok(None), that means that this file does not store
-    /// the requested page.
-    /// This should only fail (returning `Err`) if an IO error occurs.
-    pub fn read_page(&self, page_num: u64) -> Result<Option<Page>> {
-        match self.page_index.get_page_location(page_num) {
-            None => Ok(None),
-            Some(page_offset) => Ok(Some(self._read_page(page_offset)?)),
-        }
-    }
-
-    /// Read page data from the file.
-    ///
-    /// This does the work for read_page and PageIter.
-    fn _read_page(&self, page_location: PageLocation) -> Result<Page> {
-        // Compute the true byte offset in the file.
-        let page_offset = page_location.to_offset();
-        let chapter_reader = self
-            .book
-            .chapter_reader(versioned::CHAPTER_PAGES)
-            .context("snapfile missing pages chapter")?;
-
-        let mut page_data = Page::default();
-        let bytes_read = chapter_reader.read_at(page_data.as_mut(), page_offset)?;
-        if bytes_read != 8192 {
-            bail!("read truncated page");
-        }
-        Ok(page_data)
-    }
-
-    /// Iterate over pages.
-    ///
-    /// This will return an iterator over (usize, )
-    pub fn all_pages(&self) -> PageIter {
-        let inner = (&self.page_index.map).into_iter();
-        PageIter {
-            snapfile: self,
-            inner,
-        }
-    }
-}
-
-/// An iterator over all pages in the snapshot file.
-pub struct PageIter<'a> {
-    snapfile: &'a SnapFile,
-    inner: std::collections::btree_map::Iter<'a, u64, PageLocation>,
-}
-
-impl Iterator for PageIter<'_> {
-    type Item = Result<(u64, Page)>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let (page_num, page_offset) = self.inner.next()?;
-        let result = self
-            .snapfile
-            ._read_page(*page_offset)
-            .map(|page_data| (*page_num, page_data));
-        Some(result)
-    }
-}
-
-/// `SnapWriter` creates a new snapshot file.
-///
-/// A SnapWriter is created, has pages written into it, and is then closed.
-pub struct SnapWriter {
-    writer: ChapterWriter<File>,
-    page_index: PageIndex,
-    meta: SnapFileMeta,
-    current_offset: PageLocation,
-}
-
-impl SnapWriter {
-    /// Create a new `SnapWriter`.
-    ///
-    pub fn new(dir: &Path, meta: SnapFileMeta) -> Result<Self> {
-        let mut path = PathBuf::from(dir);
-        path.push(meta.to_filename());
-        let file = File::create(path)?;
-        let book = BookWriter::new(file, versioned::SNAPFILE_MAGIC)?;
-
-        // Write a chapter for the snapshot metadata.
-        let writer = book.new_chapter(versioned::CHAPTER_SNAP_META);
-        let mut sink = CborData::new(writer);
-        sink.write_message(&meta)?;
-        let book = sink.into_inner().close()?;
-
-        // Open a new chapter for raw page data.
-        let writer = book.new_chapter(versioned::CHAPTER_PAGES);
-        Ok(SnapWriter {
-            writer,
-            page_index: PageIndex::default(),
-            meta,
-            current_offset: PageLocation::default(),
-        })
-    }
-
-    /// Write a page into the snap file.
-    pub fn write_page<P>(&mut self, page_num: u64, page_data: P) -> Result<()>
-    where
-        P: Into<Page>,
-    {
-        let page_data: Page = page_data.into();
-        self.writer.write_all(page_data.as_ref())?;
-        let prev = self.page_index.map.insert(page_num, self.current_offset);
-        if prev.is_some() {
-            panic!("duplicate index for page {}", page_num);
-        }
-        self.current_offset += 1;
-        Ok(())
-    }
-
-    /// Finish writing pages.
-    ///
-    /// This consumes the PagesWriter and completes the snapshot.
-    //
-    pub fn finish(self) -> Result<SnapFileMeta> {
-        let book = self.writer.close()?;
-
-        // Write out a page index and close the book. This will write out any
-        // necessary file metadata.
-        // FIXME: these 3 lines could be combined into a single function
-        // that means "serialize this data structure with this format into this chapter".
-        let writer = book.new_chapter(versioned::CHAPTER_PAGE_INDEX);
-        let mut sink = CborData::new(writer);
-        sink.write_message(&self.page_index)?;
-
-        // Close the chapter, then close the book.
-        sink.into_inner().close()?.close()?;
-
-        // Return the snapshot metadata to the caller.
-        Ok(self.meta)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::path::PathBuf;
-    use tempfile::TempDir;
-
-    const TEST_TIMELINE: [u8; 16] = [99u8; 16];
-
-    #[test]
-    fn snap_two_pages() {
-        // When `dir` goes out of scope the directory will be unlinked.
-        let dir = TempDir::new().unwrap();
-        let snap_meta = {
-            // Write out a new snapshot file with two pages.
-            let meta = SnapFileMeta::new(None, TEST_TIMELINE, Lsn(1234));
-            let mut snap = SnapWriter::new(dir.path(), meta).unwrap();
-            // Write the pages out of order, because why not?
-            let page99 = [99u8; 8192];
-            snap.write_page(99, page99).unwrap();
-            let page33 = [33u8; 8192];
-            snap.write_page(33, page33).unwrap();
-            snap.finish().unwrap()
-        };
-
-        assert_eq!(snap_meta.lsn, 1234);
-
-        {
-            // Read the snapshot file and verify the contents.
-            let mut path = PathBuf::from(dir.path());
-            path.push(snap_meta.to_filename());
-            let mut snap = SnapFile::new(&path).unwrap();
-
-            assert_eq!(snap.page_count(), 2);
-            assert!(!snap.has_page(0));
-            assert!(snap.has_page(33));
-            assert!(!snap.has_page(98));
-            assert!(snap.has_page(99));
-            assert!(snap.read_page(0).unwrap().is_none());
-            let page = snap.read_page(33).unwrap().unwrap();
-            assert_eq!(*page.0, [33u8; 8192]);
-            let page = snap.read_page(99).unwrap().unwrap();
-            assert_eq!(*page.0, [99u8; 8192]);
-
-            // Make sure the deserialized metadata matches what we think we wrote.
-            let meta2 = snap.read_meta().unwrap();
-            assert_eq!(snap_meta, meta2);
-        }
-    }
-
-    #[test]
-    fn snap_zero_pages() {
-        // When `dir` goes out of scope the directory will be unlinked.
-        let dir = TempDir::new().unwrap();
-        let snap_meta = {
-            // Write out a new snapshot file with no pages.
-            let meta = SnapFileMeta::new(None, TEST_TIMELINE, Lsn(1234));
-            let snap = SnapWriter::new(dir.path(), meta).unwrap();
-            snap.finish().unwrap()
-        };
-
-        {
-            // Read the snapshot file.
-            let mut path = PathBuf::from(dir.path());
-            path.push(snap_meta.to_filename());
-            let snap = SnapFile::new(&path).unwrap();
-            assert_eq!(snap.page_index.page_count(), 0);
-            assert!(!snap.has_page(0));
-            assert!(!snap.has_page(99));
-            assert!(snap.read_page(0).unwrap().is_none());
-            assert!(snap.read_page(99).unwrap().is_none());
-        }
-    }
-}
--- a/snapfile/src/page.rs
+++ b/snapfile/src/page.rs
@@ -1,42 +0,0 @@
-/// A single 8KB page.
-pub struct Page(pub Box<[u8; 8192]>);
-
-impl Page {
-    /// Create a page by copying bytes from another slice.
-    ///
-    /// This is a copy, not a move. If the caller already has
-    /// an owned array then `From<[u8; 8192]>` can be used instead.
-    pub fn copy_slice(x: &[u8; 8192]) -> Self {
-        Page(Box::new(x.clone()))
-    }
-}
-
-impl Default for Page {
-    fn default() -> Self {
-        Page(Box::new([0u8; 8192]))
-    }
-}
-
-impl From<[u8; 8192]> for Page {
-    fn from(array: [u8; 8192]) -> Self {
-        Page(Box::new(array))
-    }
-}
-
-impl From<Box<[u8; 8192]>> for Page {
-    fn from(heap_array: Box<[u8; 8192]>) -> Self {
-        Page(heap_array)
-    }
-}
-
-impl AsRef<[u8; 8192]> for Page {
-    fn as_ref(&self) -> &[u8; 8192] {
-        self.0.as_ref()
-    }
-}
-
-impl AsMut<[u8; 8192]> for Page {
-    fn as_mut(&mut self) -> &mut [u8; 8192] {
-        self.0.as_mut()
-    }
-}
--- a/snapfile/src/squash.rs
+++ b/snapfile/src/squash.rs
@@ -1,100 +0,0 @@
-use crate::{Page, PageIter, SnapFile, SnapFileMeta, SnapWriter};
-use anyhow::{bail, Result};
-use std::cmp::Ordering;
-use std::path::Path;
-
-// A helper struct that holds an iterator, along with the last
-// value taken from the iterator.
-struct PageStepper<'a> {
-    it: PageIter<'a>,
-    pub cache: Option<(u64, Page)>,
-}
-
-impl<'a> PageStepper<'a> {
-    fn new(snapfile: &'a SnapFile) -> Result<Self> {
-        let mut it = snapfile.all_pages();
-        let cache = it.next().transpose()?;
-        Ok(PageStepper { it, cache })
-    }
-
-    /// Read a new page from the iterator, returning the previous page.
-    fn step(&mut self) -> Result<Option<(u64, Page)>> {
-        let mut next = self.it.next().transpose()?;
-        std::mem::swap(&mut self.cache, &mut next);
-        Ok(next)
-    }
-}
-
-/// Squash two snapshot files into one.
-///
-/// The resulting snapshot will contain all of the pages from both files.
-/// If the same page number is stored in both, it will keep the page from
-/// the newer snapshot.
-///
-/// The name of the resulting file will be automatically generated from
-/// the snapshot metadata.
-pub fn squash(older: &Path, newer: &Path, out_dir: &Path) -> Result<()> {
-    let mut snap1 = SnapFile::new(older)?;
-    let mut snap2 = SnapFile::new(newer)?;
-
-    let meta1 = snap1.read_meta()?;
-    let meta2 = snap2.read_meta()?;
-
-    // Check that snap1 is the predecessor of snap2.
-    match meta2.predecessor {
-        Some(pred) if pred.timeline == meta1.timeline => {}
-        _ => {
-            bail!(
-                "snap file {:?} is not the predecessor of {:?}",
-                &older,
-                &newer,
-            );
-        }
-    }
-
-    // The new combined snapshot will have most fields from meta2 (the later
-    // snapshot), but will have the predecessor from meta1.
-    let new_meta = SnapFileMeta {
-        // There is some danger in squashing snapshots across two timelines,
-        // in that it's possible to get confused about what the history
-        // looks like. Ultimately, it should be possible to squash our way
-        // to a "complete" snapshot (that contains all pages), so this must
-        // be possible.
-        timeline: meta2.timeline,
-        predecessor: meta1.predecessor,
-        lsn: meta2.lsn,
-    };
-    let mut snap_writer = SnapWriter::new(&out_dir, new_meta)?;
-
-    let mut iter1 = PageStepper::new(&snap1)?;
-    let mut iter2 = PageStepper::new(&snap2)?;
-
-    loop {
-        let next_page = match (&iter1.cache, &iter2.cache) {
-            (None, None) => break,
-            (Some(_), None) => iter1.step()?,
-            (None, Some(_)) => iter2.step()?,
-            (Some(x), Some(y)) => {
-                // If these are two different page numbers, then advance the iterator
-                // with the numerically lower number.
-                // If they are the same page number, then store the one from the newer
-                // snapshot, and discard the other (advancing both iterators).
-                match x.0.cmp(&y.0) {
-                    Ordering::Less => iter1.step()?,
-                    Ordering::Greater => iter2.step()?,
-                    Ordering::Equal => {
-                        let _ = iter1.step()?;
-                        iter2.step()?
-                    }
-                }
-            }
-        };
-        // This can't be None, because we would already checked inside the match
-        // statement.
-        let (page_num, page_data) = next_page.unwrap();
-        snap_writer.write_page(page_num, page_data)?;
-    }
-
-    snap_writer.finish()?;
-    Ok(())
-}
--- a/snapfile/src/versioned.rs
+++ b/snapfile/src/versioned.rs
@@ -1,88 +0,0 @@
-//! Versioned data structures for snapshot files
-//!
-//! To ensure that future versions of software can read snapshot files,
-//! all data structures that are serialized into the snapshot files should
-//! live in this module.
-//!
-//! Once released, versioned data structures should never be modified.
-//! Instead, new versions should be created and conversion functions should
-//! be defined using the `FromVersion` trait.
-
-use aversion::{assign_message_ids, UpgradeLatest, Versioned};
-use serde::{Deserialize, Serialize};
-use std::collections::BTreeMap;
-
-// A random constant, to identify this file type.
-pub(crate) const SNAPFILE_MAGIC: u32 = 0x7fb8_38a8;
-
-// Constant chapter numbers
-// FIXME: the bookfile crate should use something better to index, e.g. strings.
-/// Snapshot-specific file metadata
-pub(crate) const CHAPTER_SNAP_META: u64 = 1;
-/// A packed set of 8KB pages.
-pub(crate) const CHAPTER_PAGES: u64 = 2;
-/// An index of pages.
-pub(crate) const CHAPTER_PAGE_INDEX: u64 = 3;
-
-/// Information about the predecessor snapshot.
-///
-/// It contains the snap_id of the predecessor snapshot, and the LSN
-/// of that snapshot.
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-pub struct Predecessor {
-    /// This is the ID number of the predecessor timeline.
-    ///
-    /// This may match the current snapshot's timeline id, but
-    /// it may not (if the precessor was the branch point).
-    pub timeline: [u8; 16],
-
-    /// This is the LSN of the predecessor snapshot.
-    pub lsn: u64,
-}
-
-#[derive(Debug, PartialEq, Serialize, Deserialize, Versioned, UpgradeLatest)]
-pub struct SnapFileMetaV1 {
-    /// This is a unique ID number for this timeline.
-    ///
-    /// This number guarantees that snapshot history is unique.
-    pub timeline: [u8; 16],
-
-    /// Information about the predecessor snapshot.
-    ///
-    /// If `None`, this snapshot is the start of a new database.
-    pub predecessor: Option<Predecessor>,
-
-    /// This is the last LSN stored in this snapshot.
-    pub lsn: u64,
-}
-
-/// A type alias for the latest version of `SnapFileMeta`.
-pub type SnapFileMeta = SnapFileMetaV1;
-
-/// A page location within a file.
-///
-/// Note: this is an opaque value that may not be the true byte offset;
-/// it may be relative to some other location or measured in units other
-/// than bytes.
-#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)]
-#[serde(transparent)]
-pub struct PageLocationV1(pub(crate) u64);
-
-/// A type alias for the latest version of `PageLocation`.
-pub type PageLocation = PageLocationV1;
-
-/// An index from page number to offset within the pages chapter.
-#[derive(Debug, Default, Serialize, Deserialize, Versioned, UpgradeLatest)]
-pub struct PageIndexV1 {
-    /// A map from page number to file offset.
-    pub(crate) map: BTreeMap<u64, PageLocationV1>,
-}
-
-/// A type alias for the latest version of `PageIndex`.
-pub type PageIndex = PageIndexV1;
-
-// Each message gets a unique message id, for tracking by the aversion traits.
-assign_message_ids! {
-    PageIndex: 100,
-    SnapFileMeta: 101,
-}
--- a/test_runner/Pipfile
+++ b/test_runner/Pipfile
@@ -8,6 +8,7 @@ pytest = ">=6.0.0"
 psycopg2 = "*"
 typing-extensions = "*"
 pyjwt = {extras = ["crypto"], version = "*"}
+requests = "*"

 [dev-packages]
 yapf = "*"
--- a/test_runner/Pipfile.lock
+++ b/test_runner/Pipfile.lock
@@ -1,7 +1,7 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "f60a966726bcc19670402ad3fa57396b5dacf0a027544418ceb7cc0d42d94a52"
+            "sha256": "b666740289d9c82797e5c39b2a7f0074c865c9183ee878ce4fa5cda7928506ea"
        },
        "pipfile-spec": 6,
        "requires": {
@@ -24,6 +24,13 @@
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
            "version": "==21.2.0"
        },
+        "certifi": {
+            "hashes": [
+                "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee",
+                "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"
+            ],
+            "version": "==2021.5.30"
+        },
        "cffi": {
            "hashes": [
                "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d",
@@ -74,6 +81,14 @@
            ],
            "version": "==1.14.6"
        },
+        "charset-normalizer": {
+            "hashes": [
+                "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b",
+                "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==2.0.4"
+        },
        "cryptography": {
            "hashes": [
                "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d",
@@ -85,12 +100,22 @@
                "sha256:3d8427734c781ea5f1b41d6589c293089704d4759e34597dce91014ac125aad1",
                "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177",
                "sha256:8e56e16617872b0957d1c9742a3f94b43533447fd78321514abbe7db216aa250",
+                "sha256:b01fd6f2737816cb1e08ed4807ae194404790eac7ad030b34f2ce72b332f5586",
+                "sha256:bf40af59ca2465b24e54f671b2de2c59257ddc4f7e5706dbd6930e26823668d3",
                "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca",
                "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d",
                "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9"
            ],
            "version": "==3.4.7"
        },
+        "idna": {
+            "hashes": [
+                "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a",
+                "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==3.2"
+        },
        "iniconfig": {
            "hashes": [
                "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
@@ -172,6 +197,14 @@
            "index": "pypi",
            "version": "==6.2.4"
        },
+        "requests": {
+            "hashes": [
+                "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24",
+                "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"
+            ],
+            "index": "pypi",
+            "version": "==2.26.0"
+        },
        "toml": {
            "hashes": [
                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
@@ -188,6 +221,14 @@
            ],
            "index": "pypi",
            "version": "==3.10.0.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4",
+                "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.26.6"
        }
    },
    "develop": {
--- a/test_runner/README.md
+++ b/test_runner/README.md
@@ -7,7 +7,8 @@ Prerequisites:
 - Dependencies: install them via `pipenv install`. Note that Debian/Ubuntu
  packages are stale, as it commonly happens, so manual installation is not
  recommended.
-  Run `pipenv shell` to activate the venv.
+  Run `pipenv shell` to activate the venv or use `pipenv run` to run a single
+  command in the venv, e.g. `pipenv run pytests`.
 - Zenith and Postgres binaries
    - See the root README.md for build directions
    - Tests can be run from the git tree; or see the environment variables
@@ -19,10 +20,11 @@ Prerequisites:

 The tests are divided into a few batches, such that each batch takes roughly
 the same amount of time. The batches can be run in parallel, to minimize total
-runtime. Currently, there are only two batches:
+runtime.

- test_batch_pg_regress: Runs PostgreSQL regression tests
- test_others: All other tests
+- batch_pg_regress: Runs PostgreSQL regression tests
+- batch_wraparound: Runs xid wraparound tests.
+- batch_others: All other tests

 ### Running the tests

--- a/test_runner/batch_others/test_auth.py
+++ b/test_runner/batch_others/test_auth.py
@@ -1,61 +1,17 @@

 from contextlib import closing
-from pathlib import Path
 from uuid import uuid4
-from dataclasses import dataclass
-import jwt
 import psycopg2
 from fixtures.zenith_fixtures import Postgres, ZenithCli, ZenithPageserver
 import pytest


-@pytest.fixture
-def pageserver_auth_enabled(zenith_cli: ZenithCli):
-    with ZenithPageserver(zenith_cli).init(enable_auth=True).start() as ps:
-        # For convenience in tests, create a branch from the freshly-initialized cluster.
-        zenith_cli.run(["branch", "empty", "main"])
-        yield ps
-
-
-@dataclass
-class AuthKeys:
-    pub: bytes
-    priv: bytes
-
-    def generate_management_token(self):
-        token = jwt.encode({"scope": "pageserverapi"}, self.priv, algorithm="RS256")
-
-        # jwt.encode can return 'bytes' or 'str', depending on Python version or type
-        # hinting or something (not sure what). If it returned 'bytes', convert it to 'str'
-        # explicitly.
-        if isinstance(token, bytes):
-            token = token.decode()
-
-        return token
-
-    def generate_tenant_token(self, tenant_id):
-        token = jwt.encode({"scope": "tenant", "tenant_id": tenant_id}, self.priv, algorithm="RS256")
-
-        if isinstance(token, bytes):
-            token = token.decode()
-
-        return token
-
-
-@pytest.fixture
-def auth_keys(repo_dir: str) -> AuthKeys:
-    # TODO probably this should be specified in cli config and used in tests for single source of truth
-    pub = (Path(repo_dir) / 'auth_public_key.pem').read_bytes()
-    priv = (Path(repo_dir) / 'auth_private_key.pem').read_bytes()
-    return AuthKeys(pub=pub, priv=priv)
-
-
-def test_pageserver_auth(pageserver_auth_enabled: ZenithPageserver, auth_keys: AuthKeys):
+def test_pageserver_auth(pageserver_auth_enabled: ZenithPageserver):
    ps = pageserver_auth_enabled

-    tenant_token = auth_keys.generate_tenant_token(ps.initial_tenant)
-    invalid_tenant_token = auth_keys.generate_tenant_token(uuid4().hex)
-    management_token = auth_keys.generate_management_token()
+    tenant_token = ps.auth_keys.generate_tenant_token(ps.initial_tenant)
+    invalid_tenant_token = ps.auth_keys.generate_tenant_token(uuid4().hex)
+    management_token = ps.auth_keys.generate_management_token()

    # this does not invoke auth check and only decodes jwt and checks it for validity
    # check both tokens
@@ -86,12 +42,11 @@ def test_compute_auth_to_pageserver(
    pageserver_auth_enabled: ZenithPageserver,
    repo_dir: str,
    with_wal_acceptors: bool,
-    auth_keys: AuthKeys,
 ):
    ps = pageserver_auth_enabled
    # since we are in progress of refactoring protocols between compute safekeeper and page server
    # use hardcoded management token in safekeeper
-    management_token = auth_keys.generate_management_token()
+    management_token = ps.auth_keys.generate_management_token()

    branch = f"test_compute_auth_to_pageserver{with_wal_acceptors}"
    zenith_cli.run(["branch", branch, "empty"])
--- a/test_runner/batch_others/test_createdb.py
+++ b/test_runner/batch_others/test_createdb.py
@@ -1,38 +0,0 @@
-from contextlib import closing
-from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory, ZenithCli
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-
-#
-# Test CREATE DATABASE when there have been relmapper changes
-#
-def test_createdb(
-    zenith_cli: ZenithCli,
-    pageserver: ZenithPageserver,
-    postgres: PostgresFactory,
-    pg_bin,
-):
-    zenith_cli.run(["branch", "test_createdb", "empty"])
-
-    pg = postgres.create_start('test_createdb')
-    print("postgres is running on 'test_createdb' branch")
-
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            # Cause a 'relmapper' change in the original branch
-            cur.execute('VACUUM FULL pg_class')
-
-            cur.execute('CREATE DATABASE foodb')
-
-            cur.execute('SELECT pg_current_wal_insert_lsn()')
-            lsn = cur.fetchone()[0]
-
-    # Create a branch
-    zenith_cli.run(["branch", "test_createdb2", "test_createdb@" + lsn])
-
-    pg2 = postgres.create_start('test_createdb2')
-
-    # Test that you can connect to the new database on both branches
-    for db in (pg, pg2):
-        db.connect(dbname='foodb').close()
--- a/test_runner/batch_others/test_createdropdb.py
+++ b/test_runner/batch_others/test_createdropdb.py
@@ -0,0 +1,96 @@
+import os
+import pathlib
+
+from contextlib import closing
+from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory, ZenithCli
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+#
+# Test CREATE DATABASE when there have been relmapper changes
+#
+def test_createdb(
+    zenith_cli: ZenithCli,
+    pageserver: ZenithPageserver,
+    postgres: PostgresFactory,
+    pg_bin,
+):
+    zenith_cli.run(["branch", "test_createdb", "empty"])
+
+    pg = postgres.create_start('test_createdb')
+    print("postgres is running on 'test_createdb' branch")
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            # Cause a 'relmapper' change in the original branch
+            cur.execute('VACUUM FULL pg_class')
+
+            cur.execute('CREATE DATABASE foodb')
+
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn = cur.fetchone()[0]
+
+    # Create a branch
+    zenith_cli.run(["branch", "test_createdb2", "test_createdb@" + lsn])
+
+    pg2 = postgres.create_start('test_createdb2')
+
+    # Test that you can connect to the new database on both branches
+    for db in (pg, pg2):
+        db.connect(dbname='foodb').close()
+
+#
+# Test DROP DATABASE
+#
+def test_dropdb(
+    zenith_cli: ZenithCli,
+    pageserver: ZenithPageserver,
+    postgres: PostgresFactory,
+    pg_bin,
+):
+    zenith_cli.run(["branch", "test_dropdb", "empty"])
+
+    pg = postgres.create_start('test_dropdb')
+    print("postgres is running on 'test_dropdb' branch")
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('CREATE DATABASE foodb')
+
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn_before_drop = cur.fetchone()[0]
+
+            cur.execute("SELECT oid FROM pg_database WHERE datname='foodb';")
+            dboid = cur.fetchone()[0]
+
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('DROP DATABASE foodb')
+
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn_after_drop = cur.fetchone()[0]
+
+
+    # Create two branches before and after database drop.
+    zenith_cli.run(["branch", "test_before_dropdb", "test_dropdb@" + lsn_before_drop])
+    pg_before = postgres.create_start('test_before_dropdb')
+
+    zenith_cli.run(["branch", "test_after_dropdb", "test_dropdb@" + lsn_after_drop])
+    pg_after = postgres.create_start('test_after_dropdb')
+
+    # Test that database exists on the branch before drop
+    pg_before.connect(dbname='foodb').close()
+
+    # Test that database subdir exists on the branch before drop
+    dbpath = pathlib.Path(pg_before.pgdata_dir) / 'base' / str(dboid)
+    print(dbpath)
+
+    assert os.path.isdir(dbpath) == True
+
+    # Test that database subdir doesn't exist on the branch after drop
+    dbpath = pathlib.Path(pg_after.pgdata_dir) / 'base' / str(dboid)
+    print(dbpath)
+
+    assert os.path.isdir(dbpath) == False
--- a/test_runner/batch_others/test_gc.py
+++ b/test_runner/batch_others/test_gc.py
@@ -1,105 +0,0 @@
-import pytest
-
-from contextlib import closing
-from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
-import psycopg2.extras
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-#
-# Test Garbage Collection of old page versions.
-#
-# This test is pretty tightly coupled with the current implementation of page version storage
-# and garbage collection in object_repository.rs.
-#
-@pytest.mark.skip(reason=""""
-    Current GC test is flaky and overly strict. Since we are migrating to the layered repo format
-    with different GC implementation let's just silence this test for now. This test only
-    works with the RocksDB implementation.
-""")
-def test_gc(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
-    zenith_cli.run(["branch", "test_gc", "empty"])
-    pg = postgres.create_start('test_gc')
-
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            with closing(pageserver.connect()) as psconn:
-                with psconn.cursor(cursor_factory = psycopg2.extras.DictCursor) as pscur:
-
-                    # Get the timeline ID of our branch. We need it for the 'do_gc' command
-                    cur.execute("SHOW zenith.zenith_timeline")
-                    timeline = cur.fetchone()[0]
-
-                    # Create a test table
-                    cur.execute("CREATE TABLE foo(x integer)")
-
-                    # Run GC, to clear out any old page versions left behind in the catalogs by
-                    # the CREATE TABLE command. We want to have a clean slate with no garbage
-                    # before running the actual tests below, otherwise the counts won't match
-                    # what we expect.
-                    print("Running GC before test")
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    # remember the number of relations
-                    n_relations = row['n_relations']
-                    assert n_relations > 0
-
-                    # Insert a row. The first insert will also create a metadata entry for the
-                    # relation, with size == 1 block. Hence, bump up the expected relation count.
-                    n_relations += 1
-                    print("Inserting one row and running GC")
-                    cur.execute("INSERT INTO foo VALUES (1)")
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    assert row['n_relations'] == n_relations
-                    assert row['dropped'] == 0
-                    assert row['truncated'] == 31
-                    assert row['deleted'] == 4
-
-                    # Insert two more rows and run GC.
-                    print("Inserting two more rows and running GC")
-                    cur.execute("INSERT INTO foo VALUES (2)")
-                    cur.execute("INSERT INTO foo VALUES (3)")
-
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    assert row['n_relations'] == n_relations
-                    assert row['dropped'] == 0
-                    assert row['truncated'] == 31
-                    assert row['deleted'] == 4
-
-                    # Insert one more row. It creates one more page version, but doesn't affect the
-                    # relation size.
-                    print("Inserting one more row")
-                    cur.execute("INSERT INTO foo VALUES (3)")
-
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    assert row['n_relations'] == n_relations
-                    assert row['dropped'] == 0
-                    assert row['truncated'] == 31
-                    assert row['deleted'] == 2
-
-                    # Run GC again, with no changes in the database. Should not remove anything.
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    assert row['n_relations'] == n_relations
-                    assert row['dropped'] == 0
-                    assert row['truncated'] == 31
-                    assert row['deleted'] == 0
-
-                    #
-                    # Test DROP TABLE checks that relation data and metadata was deleted by GC from object storage
-                    #
-                    cur.execute("DROP TABLE foo")
-
-                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
-                    row = pscur.fetchone()
-                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
-                    # Each relation fork is counted separately, hence 3.
-                    assert row['dropped'] == 3
--- a/test_runner/batch_others/test_pageserver_api.py
+++ b/test_runner/batch_others/test_pageserver_api.py
@@ -1,19 +1,20 @@
 import json
-import uuid
+from uuid import uuid4
 import pytest
 import psycopg2
-from fixtures.zenith_fixtures import ZenithPageserver
+import requests
+from fixtures.zenith_fixtures import ZenithPageserver, ZenithPageserverHttpClient

 pytest_plugins = ("fixtures.zenith_fixtures")


-def test_status(pageserver):
+def test_status_psql(pageserver):
    assert pageserver.safe_psql('status') == [
        ('hello world', ),
    ]


-def test_branch_list(pageserver: ZenithPageserver, zenith_cli):
+def test_branch_list_psql(pageserver: ZenithPageserver, zenith_cli):
    # Create a branch for us
    zenith_cli.run(["branch", "test_branch_list_main", "empty"])

@@ -52,7 +53,7 @@ def test_branch_list(pageserver: ZenithPageserver, zenith_cli):
    conn.close()


-def test_tenant_list(pageserver: ZenithPageserver, zenith_cli):
+def test_tenant_list_psql(pageserver: ZenithPageserver, zenith_cli):
    res = zenith_cli.run(["tenant", "list"])
    res.check_returncode()
    tenants = res.stdout.splitlines()
@@ -66,7 +67,7 @@ def test_tenant_list(pageserver: ZenithPageserver, zenith_cli):
        cur.execute(f'tenant_create {pageserver.initial_tenant}')

    # create one more tenant
-    tenant1 = uuid.uuid4().hex
+    tenant1 = uuid4().hex
    cur.execute(f'tenant_create {tenant1}')

    cur.execute('tenant_list')
@@ -74,3 +75,32 @@ def test_tenant_list(pageserver: ZenithPageserver, zenith_cli):
    # compare tenants list
    new_tenants = sorted(json.loads(cur.fetchone()[0]))
    assert sorted([pageserver.initial_tenant, tenant1]) == new_tenants
+
+
+def check_client(client: ZenithPageserverHttpClient, initial_tenant: str):
+    client.check_status()
+
+    # check initial tenant is there
+    assert initial_tenant in set(client.tenant_list())
+
+    # create new tenant and check it is also there
+    tenant_id = uuid4()
+    client.tenant_create(tenant_id)
+    assert tenant_id.hex in set(client.tenant_list())
+
+    # create branch
+    branch_name = uuid4().hex
+    client.branch_create(tenant_id, branch_name, "main")
+
+    # check it is there
+    assert branch_name in {b['name'] for b in client.branch_list(tenant_id)}
+
+
+def test_pageserver_http_api_client(pageserver: ZenithPageserver):
+    client = pageserver.http_client()
+    check_client(client, pageserver.initial_tenant)
+
+
+def test_pageserver_http_api_client_auth_enabled(pageserver_auth_enabled: ZenithPageserver):
+    client = pageserver_auth_enabled.http_client(auth_token=pageserver_auth_enabled.auth_keys.generate_management_token())
+    check_client(client, pageserver_auth_enabled.initial_tenant)
--- a/test_runner/batch_others/test_pageserver_restart.py
+++ b/test_runner/batch_others/test_pageserver_restart.py
@@ -0,0 +1,66 @@
+import pytest
+import random
+import time
+
+from contextlib import closing
+from multiprocessing import Process, Value
+from fixtures.zenith_fixtures import WalAcceptorFactory, ZenithPageserver, PostgresFactory
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+# Check that dead minority doesn't prevent the commits: execute insert n_inserts
+# times, with fault_probability chance of getting a wal acceptor down or up
+# along the way. 2 of 3 are always alive, so the work keeps going.
+def test_pageserver_restart(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory: WalAcceptorFactory):
+
+    # One safekeeper is enough for this test.
+    wa_factory.start_n_new(1)
+
+    zenith_cli.run(["branch", "test_pageserver_restart", "empty"])
+    pg = postgres.create_start('test_pageserver_restart',
+                               wal_acceptors=wa_factory.get_connstrs())
+
+    pg_conn = pg.connect()
+    cur = pg_conn.cursor()
+
+    # Create table, and insert some rows. Make it big enough that it doesn't fit in
+    # shared_buffers, otherwise the SELECT after restart will just return answer
+    # from shared_buffers without hitting the page server, which defeats the point
+    # of this test.
+    cur.execute('CREATE TABLE foo (t text)')
+    cur.execute('''
+        INSERT INTO foo
+            SELECT 'long string to consume some space' || g
+            FROM generate_series(1, 100000) g
+    ''')
+
+    # Verify that the table is larger than shared_buffers
+    cur.execute('''
+        select setting::int * pg_size_bytes(unit) as shared_buffers, pg_relation_size('foo') as tbl_ize
+        from pg_settings where name = 'shared_buffers'
+    ''')
+    row = cur.fetchone()
+    print("shared_buffers is {}, table size {}", row[0], row[1]);
+    assert int(row[0]) < int(row[1])
+
+    # Stop and restart pageserver. This is a more or less graceful shutdown, although
+    # the page server doesn't currently have a shutdown routine so there's no difference
+    # between stopping and crashing.
+    pageserver.stop();
+    pageserver.start();
+
+    # Stopping the pageserver breaks the connection from the postgres backend to
+    # the page server, and causes the next query on the connection to fail. Start a new
+    # postgres connection too, to avoid that error. (Ideally, the compute node would
+    # handle that and retry internally, without propagating the error to the user, but
+    # currently it doesn't...)
+    pg_conn = pg.connect()
+    cur = pg_conn.cursor()
+
+    cur.execute("SELECT count(*) FROM foo")
+    assert cur.fetchone() == (100000, )
+
+    # Stop the page server by force, and restart it
+    pageserver.stop();
+    pageserver.start();
+
--- a/test_runner/batch_others/test_twophase.py
+++ b/test_runner/batch_others/test_twophase.py
@@ -1,3 +1,5 @@
+import os
+
 from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver


@@ -28,24 +30,59 @@ def test_twophase(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFa
    cur.execute("INSERT INTO foo VALUES ('two')")
    cur.execute("PREPARE TRANSACTION 'insert_two'")

+    # Prepare a transaction that will insert a row
+    cur.execute('BEGIN')
+    cur.execute("INSERT INTO foo VALUES ('three')")
+    cur.execute("PREPARE TRANSACTION 'insert_three'")
+
+    # Prepare another transaction that will insert a row
+    cur.execute('BEGIN')
+    cur.execute("INSERT INTO foo VALUES ('four')")
+    cur.execute("PREPARE TRANSACTION 'insert_four'")
+
+    # On checkpoint state data copied to files in
+    # pg_twophase directory and fsynced
+    cur.execute('CHECKPOINT')
+
+    twophase_files = os.listdir(pg.pg_twophase_dir_path())
+    print(twophase_files)
+    assert len(twophase_files) == 4
+
+    cur.execute("COMMIT PREPARED 'insert_three'")
+    cur.execute("ROLLBACK PREPARED 'insert_four'")
+    cur.execute('CHECKPOINT')
+
+    twophase_files = os.listdir(pg.pg_twophase_dir_path())
+    print(twophase_files)
+    assert len(twophase_files) == 2
+
    # Create a branch with the transaction in prepared state
    zenith_cli.run(["branch", "test_twophase_prepared", "test_twophase"])

-    pg2 = postgres.create_start(
+    # Create compute node, but don't start.
+    # We want to observe pgdata before postgres starts
+    pg2 = postgres.create(
        'test_twophase_prepared',
        config_lines=['max_prepared_transactions=5'],
    )
+
+    # Check that we restored only needed twophase files
+    twophase_files2 = os.listdir(pg2.pg_twophase_dir_path())
+    print(twophase_files2)
+    assert twophase_files2.sort() == twophase_files.sort()
+
+    pg2 = pg2.start()
    conn2 = pg2.connect()
    cur2 = conn2.cursor()

-    # On the new branch, commit one of the prepared transactions, abort the other one.
+    # On the new branch, commit one of the prepared transactions,
+    # abort the other one.
    cur2.execute("COMMIT PREPARED 'insert_one'")
    cur2.execute("ROLLBACK PREPARED 'insert_two'")

    cur2.execute('SELECT * FROM foo')
-    assert cur2.fetchall() == [('one', )]
+    assert cur2.fetchall() == [('one',), ('three',)]

-    # Neither insert is visible on the original branch, the transactions are still
-    # in prepared state there.
+    # Only one committed insert is visible on the original branch
    cur.execute('SELECT * FROM foo')
-    assert cur.fetchall() == []
+    assert cur.fetchall() == [('three',)]
--- a/test_runner/batch_others/test_vm_bits.py
+++ b/test_runner/batch_others/test_vm_bits.py
@@ -0,0 +1,79 @@
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+#
+# Test that the VM bit is cleared correctly at a HEAP_DELETE and
+# HEAP_UPDATE record.
+#
+def test_vm_bit_clear(pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin, zenith_cli, base_dir):
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_vm_bit_clear", "empty"])
+    pg = postgres.create_start('test_vm_bit_clear')
+
+    print("postgres is running on 'test_vm_bit_clear' branch")
+    pg_conn = pg.connect()
+    cur = pg_conn.cursor()
+
+    # Install extension containing function needed for test
+    cur.execute('CREATE EXTENSION zenith_test_utils')
+
+    # Create a test table and freeze it to set the VM bit.
+    cur.execute('CREATE TABLE vmtest_delete (id integer PRIMARY KEY)')
+    cur.execute('INSERT INTO vmtest_delete VALUES (1)')
+    cur.execute('VACUUM FREEZE vmtest_delete')
+
+    cur.execute('CREATE TABLE vmtest_update (id integer PRIMARY KEY)')
+    cur.execute('INSERT INTO vmtest_update SELECT g FROM generate_series(1, 1000) g')
+    cur.execute('VACUUM FREEZE vmtest_update')
+
+    # DELETE and UDPATE the rows.
+    cur.execute('DELETE FROM vmtest_delete WHERE id = 1')
+    cur.execute('UPDATE vmtest_update SET id = 5000 WHERE id = 1')
+
+    # Branch at this point, to test that later
+    zenith_cli.run(["branch", "test_vm_bit_clear_new", "test_vm_bit_clear"])
+
+    # Clear the buffer cache, to force the VM page to be re-fetched from
+    # the page server
+    cur.execute('SELECT clear_buffer_cache()')
+
+    # Check that an index-only scan doesn't see the deleted row. If the
+    # clearing of the VM bit was not replayed correctly, this would incorrectly
+    # return deleted row.
+    cur.execute('''
+    set enable_seqscan=off;
+    set enable_indexscan=on;
+    set enable_bitmapscan=off;
+    ''')
+
+    cur.execute('SELECT * FROM vmtest_delete WHERE id = 1')
+    assert(cur.fetchall() == []);
+    cur.execute('SELECT * FROM vmtest_update WHERE id = 1')
+    assert(cur.fetchall() == []);
+
+    cur.close()
+
+
+    # Check the same thing on the branch that we created right after the DELETE
+    #
+    # As of this writing, the code in smgrwrite() creates a full-page image whenever
+    # a dirty VM page is evicted. If the VM bit was not correctly cleared by the
+    # earlier WAL record, the full-page image hides the problem. Starting a new
+    # server at the right point-in-time avoids that full-page image.
+    pg_new = postgres.create_start('test_vm_bit_clear_new')
+
+    print("postgres is running on 'test_vm_bit_clear_new' branch")
+    pg_new_conn = pg_new.connect()
+    cur_new = pg_new_conn.cursor()
+
+    cur_new.execute('''
+    set enable_seqscan=off;
+    set enable_indexscan=on;
+    set enable_bitmapscan=off;
+    ''')
+
+    cur_new.execute('SELECT * FROM vmtest_delete WHERE id = 1')
+    assert(cur_new.fetchall() == []);
+    cur_new.execute('SELECT * FROM vmtest_update WHERE id = 1')
+    assert(cur_new.fetchall() == []);
--- a/test_runner/batch_wraparound/test_xid_wraparound.py
+++ b/test_runner/batch_wraparound/test_xid_wraparound.py
@@ -0,0 +1,139 @@
+import time
+import os
+
+from contextlib import closing
+
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+#
+# Test compute node start after xid wraparound
+# Note: this is a long test (> 15 min)
+#
+def test_xid_wraparound(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory,
+                        pg_bin):
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_xid_wraparound", "empty"])
+
+    # set agressive autovacuum
+    config = [
+        'autovacuum_max_workers=10', 'autovacuum_vacuum_threshold=0',
+        'autovacuum_vacuum_insert_threshold=0', 'autovacuum_vacuum_cost_delay=0',
+        'autovacuum_vacuum_cost_limit=10000', 'autovacuum_naptime =1s'
+    ]
+
+    pg = postgres.create_start('test_xid_wraparound', config_lines=config)
+    print('postgres is running on test_xid_wraparound branch')
+
+    # Install extension containing function needed for test
+    pg.safe_psql('CREATE EXTENSION zenith_test_utils')
+
+    # Consume many xids
+    for i in range(4):
+        with closing(pg.connect()) as conn:
+            with conn.cursor() as cur:
+                cur.execute('select test_consume_xids(1000*1000*1000);')
+                print('xids consumed')
+
+                # This query is just for debugging
+                cur.execute('select datname, datfrozenxid from pg_database;')
+                res = cur.fetchall()
+                print(res)
+
+                # call a checkpoint to trigger TruncateSubtrans
+                cur.execute('CHECKPOINT;')
+
+                # ensure WAL flush
+                cur.execute('select txid_current();')
+
+                # This query is just for debugging
+                cur.execute('select pg_current_wal_insert_lsn(),'
+                            'next_xid, oldest_xid, oldest_xid_dbid,'
+                            'oldest_active_xid from pg_control_checkpoint();')
+                res = cur.fetchone()
+                print(res)
+
+    # This run triggers xid wraparound
+    pg.safe_psql('select test_consume_xids(1000*1000*1000);')
+
+    print('xids consumed. expect wraparound')
+
+    # call a checkpoint to trigger TruncateSubtrans
+    pg.safe_psql('CHECKPOINT;')
+    # ensure WAL flush
+    txid = pg.safe_psql('select txid_current();')
+    print(txid)
+
+    # wait for autovacuum to truncate the pg_xact
+    i = 0
+    pg_xact_0FFF_path = os.path.join(pg.pg_xact_dir_path(), '0FFF')
+    while os.path.isfile(pg_xact_0FFF_path) and i < 10:
+        print("file exists. wait for truncation. pg_xact_0FFF_path = " + pg_xact_0FFF_path)
+        time.sleep(5)
+        i += 1
+
+    # check that compute node doesn't contain the truncated segment
+    assert os.path.isfile(pg_xact_0FFF_path) == False
+
+    # A bunch of sanity checks for pg_control_checkpoint() fields
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('select pg_current_wal_insert_lsn(),'
+                        'next_xid, oldest_xid, oldest_xid_dbid,'
+                        'oldest_active_xid from pg_control_checkpoint();')
+            res = cur.fetchone()
+            print(res)
+
+    next_full_xid = res[1].split(':')
+    next_xid_epoch = int(next_full_xid[0])
+    next_xid_xid = int(next_full_xid[1])
+    oldest_xid = int(res[2])
+    oldest_xid_dbid = int(res[3])
+
+    # ensure that wraparound happened
+    assert next_xid_xid < oldest_xid
+    # ensure that wraparound happened. check that next_xid epoch > 0
+    assert next_xid_epoch == 1
+
+    # create new branch after xid wraparound and start a compute node on it
+    lsn_after_wraparound = res[0]
+    print('create branch at lsn_after_wraparound ' + lsn_after_wraparound)
+    zenith_cli.run(
+        ["branch", "test_xid_wraparound_new", "test_xid_wraparound@" + lsn_after_wraparound])
+
+    pg2 = postgres.create_start('test_xid_wraparound_new')
+    print('postgres is running on test_xid_wraparound_new branch')
+
+    with closing(pg2.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('select pg_current_wal_insert_lsn(),'
+                        'next_xid, oldest_xid, oldest_xid_dbid,'
+                        'oldest_active_xid from pg_control_checkpoint();')
+            res2 = cur.fetchone()
+            print(res2)
+
+    next_full_xid2 = res2[1].split(':')
+    next_xid_epoch2 = int(next_full_xid2[0])
+    next_xid_xid2 = int(next_full_xid2[1])
+    oldest_xid2 = int(res2[2])
+    oldest_xid_dbid2 = int(res2[3])
+
+    # ensure that wraparound happened
+    assert next_xid_epoch2 == 1
+    assert next_xid_xid2 < oldest_xid2
+
+    # check that new compute node doesn't contain truncated clog segment
+    pg_xact_0FFF_path_new = os.path.join(pg2.pg_xact_dir_path(), '0FFF')
+    print("pg_xact_0FFF_path_new = " + pg_xact_0FFF_path_new)
+    assert os.path.isfile(pg_xact_0FFF_path_new) == False
+
+    # ensure that pg_control_checkpoint() fields are restored correctly
+    assert next_xid_xid <= next_xid_xid2
+    assert oldest_xid == oldest_xid2
+    assert oldest_xid_dbid == oldest_xid_dbid2
+    #TODO check oldest_active_xid?
+
+    # TODO Test transaction visibility like in vendor/postgres/src/test/regress/sql/xid.sql
+    # Transactions from the future shouldn't be visible
--- a/test_runner/fixtures/benchmark_fixture.py
+++ b/test_runner/fixtures/benchmark_fixture.py
@@ -0,0 +1,162 @@
+from pprint import pprint
+
+import os
+import timeit
+import pathlib
+import uuid
+import psycopg2
+import pytest
+from _pytest.config import Config
+from _pytest.runner import CallInfo
+from _pytest.terminal import TerminalReporter
+import shutil
+import signal
+import subprocess
+import time
+
+from contextlib import contextmanager
+from contextlib import closing
+from pathlib import Path
+from dataclasses import dataclass
+
+# Type-related stuff
+from psycopg2.extensions import connection as PgConnection
+from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar, cast
+from typing_extensions import Literal
+
+from .utils import (get_self_dir, mkdir_if_needed, subprocess_capture)
+
+"""
+This file contains fixtures for micro-benchmarks.
+
+To use, declare the 'zenbenchmark' fixture in the test function. Run the
+bencmark, and then record the result by calling zenbenchmark.record. For example:
+
+import timeit
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
+
+def test_mybench(postgres: PostgresFactory, pageserver: ZenithPageserver, zenbenchmark):
+
+    # Initialize the test
+    ...
+    
+    # Run the test, timing how long it takes
+    with zenbenchmark.record_duration('test_query'):
+        cur.execute('SELECT test_query(...)')
+
+    # Record another measurement
+    zenbenchmark.record('speed_of_light', 300000, 'km/s')
+
+
+You can measure multiple things in one test, and record each one with a separate
+call to zenbenchmark. For example, you could time the bulk loading that happens
+in the test initialization, or measure disk usage after the test query.
+
+"""
+
+
+# All the results are collected in this list, as a tuple:
+# (test_name: str, metric_name: str, metric_value: float, unit: str)
+#
+# TODO: It would perhaps be better to store the results as additional
+# properties in the pytest TestReport objects, to make them visible to
+# other pytest tools.
+global zenbenchmark_results
+zenbenchmark_results = []
+
+class ZenithBenchmarkResults:
+    """ An object for recording benchmark results. """
+    def __init__(self):
+        self.results = []
+
+    def record(self, test_name: str, metric_name: str, metric_value: float, unit: str):
+        """
+        Record a benchmark result.
+        """
+
+        self.results.append((test_name, metric_name, metric_value, unit))
+
+# Sesssion scope fixture that initializes the results object
+@pytest.fixture(autouse=True, scope='session')
+def zenbenchmark_global(request) -> Iterator[ZenithBenchmarkResults]:
+    """
+    This is a python decorator for benchmark fixtures
+    """
+    global zenbenchmark_results
+    zenbenchmark_results = ZenithBenchmarkResults()
+
+    yield zenbenchmark_results
+
+class ZenithBenchmarker:
+    """
+    An object for recording benchmark results. This is created for each test
+    function by the zenbenchmark fixture
+    """
+    def __init__(self, results, request):
+        self.results = results
+        self.request = request
+
+    def record(self, metric_name: str, metric_value: float, unit: str):
+        """
+        Record a benchmark result.
+        """
+        self.results.record(self.request.node.name, metric_name, metric_value, unit)
+
+
+    @contextmanager
+    def record_duration(self, metric_name):
+        """
+        Record a duration. Usage:
+        
+        with zenbenchmark.record_duration('foobar_runtime'):
+            foobar()   # measure this
+        
+        """
+        start = timeit.default_timer()
+        yield
+        end = timeit.default_timer()
+
+        self.results.record(self.request.node.name, metric_name, end - start, 's')
+
+@pytest.fixture(scope='function')
+def zenbenchmark(zenbenchmark_global, request) -> Iterator[ZenithBenchmarker]:
+    """
+    This is a python decorator for benchmark fixtures. It contains functions for
+    recording measurements, and prints them out at the end.
+    """
+    benchmarker = ZenithBenchmarker(zenbenchmark_global, request)
+    yield benchmarker
+
+
+# Hook to print the results at the end
+@pytest.hookimpl(hookwrapper=True)
+def pytest_terminal_summary(
+    terminalreporter: TerminalReporter, exitstatus: int, config: Config
+):
+    yield
+
+    global zenbenchmark_results
+
+    if not zenbenchmark_results:
+        return
+
+    terminalreporter.section('Benchmark results', "-")
+
+    for result in zenbenchmark_results.results:
+        func = result[0]
+        metric_name = result[1]
+        metric_value = result[2]
+        unit = result[3]
+
+        terminalreporter.write("{}.{}: ".format(func, metric_name))
+
+        if unit == 'MB':
+            terminalreporter.write("{0:,.0f}".format(metric_value), green=True)
+        elif unit == 's':
+            terminalreporter.write("{0:,.3f}".format(metric_value), green=True)
+        else:
+            terminalreporter.write("{0:,.4f}".format(metric_value), green=True)
+
+        terminalreporter.line(" {}".format(unit))
--- a/test_runner/fixtures/zenith_fixtures.py
+++ b/test_runner/fixtures/zenith_fixtures.py
@@ -1,6 +1,9 @@
+from dataclasses import dataclass
+from functools import cached_property
 import os
 import pathlib
 import uuid
+import jwt
 import psycopg2
 import pytest
 import shutil
@@ -17,6 +20,8 @@ from psycopg2.extensions import connection as PgConnection
 from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar, cast
 from typing_extensions import Literal

+import requests
+
 from .utils import (get_self_dir, mkdir_if_needed, subprocess_capture)
 """
 This file contains pytest fixtures. A fixture is a test resource that can be
@@ -42,7 +47,8 @@ Fn = TypeVar('Fn', bound=Callable[..., Any])
 DEFAULT_OUTPUT_DIR = 'test_output'
 DEFAULT_POSTGRES_DIR = 'tmp_install'

-DEFAULT_PAGESERVER_PORT = 64000
+DEFAULT_PAGESERVER_PG_PORT = 64000
+DEFAULT_PAGESERVER_HTTP_PORT = 9898


 def determine_scope(fixture_name: str, config: Any) -> str:
@@ -141,6 +147,7 @@ class ZenithCli:
        assert os.path.isdir(binpath)
        self.binpath = binpath
        self.bin_zenith = os.path.join(binpath, 'zenith')
+        self.repo_dir = repo_dir
        self.env = os.environ.copy()
        self.env['ZENITH_REPO_DIR'] = repo_dir
        self.env['POSTGRES_DISTRIB_DIR'] = pg_distrib_dir
@@ -175,13 +182,87 @@ def zenith_cli(zenith_binpath: str, repo_dir: str, pg_distrib_dir: str) -> Zenit
    return ZenithCli(zenith_binpath, repo_dir, pg_distrib_dir)


+class ZenithPageserverHttpClient(requests.Session):
+    def __init__(self, port: int, auth_token: Optional[str] = None) -> None:
+        super().__init__()
+        self.port = port
+        self.auth_token = auth_token
+
+        if auth_token is not None:
+            self.headers['Authorization'] = f'Bearer {auth_token}'
+
+    def check_status(self):
+        self.get(f"http://localhost:{self.port}/v1/status").raise_for_status()
+
+    def branch_list(self, tenant_id: uuid.UUID) -> List[Dict]:
+        res = self.get(f"http://localhost:{self.port}/v1/branch/{tenant_id.hex}")
+        res.raise_for_status()
+        return res.json()
+
+    def branch_create(self, tenant_id: uuid.UUID, name: str, start_point: str) -> Dict:
+        res = self.post(
+            f"http://localhost:{self.port}/v1/branch",
+            json={
+                'tenant_id': tenant_id.hex,
+                'name': name,
+                'start_point': start_point,
+            }
+        )
+        res.raise_for_status()
+        return res.json()
+
+    def tenant_list(self) -> List[str]:
+        res = self.get(f"http://localhost:{self.port}/v1/tenant")
+        res.raise_for_status()
+        return res.json()
+
+    def tenant_create(self, tenant_id: uuid.UUID):
+        res = self.post(
+            f"http://localhost:{self.port}/v1/tenant",
+            json={
+                'tenant_id': tenant_id.hex,
+            },
+        )
+        res.raise_for_status()
+        return res.json()
+
+
+@dataclass
+class AuthKeys:
+    pub: bytes
+    priv: bytes
+
+    def generate_management_token(self):
+        token = jwt.encode({"scope": "pageserverapi"}, self.priv, algorithm="RS256")
+
+        # jwt.encode can return 'bytes' or 'str', depending on Python version or type
+        # hinting or something (not sure what). If it returned 'bytes', convert it to 'str'
+        # explicitly.
+        if isinstance(token, bytes):
+            token = token.decode()
+
+        return token
+
+    def generate_tenant_token(self, tenant_id):
+        token = jwt.encode({"scope": "tenant", "tenant_id": tenant_id}, self.priv, algorithm="RS256")
+
+        if isinstance(token, bytes):
+            token = token.decode()
+
+        return token
+
+
+
+
+
 class ZenithPageserver(PgProtocol):
    """ An object representing a running pageserver. """
-    def __init__(self, zenith_cli: ZenithCli):
-        super().__init__(host='localhost', port=DEFAULT_PAGESERVER_PORT)
+    def __init__(self, zenith_cli: ZenithCli, repo_dir: str):
+        super().__init__(host='localhost', port=DEFAULT_PAGESERVER_PG_PORT)
        self.zenith_cli = zenith_cli
        self.running = False
        self.initial_tenant = None
+        self.repo_dir = repo_dir

    def init(self, enable_auth: bool = False) -> 'ZenithPageserver':
        """
@@ -194,6 +275,12 @@ class ZenithPageserver(PgProtocol):
        self.zenith_cli.run(cmd)
        return self

+    def repo_dir(self) -> str:
+        """
+        Return path to repository dir
+        """
+        return self.zenith_cli.repo_dir
+
    def start(self) -> 'ZenithPageserver':
        """
        Start the page server.
@@ -224,9 +311,21 @@ class ZenithPageserver(PgProtocol):
    def __exit__(self, exc_type, exc, tb):
        self.stop()

+    @cached_property
+    def auth_keys(self) -> AuthKeys:
+        pub = (Path(self.repo_dir) / 'auth_public_key.pem').read_bytes()
+        priv = (Path(self.repo_dir) / 'auth_private_key.pem').read_bytes()
+        return AuthKeys(pub=pub, priv=priv)
+
+    def http_client(self, auth_token: Optional[str] = None):
+        return ZenithPageserverHttpClient(
+            port=DEFAULT_PAGESERVER_HTTP_PORT,
+            auth_token=auth_token,
+        )
+

@zenfixture
-def pageserver(zenith_cli: ZenithCli) -> Iterator[ZenithPageserver]:
+def pageserver(zenith_cli: ZenithCli, repo_dir: str) -> Iterator[ZenithPageserver]:
    """
    The 'pageserver' fixture provides a Page Server that's up and running.

@@ -239,7 +338,7 @@ def pageserver(zenith_cli: ZenithCli) -> Iterator[ZenithPageserver]:
    test called 'test_foo' would create and use branches with the 'test_foo' prefix.
    """

-    ps = ZenithPageserver(zenith_cli).init().start()
+    ps = ZenithPageserver(zenith_cli, repo_dir).init().start()
    # For convenience in tests, create a branch from the freshly-initialized cluster.
    zenith_cli.run(["branch", "empty", "main"])

@@ -250,6 +349,14 @@ def pageserver(zenith_cli: ZenithCli) -> Iterator[ZenithPageserver]:
    ps.stop()


+@pytest.fixture
+def pageserver_auth_enabled(zenith_cli: ZenithCli, repo_dir: str):
+    with ZenithPageserver(zenith_cli, repo_dir).init(enable_auth=True).start() as ps:
+        # For convenience in tests, create a branch from the freshly-initialized cluster.
+        zenith_cli.run(["branch", "empty", "main"])
+        yield ps
+
+
 class Postgres(PgProtocol):
    """ An object representing a running postgres daemon. """
    def __init__(self, zenith_cli: ZenithCli, repo_dir: str, tenant_id: str, port: int):
@@ -259,8 +366,8 @@ class Postgres(PgProtocol):
        self.running = False
        self.repo_dir = repo_dir
        self.branch: Optional[str] = None  # dubious, see asserts below
+        self.pgdata_dir: Optional[str] = None # Path to computenode PGDATA
        self.tenant_id = tenant_id
-        # path to conf is <repo_dir>/pgdatadirs/tenants/<tenant_id>/<branch_name>/postgresql.conf

    def create(
        self,
@@ -284,6 +391,9 @@ class Postgres(PgProtocol):
        else:
            self.zenith_cli.run(['pg', 'create', branch, f'--tenantid={self.tenant_id}'])
        self.branch = branch
+        path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch
+        self.pgdata_dir = os.path.join(self.repo_dir, path)
+
        if wal_acceptors is not None:
            self.adjust_for_wal_acceptors(wal_acceptors)
        if config_lines is None:
@@ -306,13 +416,18 @@ class Postgres(PgProtocol):

    def pg_xact_dir_path(self) -> str:
        """ Path to pg_xact dir """
-        path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch / 'pg_xact'
+        return os.path.join(self.pgdata_dir, 'pg_xact')
+
+    def pg_twophase_dir_path(self) -> str:
+        """ Path to pg_twophase dir """
+        print(self.tenant_id)
+        print(self.branch)
+        path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch / 'pg_twophase'
        return os.path.join(self.repo_dir, path)

    def config_file_path(self) -> str:
        """ Path to postgresql.conf """
-        filename = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch / 'postgresql.conf'
-        return os.path.join(self.repo_dir, filename)
+        return os.path.join(self.pgdata_dir, 'postgresql.conf')

    def adjust_for_wal_acceptors(self, wal_acceptors: str) -> 'Postgres':
        """
@@ -591,7 +706,7 @@ class WalAcceptor:
        cmd.append("--daemonize")
        cmd.append("--no-sync")
        # Tell page server it can receive WAL from this WAL safekeeper
-        cmd.extend(["--pageserver", "localhost:{}".format(DEFAULT_PAGESERVER_PORT)])
+        cmd.extend(["--pageserver", "localhost:{}".format(DEFAULT_PAGESERVER_PG_PORT)])
        cmd.extend(["--recall", "1 second"])
        print('Running command "{}"'.format(' '.join(cmd)))
        env = {'PAGESERVER_AUTH_TOKEN': self.auth_token} if self.auth_token else None
--- a/test_runner/performance/test_perf_pgbench.py
+++ b/test_runner/performance/test_perf_pgbench.py
@@ -0,0 +1,67 @@
+import os
+from contextlib import closing
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
+
+def get_timeline_size(repo_dir: str, tenantid: str, timelineid: str):
+    path = "{}/tenants/{}/timelines/{}".format(repo_dir, tenantid, timelineid)
+
+    totalbytes = 0
+    for root, dirs, files in os.walk(path):
+        for name in files:
+            totalbytes += os.path.getsize(os.path.join(root, name))
+
+        if 'wal' in dirs:
+            dirs.remove('wal')  # don't visit 'wal' subdirectory
+
+    return totalbytes
+
+#
+# Run a very short pgbench test.
+#
+# Collects three metrics:
+#
+# 1. Time to initialize the pgbench database (pgbench -s5 -i)
+# 2. Time to run 5000 pgbench transactions
+# 3. Disk space used
+#
+def test_pgbench(postgres: PostgresFactory, pageserver: ZenithPageserver, pg_bin, zenith_cli, zenbenchmark, repo_dir: str):
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_pgbench_perf", "empty"])
+
+    pg = postgres.create_start('test_pgbench_perf')
+    print("postgres is running on 'test_pgbench_perf' branch")
+
+    # Open a connection directly to the page server that we'll use to force
+    # flushing the layers to disk
+    psconn = pageserver.connect();
+    pscur = psconn.cursor()
+
+    # Get the timeline ID of our branch. We need it for the 'do_gc' command
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute("SHOW zenith.zenith_timeline")
+            timeline = cur.fetchone()[0]
+
+    connstr = pg.connstr()
+
+    # Initialize pgbench database
+    with zenbenchmark.record_duration('init'):
+        pg_bin.run_capture(['pgbench', '-s5', '-i', connstr])
+
+        # Flush the layers from memory to disk. The time to do that is included in the
+        # reported init time.
+        pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+
+    # Run pgbench for 5000 transactions
+    with zenbenchmark.record_duration('5000_xacts'):
+        pg_bin.run_capture(['pgbench', '-c1', '-t5000', connstr])
+
+    # Flush the layers to disk again. This is *not' included in the reported time,
+    # though.
+    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+
+    # Report disk space used by the repository
+    timeline_size = get_timeline_size(repo_dir, pageserver.initial_tenant, timeline)
+    zenbenchmark.record('size', timeline_size / (1024*1024), 'MB')
--- a/vendor/postgres
+++ b/vendor/postgres
--- a/walkeeper/src/lib.rs
+++ b/walkeeper/src/lib.rs
@@ -5,6 +5,7 @@ use std::time::Duration;
 pub mod receive_wal;
 pub mod replication;
 pub mod s3_offload;
+pub mod safekeeper;
 pub mod send_wal;
 pub mod timeline;
 pub mod wal_service;
--- a/walkeeper/src/receive_wal.rs
+++ b/walkeeper/src/receive_wal.rs
@@ -1,142 +1,26 @@
-//! This implements the Safekeeper protocol, picking up immediately after the "START_WAL_PUSH" message
-//!
-//! FIXME: better description needed here
+//! Safekeeper communication endpoint to wal proposer (compute node).
+//! Gets messages from the network, passes them down to consensus module and
+//! sends replies back.

-use anyhow::{bail, Result};
-use bincode::config::Options;
-use bytes::{Buf, Bytes};
+use anyhow::{bail, Context, Result};
+use bytes::Bytes;
 use log::*;
 use postgres::{Client, Config, NoTls};
-use serde::{Deserialize, Serialize};
-use std::cmp::{max, min};
-use std::fs::{self, File, OpenOptions};
-use std::io::{Read, Seek, SeekFrom, Write};
+
 use std::net::SocketAddr;
-use std::str;
-use std::sync::Arc;
 use std::thread;
 use std::thread::sleep;
-use zenith_utils::bin_ser::{self, le_coder, LeSer};
-use zenith_utils::connstring::connection_host_port;
-use zenith_utils::lsn::Lsn;
-use zenith_utils::postgres_backend::PostgresBackend;
-use zenith_utils::pq_proto::{BeMessage, FeMessage, SystemId};
-use zenith_utils::zid::{ZTenantId, ZTimelineId};

-use crate::replication::HotStandbyFeedback;
+use crate::safekeeper::AcceptorProposerMessage;
+use crate::safekeeper::ProposerAcceptorMessage;
+
 use crate::send_wal::SendWalHandler;
-use crate::timeline::{Timeline, TimelineTools};
+use crate::timeline::TimelineTools;
 use crate::WalAcceptorConf;
-use postgres_ffi::xlog_utils::{TimeLineID, XLogFileName, MAX_SEND_SIZE, XLOG_BLCKSZ};
-
-pub const SK_MAGIC: u32 = 0xcafeceefu32;
-pub const SK_FORMAT_VERSION: u32 = 1;
-const SK_PROTOCOL_VERSION: u32 = 1;
-const UNKNOWN_SERVER_VERSION: u32 = 0;
-const END_OF_STREAM: Lsn = Lsn(0);
-pub const CONTROL_FILE_NAME: &str = "safekeeper.control";
-
-/// Unique node identifier used by Paxos
-#[derive(Debug, Clone, Copy, Ord, PartialOrd, PartialEq, Eq, Serialize, Deserialize)]
-pub struct NodeId {
-    term: u64,
-    uuid: [u8; 16],
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub struct ServerInfo {
-    /// proposer-safekeeper protocol version
-    pub protocol_version: u32,
-    /// Postgres server version
-    pub pg_version: u32,
-    pub node_id: NodeId,
-    pub system_id: SystemId,
-    /// Zenith timelineid
-    pub timeline_id: ZTimelineId,
-    pub wal_end: Lsn,
-    pub timeline: TimeLineID,
-    pub wal_seg_size: u32,
-    pub tenant_id: ZTenantId,
-}
-
-/// Vote request sent from proposer to safekeepers
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-struct RequestVote {
-    node_id: NodeId,
-    /// volume commit LSN
-    vcl: Lsn,
-    /// new epoch when safekeeper reaches vcl
-    epoch: u64,
-}
-
-/// Information of about storage node
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub struct SafeKeeperInfo {
-    /// magic for verifying content the control file
-    pub magic: u32,
-    /// safekeeper format version
-    pub format_version: u32,
-    /// safekeeper's epoch
-    pub epoch: u64,
-    /// information about server
-    pub server: ServerInfo,
-    /// part of WAL acknowledged by quorum
-    pub commit_lsn: Lsn,
-    /// locally flushed part of WAL
-    pub flush_lsn: Lsn,
-    /// minimal LSN which may be needed for recovery of some safekeeper: min(commit_lsn) for all safekeepers
-    pub restart_lsn: Lsn,
-}
-
-impl SafeKeeperInfo {
-    pub fn new() -> SafeKeeperInfo {
-        SafeKeeperInfo {
-            magic: SK_MAGIC,
-            format_version: SK_FORMAT_VERSION,
-            epoch: 0,
-            server: ServerInfo {
-                protocol_version: SK_PROTOCOL_VERSION, /* proposer-safekeeper protocol version */
-                pg_version: UNKNOWN_SERVER_VERSION,    /* Postgres server version */
-                node_id: NodeId {
-                    term: 0,
-                    uuid: [0; 16],
-                },
-                system_id: 0, /* Postgres system identifier */
-                timeline_id: ZTimelineId::from([0u8; 16]),
-                wal_end: Lsn(0),
-                timeline: 0,
-                wal_seg_size: 0,
-                tenant_id: ZTenantId::from([0u8; 16]),
-            },
-            commit_lsn: Lsn(0),  /* part of WAL acknowledged by quorum */
-            flush_lsn: Lsn(0),   /* locally flushed part of WAL */
-            restart_lsn: Lsn(0), /* minimal LSN which may be needed for recovery of some safekeeper */
-        }
-    }
-}
-
-/// Request with WAL message sent from proposer to safekeeper.
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-struct SafeKeeperRequest {
-    /// Sender's node identifier (looks like we do not need it for TCP streaming connection)
-    sender_id: NodeId,
-    /// start position of message in WAL
-    begin_lsn: Lsn,
-    /// end position of message in WAL
-    end_lsn: Lsn,
-    /// restart LSN position  (minimal LSN which may be needed by proposer to perform recovery)
-    restart_lsn: Lsn,
-    /// LSN committed by quorum of safekeepers
-    commit_lsn: Lsn,
-}
-
-/// Report safekeeper state to proposer
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-struct SafeKeeperResponse {
-    epoch: u64,
-    flush_lsn: Lsn,
-    hs_feedback: HotStandbyFeedback,
-}
+use zenith_utils::connstring::connection_host_port;
+use zenith_utils::postgres_backend::PostgresBackend;
+use zenith_utils::pq_proto::{BeMessage, FeMessage};
+use zenith_utils::zid::{ZTenantId, ZTimelineId};

 pub struct ReceiveWalConn<'pg> {
    /// Postgres connection
@@ -206,310 +90,58 @@ impl<'pg> ReceiveWalConn<'pg> {
        }
    }

-    // Read the result of a `CopyData` message sent from the postgres instance
-    //
-    // As the trait bound implies, this always encodes little-endian.
-    fn read_msg<T: LeSer>(&mut self) -> Result<T> {
+    // Read and parse message sent from the postgres instance
+    fn read_msg(&mut self) -> Result<ProposerAcceptorMessage> {
        let data = self.read_msg_bytes()?;
-        // Taken directly from `LeSer::des`:
-        let value = le_coder()
-            .reject_trailing_bytes()
-            .deserialize(&data)
-            .or(Err(bin_ser::DeserializeError::BadInput))?;
-        Ok(value)
+        ProposerAcceptorMessage::parse(data)
    }

-    // Writes the value into a `CopyData` message sent to the postgres instance
-    fn write_msg<T: LeSer>(&mut self, value: &T) -> Result<()> {
+    // Send message to the postgres
+    fn write_msg(&mut self, msg: &AcceptorProposerMessage) -> Result<()> {
        let mut buf = Vec::new();
-        value.ser_into(&mut buf)?;
+        msg.serialize(&mut buf)?;
        self.pg_backend.write_message(&BeMessage::CopyData(&buf))?;
        Ok(())
    }

    /// Receive WAL from wal_proposer
    pub fn run(&mut self, swh: &mut SendWalHandler) -> Result<()> {
-        let mut this_timeline: Option<Arc<Timeline>> = None;
-
        // Notify the libpq client that it's allowed to send `CopyData` messages
        self.pg_backend
            .write_message(&BeMessage::CopyBothResponse)?;

        // Receive information about server
-        let server_info = self.read_msg::<ServerInfo>()?;
-        info!(
-            "Start handshake with wal_proposer {} sysid {} timeline {} tenant {}",
-            self.peer_addr, server_info.system_id, server_info.timeline_id, server_info.tenant_id,
-        );
-        // FIXME: also check that the system identifier matches
-        this_timeline.set(server_info.timeline_id)?;
-        this_timeline.get().load_control_file(&swh.conf)?;
-
-        let mut my_info = this_timeline.get().get_info();
-
-        /* Check protocol compatibility */
-        if server_info.protocol_version != SK_PROTOCOL_VERSION {
-            bail!(
-                "Incompatible protocol version {}, expected {}",
-                server_info.protocol_version,
-                SK_PROTOCOL_VERSION
-            );
-        }
-        /* Postgres upgrade is not treated as fatal error */
-        if server_info.pg_version != my_info.server.pg_version
-            && my_info.server.pg_version != UNKNOWN_SERVER_VERSION
-        {
-            info!(
-                "Incompatible server version {}, expected {}",
-                server_info.pg_version, my_info.server.pg_version
-            );
+        let mut msg = self
+            .read_msg()
+            .context("failed to receive proposer greeting")?;
+        let tenant_id: ZTenantId;
+        match msg {
+            ProposerAcceptorMessage::Greeting(ref greeting) => {
+                info!(
+                    "start handshake with wal proposer {} sysid {} timeline {}",
+                    self.peer_addr, greeting.system_id, greeting.tli,
+                );
+                tenant_id = greeting.tenant_id;
+            }
+            _ => bail!("unexpected message {:?} instead of greeting", msg),
        }

-        /* Update information about server, but preserve locally stored node_id */
-        let node_id = my_info.server.node_id;
-        my_info.server = server_info.clone();
-        my_info.server.node_id = node_id;
-
-        /* Calculate WAL end based on local data */
-        let (flush_lsn, timeline_id) = this_timeline.find_end_of_wal(&swh.conf.data_dir, true);
-        my_info.flush_lsn = flush_lsn;
-        my_info.server.timeline = timeline_id;
-
-        info!(
-            "find_end_of_wal in {:?}: timeline={} flush_lsn={}",
-            &swh.conf.data_dir, timeline_id, flush_lsn
-        );
-
-        /* Report my identifier to proposer */
-        self.write_msg(&my_info)?;
-
-        /* Wait for vote request */
-        let prop = self.read_msg::<RequestVote>()?;
-        /* This is Paxos check which should ensure that only one master can perform commits */
-        if prop.node_id < my_info.server.node_id {
-            /* Send my node-id to inform proposer that it's candidate was rejected */
-            self.write_msg(&my_info.server.node_id)?;
-            bail!(
-                "Reject connection attempt with term {} because my term is {}",
-                prop.node_id.term,
-                my_info.server.node_id.term,
-            );
-        }
-        my_info.server.node_id = prop.node_id;
-        this_timeline.get().set_info(&my_info);
-        /* Need to persist our vote first */
-        this_timeline.get().save_control_file(true)?;
-
-        let mut flushed_restart_lsn = Lsn(0);
-        let wal_seg_size = server_info.wal_seg_size as usize;
-
-        /* Acknowledge the proposed candidate by returning it to the proposer */
-        self.write_msg(&prop.node_id)?;
-
+        // if requested, ask pageserver to fetch wal from us
+        // xxx: this place seems not really fitting
        if swh.conf.pageserver_addr.is_some() {
            // Need to establish replication channel with page server.
            // Add far as replication in postgres is initiated by receiver, we should use callme mechanism
            let conf = swh.conf.clone();
-            let timelineid = this_timeline.get().timelineid;
-            let tenantid = server_info.tenant_id;
+            let timelineid = swh.timeline.get().timelineid;
            thread::spawn(move || {
-                request_callback(conf, timelineid, tenantid);
+                request_callback(conf, timelineid, tenant_id);
            });
        }

-        info!(
-            "Start streaming from timeline {} tenant {} address {:?} flush_lsn={}",
-            server_info.timeline_id, server_info.tenant_id, self.peer_addr, my_info.flush_lsn
-        );
-
-        // Main loop
        loop {
-            let mut sync_control_file = false;
-
-            /* Receive message header */
-            let msg_bytes = self.read_msg_bytes()?;
-            let mut msg_reader = msg_bytes.reader();
-
-            let req = SafeKeeperRequest::des_from(&mut msg_reader)?;
-            if req.sender_id != my_info.server.node_id {
-                bail!("Sender NodeId is changed");
-            }
-            if req.begin_lsn == END_OF_STREAM {
-                info!("Server stops streaming");
-                break;
-            }
-            let start_pos = req.begin_lsn;
-            let end_pos = req.end_lsn;
-            let rec_size = end_pos.checked_sub(start_pos).unwrap().0 as usize;
-            assert!(rec_size <= MAX_SEND_SIZE);
-
-            debug!(
-                "received for {} bytes between {} and {}",
-                rec_size, start_pos, end_pos,
-            );
-
-            /* Receive message body (from the rest of the message) */
-            let mut buf = Vec::with_capacity(rec_size);
-            msg_reader.read_to_end(&mut buf)?;
-            assert_eq!(buf.len(), rec_size);
-
-            /* Save message in file */
-            Self::write_wal_file(
-                swh,
-                start_pos,
-                timeline_id,
-                this_timeline.get(),
-                wal_seg_size,
-                &buf,
-            )?;
-
-            my_info.restart_lsn = req.restart_lsn;
-            my_info.commit_lsn = req.commit_lsn;
-
-            /*
-             * Epoch switch happen when written WAL record cross the boundary.
-             * The boundary is maximum of last WAL position at this node (FlushLSN) and global
-             * maximum (vcl) determined by WAL proposer during handshake.
-             * Switching epoch means that node completes recovery and start writing in the WAL new data.
-             */
-            if my_info.epoch < prop.epoch && end_pos > max(my_info.flush_lsn, prop.vcl) {
-                info!("Switch to new epoch {}", prop.epoch);
-                my_info.epoch = prop.epoch; /* bump epoch */
-                sync_control_file = true;
-            }
-            if end_pos > my_info.flush_lsn {
-                my_info.flush_lsn = end_pos;
-            }
-            /*
-             * Update restart LSN in control file.
-             * To avoid negative impact on performance of extra fsync, do it only
-             * when restart_lsn delta exceeds WAL segment size.
-             */
-            sync_control_file |= flushed_restart_lsn + (wal_seg_size as u64) < my_info.restart_lsn;
-            this_timeline.get().save_control_file(sync_control_file)?;
-
-            if sync_control_file {
-                flushed_restart_lsn = my_info.restart_lsn;
-            }
-
-            /* Report flush position */
-            //info!("Confirm LSN: {:X}/{:>08X}", (end_pos>>32) as u32, end_pos as u32);
-            let resp = SafeKeeperResponse {
-                epoch: my_info.epoch,
-                flush_lsn: end_pos,
-                hs_feedback: this_timeline.get().get_hs_feedback(),
-            };
-            self.write_msg(&resp)?;
-
-            /*
-             * Ping wal sender that new data is available.
-             * FlushLSN (end_pos) can be smaller than commitLSN in case we are at catching-up safekeeper.
-             */
-            this_timeline
-                .get()
-                .notify_wal_senders(min(req.commit_lsn, end_pos));
+            let reply = swh.timeline.get().process_msg(&msg)?;
+            self.write_msg(&reply)?;
+            msg = self.read_msg()?;
        }
-
-        Ok(())
-    }
-
-    fn write_wal_file(
-        swh: &SendWalHandler,
-        startpos: Lsn,
-        timeline_id: TimeLineID,
-        timeline: &Arc<Timeline>,
-        wal_seg_size: usize,
-        buf: &[u8],
-    ) -> Result<()> {
-        let mut bytes_left: usize = buf.len();
-        let mut bytes_written: usize = 0;
-        let mut partial;
-        let mut start_pos = startpos;
-        const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ];
-
-        /* Extract WAL location for this block */
-        let mut xlogoff = start_pos.segment_offset(wal_seg_size) as usize;
-
-        while bytes_left != 0 {
-            let bytes_to_write;
-
-            /*
-             * If crossing a WAL boundary, only write up until we reach wal
-             * segment size.
-             */
-            if xlogoff + bytes_left > wal_seg_size {
-                bytes_to_write = wal_seg_size - xlogoff;
-            } else {
-                bytes_to_write = bytes_left;
-            }
-
-            /* Open file */
-            let segno = start_pos.segment_number(wal_seg_size);
-            let wal_file_name = XLogFileName(timeline_id, segno, wal_seg_size);
-            let wal_file_path = swh
-                .conf
-                .data_dir
-                .join(timeline.timelineid.to_string())
-                .join(wal_file_name.clone());
-            let wal_file_partial_path = swh
-                .conf
-                .data_dir
-                .join(timeline.timelineid.to_string())
-                .join(wal_file_name.clone() + ".partial");
-
-            {
-                let mut wal_file: File;
-                /* Try to open already completed segment */
-                if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_path) {
-                    wal_file = file;
-                    partial = false;
-                } else if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_partial_path)
-                {
-                    /* Try to open existed partial file */
-                    wal_file = file;
-                    partial = true;
-                } else {
-                    /* Create and fill new partial file */
-                    partial = true;
-                    match OpenOptions::new()
-                        .create(true)
-                        .write(true)
-                        .open(&wal_file_partial_path)
-                    {
-                        Ok(mut file) => {
-                            for _ in 0..(wal_seg_size / XLOG_BLCKSZ) {
-                                file.write_all(&ZERO_BLOCK)?;
-                            }
-                            wal_file = file;
-                        }
-                        Err(e) => {
-                            error!("Failed to open log file {:?}: {}", &wal_file_path, e);
-                            return Err(e.into());
-                        }
-                    }
-                }
-                wal_file.seek(SeekFrom::Start(xlogoff as u64))?;
-                wal_file.write_all(&buf[bytes_written..(bytes_written + bytes_to_write)])?;
-
-                // Flush file is not prohibited
-                if !swh.conf.no_sync {
-                    wal_file.sync_all()?;
-                }
-            }
-            /* Write was successful, advance our position */
-            bytes_written += bytes_to_write;
-            bytes_left -= bytes_to_write;
-            start_pos += bytes_to_write as u64;
-            xlogoff += bytes_to_write;
-
-            /* Did we reach the end of a WAL segment? */
-            if start_pos.segment_offset(wal_seg_size) == 0 {
-                xlogoff = 0;
-                if partial {
-                    fs::rename(&wal_file_partial_path, &wal_file_path)?;
-                }
-            }
-        }
-        Ok(())
    }
 }
--- a/walkeeper/src/replication.rs
+++ b/walkeeper/src/replication.rs
@@ -35,6 +35,16 @@ pub struct HotStandbyFeedback {
    pub catalog_xmin: FullTransactionId,
 }

+impl HotStandbyFeedback {
+    pub fn empty() -> HotStandbyFeedback {
+        HotStandbyFeedback {
+            ts: 0,
+            xmin: 0,
+            catalog_xmin: 0,
+        }
+    }
+}
+
 /// A network connection that's speaking the replication protocol.
 pub struct ReplicationConn {
    /// This is an `Option` because we will spawn a background thread that will
@@ -76,8 +86,10 @@ impl ReplicationConn {
                    let feedback = HotStandbyFeedback::des(&m)?;
                    subscriber.add_hs_feedback(feedback);
                }
+                FeMessage::Sync => {}
+                FeMessage::CopyFail => return Err(anyhow!("Copy failed")),
                _ => {
-                    // We only handle `CopyData` messages. Anything else is ignored.
+                    // We only handle `CopyData`, 'Sync', 'CopyFail' messages. Anything else is ignored.
                    info!("unexpected message {:?}", msg);
                }
            }
@@ -148,7 +160,7 @@ impl ReplicationConn {
                break;
            }
        }
-        let (wal_end, timeline) = swh.timeline.find_end_of_wal(&swh.conf.data_dir, true);
+        let (wal_end, timeline) = swh.timeline.get().get_end_of_wal();
        if start_pos == Lsn(0) {
            start_pos = wal_end;
        }
--- a/walkeeper/src/safekeeper.rs
+++ b/walkeeper/src/safekeeper.rs
@@ -0,0 +1,550 @@
+//! Acceptor part of proposer-acceptor consensus algorithm.
+
+use anyhow::{anyhow, bail, Result};
+use byteorder::LittleEndian;
+use byteorder::ReadBytesExt;
+use byteorder::WriteBytesExt;
+use bytes::Buf;
+use bytes::Bytes;
+use log::*;
+use postgres_ffi::xlog_utils::TimeLineID;
+use serde::{Deserialize, Serialize};
+use std::cmp::max;
+use std::io;
+use std::io::Read;
+
+use crate::replication::HotStandbyFeedback;
+use postgres_ffi::xlog_utils::MAX_SEND_SIZE;
+use zenith_utils::bin_ser::LeSer;
+use zenith_utils::lsn::Lsn;
+use zenith_utils::pq_proto::SystemId;
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
+
+pub const SK_MAGIC: u32 = 0xcafeceefu32;
+pub const SK_FORMAT_VERSION: u32 = 1;
+const SK_PROTOCOL_VERSION: u32 = 1;
+const UNKNOWN_SERVER_VERSION: u32 = 0;
+
+/// Consensus logical timestamp.
+type Term = u64;
+
+/// Unique id of proposer. Not needed for correctness, used for monitoring.
+type PgUuid = [u8; 16];
+
+/// Persistent consensus state of the acceptor.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AcceptorState {
+    /// acceptor's last term it voted for (advanced in 1 phase)
+    pub term: Term,
+    /// acceptor's epoch (advanced, i.e. bumped to 'term' when VCL is reached).
+    pub epoch: Term,
+}
+
+/// Information about Postgres. Safekeeper gets it once and then verifies
+/// all further connections from computes match.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct ServerInfo {
+    /// Postgres server version
+    pub pg_version: u32,
+    pub system_id: SystemId,
+    /// Zenith timelineid
+    pub ztli: ZTimelineId,
+    pub tli: TimeLineID,
+    pub wal_seg_size: u32,
+}
+
+/// Persistent information stored on safekeeper node
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SafeKeeperState {
+    /// magic for verifying content the control file
+    pub magic: u32,
+    /// safekeeper format version
+    pub format_version: u32,
+    /// persistent acceptor state
+    pub acceptor_state: AcceptorState,
+    /// information about server
+    pub server: ServerInfo,
+    /// Unique id of the last *elected* proposer we dealed with. Not needed
+    /// correctness, exists for monitoring purposes.
+    pub proposer_uuid: PgUuid,
+    /// part of WAL acknowledged by quorum (note that we might not have wal to
+    /// up this point locally)
+    pub commit_lsn: Lsn,
+    /// minimal LSN which may be needed for recovery of some safekeeper (end lsn
+    /// + 1 of last record streamed to everyone)
+    pub truncate_lsn: Lsn,
+}
+
+impl SafeKeeperState {
+    pub fn new() -> SafeKeeperState {
+        SafeKeeperState {
+            magic: SK_MAGIC,
+            format_version: SK_FORMAT_VERSION,
+            acceptor_state: AcceptorState { term: 0, epoch: 0 },
+            server: ServerInfo {
+                pg_version: UNKNOWN_SERVER_VERSION, /* Postgres server version */
+                system_id: 0,                       /* Postgres system identifier */
+                ztli: ZTimelineId::from([0u8; 16]),
+                tli: 0,
+                wal_seg_size: 0,
+            },
+            proposer_uuid: [0; 16],
+            commit_lsn: Lsn(0),   /* part of WAL acknowledged by quorum */
+            truncate_lsn: Lsn(0), /* minimal LSN which may be needed for recovery of some safekeeper */
+        }
+    }
+}
+
+// protocol messages
+
+/// Initial Proposer -> Acceptor message
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ProposerGreeting {
+    /// proposer-acceptor protocol version
+    pub protocol_version: u32,
+    /// Postgres server version
+    pub pg_version: u32,
+    pub proposer_id: PgUuid,
+    pub system_id: SystemId,
+    /// Zenith timelineid
+    pub ztli: ZTimelineId,
+    pub tenant_id: ZTenantId,
+    pub tli: TimeLineID,
+    pub wal_seg_size: u32,
+}
+
+/// Acceptor -> Proposer initial response: the highest term known to me
+/// (acceptor voted for).
+#[derive(Debug, Serialize, Deserialize)]
+pub struct AcceptorGreeting {
+    term: u64,
+}
+
+/// Vote request sent from proposer to safekeepers
+#[derive(Debug, Serialize, Deserialize)]
+pub struct VoteRequest {
+    term: Term,
+}
+
+/// Vote itself, sent from safekeeper to proposer
+#[derive(Debug, Serialize, Deserialize)]
+pub struct VoteResponse {
+    term: Term,      // not really needed, just a sanity check
+    vote_given: u64, // fixme u64 due to padding
+    /// Safekeeper's log position, to let proposer choose the most advanced one
+    epoch: Term,
+    flush_lsn: Lsn,
+    restart_lsn: Lsn,
+}
+
+/// Request with WAL message sent from proposer to safekeeper. Along the way it
+/// announces 1) successful election (with epoch_start_lsn); 2) commit_lsn.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct AppendRequest {
+    h: AppendRequestHeader,
+    wal_data: Bytes,
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AppendRequestHeader {
+    term: Term,
+    // LSN since the proposer appends WAL; determines epoch switch point.
+    epoch_start_lsn: Lsn,
+    /// start position of message in WAL
+    begin_lsn: Lsn,
+    /// end position of message in WAL
+    end_lsn: Lsn,
+    /// LSN committed by quorum of safekeepers
+    commit_lsn: Lsn,
+    /// restart LSN position  (minimal LSN which may be needed by proposer to perform recovery)
+    restart_lsn: Lsn,
+    // only for logging/debugging
+    proposer_uuid: PgUuid,
+}
+
+/// Report safekeeper state to proposer
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct AppendResponse {
+    // Current term of the safekeeper; if it is higher than proposer's, the
+    // compute is out of date.
+    pub term: Term,
+    pub epoch: Term,
+    // NOTE: this is physical end of wal on safekeeper; currently it doesn't
+    // make much sense without taking epoch into account, as history can be
+    // diverged.
+    pub flush_lsn: Lsn,
+    pub hs_feedback: HotStandbyFeedback,
+}
+
+/// Proposer -> Acceptor messages
+#[derive(Debug)]
+pub enum ProposerAcceptorMessage {
+    Greeting(ProposerGreeting),
+    VoteRequest(VoteRequest),
+    AppendRequest(AppendRequest),
+}
+
+impl ProposerAcceptorMessage {
+    /// Parse proposer message.
+    pub fn parse(msg: Bytes) -> Result<ProposerAcceptorMessage> {
+        // xxx using Reader is inefficient but easy to work with bincode
+        let mut stream = msg.reader();
+        // u64 is here to avoid padding; it will be removed once we stop packing C structs into the wire as is
+        let tag = stream.read_u64::<LittleEndian>()? as u8 as char;
+        match tag {
+            'g' => {
+                let msg = ProposerGreeting::des_from(&mut stream)?;
+                Ok(ProposerAcceptorMessage::Greeting(msg))
+            }
+            'v' => {
+                let msg = VoteRequest::des_from(&mut stream)?;
+                Ok(ProposerAcceptorMessage::VoteRequest(msg))
+            }
+            'a' => {
+                // read header followed by wal data
+                let hdr = AppendRequestHeader::des_from(&mut stream)?;
+                let rec_size = hdr
+                    .end_lsn
+                    .checked_sub(hdr.begin_lsn)
+                    .ok_or(anyhow!("begin_lsn > end_lsn in AppendRequest"))?
+                    .0 as usize;
+                if rec_size > MAX_SEND_SIZE {
+                    bail!(
+                        "AppendRequest is longer than MAX_SEND_SIZE ({})",
+                        MAX_SEND_SIZE
+                    );
+                }
+
+                let mut wal_data_vec: Vec<u8> = vec![0; rec_size];
+                stream.read_exact(&mut wal_data_vec)?;
+                let wal_data = Bytes::from(wal_data_vec);
+                let msg = AppendRequest {
+                    h: hdr,
+                    wal_data: wal_data,
+                };
+
+                Ok(ProposerAcceptorMessage::AppendRequest(msg))
+            }
+            _ => Err(anyhow!("unknown proposer-acceptor message tag: {}", tag,)),
+        }
+    }
+}
+
+/// Acceptor -> Proposer messages
+#[derive(Debug)]
+pub enum AcceptorProposerMessage {
+    Greeting(AcceptorGreeting),
+    VoteResponse(VoteResponse),
+    AppendResponse(AppendResponse),
+}
+
+impl AcceptorProposerMessage {
+    /// Serialize acceptor -> proposer message.
+    pub fn serialize(&self, stream: &mut impl io::Write) -> Result<()> {
+        match self {
+            AcceptorProposerMessage::Greeting(msg) => {
+                stream.write_u64::<LittleEndian>('g' as u64)?;
+                msg.ser_into(stream)?;
+            }
+            AcceptorProposerMessage::VoteResponse(msg) => {
+                stream.write_u64::<LittleEndian>('v' as u64)?;
+                msg.ser_into(stream)?;
+            }
+            AcceptorProposerMessage::AppendResponse(msg) => {
+                stream.write_u64::<LittleEndian>('a' as u64)?;
+                msg.ser_into(stream)?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
+pub trait Storage {
+    /// Persist safekeeper state on disk, optionally syncing it.
+    fn persist(&mut self, s: &SafeKeeperState, sync: bool) -> Result<()>;
+    /// Write piece of wal in buf to disk.
+    fn write_wal(&mut self, s: &SafeKeeperState, startpos: Lsn, buf: &[u8]) -> Result<()>;
+}
+
+/// SafeKeeper which consumes events (messages from compute) and provides
+/// replies.
+#[derive(Debug)]
+pub struct SafeKeeper<ST: Storage> {
+    /// Locally flushed part of WAL (end_lsn of last record). Established by
+    /// reading wal.
+    pub flush_lsn: Lsn,
+    pub tli: u32,
+    pub flushed_truncate_lsn: Lsn,
+    pub storage: ST,
+    pub s: SafeKeeperState,          // persistent part
+    pub elected_proposer_term: Term, // for monitoring/debugging
+}
+
+impl<ST> SafeKeeper<ST>
+where
+    ST: Storage,
+{
+    // constructor
+    pub fn new(flush_lsn: Lsn, tli: u32, storage: ST, state: SafeKeeperState) -> SafeKeeper<ST> {
+        SafeKeeper {
+            flush_lsn,
+            tli,
+            flushed_truncate_lsn: Lsn(0),
+            storage,
+            s: state,
+            elected_proposer_term: 0,
+        }
+    }
+
+    /// Process message from proposer and possibly form reply. Concurrent
+    /// callers must exclude each other.
+    pub fn process_msg(
+        &mut self,
+        msg: &ProposerAcceptorMessage,
+    ) -> Result<AcceptorProposerMessage> {
+        match msg {
+            ProposerAcceptorMessage::Greeting(msg) => self.handle_greeting(msg),
+            ProposerAcceptorMessage::VoteRequest(msg) => self.handle_vote_request(msg),
+            ProposerAcceptorMessage::AppendRequest(msg) => self.handle_append_request(msg),
+        }
+    }
+
+    /// Handle initial message from proposer: check its sanity and send my
+    /// current term.
+    fn handle_greeting(&mut self, msg: &ProposerGreeting) -> Result<AcceptorProposerMessage> {
+        /* Check protocol compatibility */
+        if msg.protocol_version != SK_PROTOCOL_VERSION {
+            bail!(
+                "incompatible protocol version {}, expected {}",
+                msg.protocol_version,
+                SK_PROTOCOL_VERSION
+            );
+        }
+        if self.s.server.system_id != 0 && self.s.server.system_id != msg.system_id {
+            bail!(
+                "system identifier changed: got {}, expected {}",
+                msg.system_id,
+                self.s.server.system_id,
+            );
+        }
+        /* Postgres upgrade is not treated as fatal error */
+        if msg.pg_version != self.s.server.pg_version
+            && self.s.server.pg_version != UNKNOWN_SERVER_VERSION
+        {
+            info!(
+                "incompatible server version {}, expected {}",
+                msg.pg_version, self.s.server.pg_version
+            );
+        }
+
+        // set basic info about server, if not yet
+        self.s.server.system_id = msg.system_id;
+        self.s.server.ztli = msg.ztli;
+        self.s.server.tli = msg.tli;
+        self.s.server.wal_seg_size = msg.wal_seg_size;
+        self.s.proposer_uuid = msg.proposer_id;
+        self.storage.persist(&self.s, true)?;
+
+        info!(
+            "processed greeting from proposer {:?}, sending term {:?}",
+            msg.proposer_id, self.s.acceptor_state.term
+        );
+        Ok(AcceptorProposerMessage::Greeting(AcceptorGreeting {
+            term: self.s.acceptor_state.term,
+        }))
+    }
+
+    /// Give vote for the given term, if we haven't done that previously.
+    fn handle_vote_request(&mut self, msg: &VoteRequest) -> Result<AcceptorProposerMessage> {
+        // initialize with refusal
+        let mut resp = VoteResponse {
+            term: msg.term,
+            vote_given: false as u64,
+            epoch: 0,
+            flush_lsn: Lsn(0),
+            restart_lsn: Lsn(0),
+        };
+        if self.s.acceptor_state.term < msg.term {
+            self.s.acceptor_state.term = msg.term;
+            // persist vote before sending it out
+            self.storage.persist(&self.s, true)?;
+            resp.vote_given = true as u64;
+            resp.epoch = self.s.acceptor_state.epoch;
+            resp.flush_lsn = self.flush_lsn;
+            resp.restart_lsn = self.s.truncate_lsn;
+        }
+        info!("processed VoteRequest for term {}: {:?}", msg.term, &resp);
+        Ok(AcceptorProposerMessage::VoteResponse(resp))
+    }
+
+    /// Handle request to append WAL.
+    fn handle_append_request(&mut self, msg: &AppendRequest) -> Result<AcceptorProposerMessage> {
+        // log first AppendRequest from this proposer
+        if self.elected_proposer_term < msg.h.term {
+            info!(
+                "start receiving WAL from timeline {} term {}",
+                self.s.server.ztli, msg.h.term,
+            );
+            self.elected_proposer_term = msg.h.term;
+        }
+
+        // If our term is lower than elected proposer one, bump it.
+        if self.s.acceptor_state.term < msg.h.term {
+            self.s.acceptor_state.term = msg.h.term;
+            self.storage.persist(&self.s, true)?;
+        }
+        // OTOH, if it is higher, immediately refuse the message.
+        else if self.s.acceptor_state.term > msg.h.term {
+            let resp = AppendResponse {
+                term: self.s.acceptor_state.term,
+                epoch: self.s.acceptor_state.epoch,
+                flush_lsn: Lsn(0),
+                hs_feedback: HotStandbyFeedback::empty(),
+            };
+            return Ok(AcceptorProposerMessage::AppendResponse(resp));
+        }
+
+        // do the job
+        self.storage
+            .write_wal(&self.s, msg.h.begin_lsn, &msg.wal_data)?;
+        let mut sync_control_file = false;
+        /*
+         * Epoch switch happen when written WAL record cross the boundary.
+         * The boundary is maximum of last WAL position at this node (FlushLSN) and global
+         * maximum (vcl) determined by WAL proposer during handshake.
+         * Switching epoch means that node completes recovery and start writing in the WAL new data.
+         * XXX: this is wrong, we must actively truncate not matching part of log.
+         */
+        if self.s.acceptor_state.epoch < msg.h.term
+            && msg.h.end_lsn > max(self.flush_lsn, msg.h.epoch_start_lsn)
+        {
+            info!("switched to new epoch {}", msg.h.term);
+            self.s.acceptor_state.epoch = msg.h.term; /* bump epoch */
+            sync_control_file = true;
+        }
+        if msg.h.end_lsn > self.flush_lsn {
+            self.flush_lsn = msg.h.end_lsn;
+        }
+
+        self.s.proposer_uuid = msg.h.proposer_uuid;
+        self.s.commit_lsn = msg.h.commit_lsn;
+        self.s.truncate_lsn = msg.h.restart_lsn;
+
+        /*
+         * Update restart LSN in control file.
+         * To avoid negative impact on performance of extra fsync, do it only
+         * when restart_lsn delta exceeds WAL segment size.
+         */
+        sync_control_file |=
+            self.flushed_truncate_lsn + (self.s.server.wal_seg_size as u64) < self.s.truncate_lsn;
+        self.storage.persist(&self.s, sync_control_file)?;
+        if sync_control_file {
+            self.flushed_truncate_lsn = self.s.truncate_lsn;
+        }
+
+        let resp = AppendResponse {
+            term: self.s.acceptor_state.term,
+            epoch: self.s.acceptor_state.epoch,
+            flush_lsn: self.flush_lsn,
+            // will be filled by caller code to avoid bothering safekeeper
+            hs_feedback: HotStandbyFeedback::empty(),
+        };
+        trace!(
+            "processed AppendRequest of len {}, flush_lsn={:X}/{:>08X}, resp {:?}",
+            msg.wal_data.len(),
+            (self.flush_lsn.0 >> 32) as u32,
+            self.flush_lsn.0 as u32,
+            &resp,
+        );
+        Ok(AcceptorProposerMessage::AppendResponse(resp))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // fake storage for tests
+    struct InMemoryStorage {
+        persisted_state: SafeKeeperState,
+    }
+
+    impl Storage for InMemoryStorage {
+        fn persist(&mut self, s: &SafeKeeperState, _sync: bool) -> Result<()> {
+            self.persisted_state = s.clone();
+            Ok(())
+        }
+
+        fn write_wal(&mut self, _s: &SafeKeeperState, _startpos: Lsn, _buf: &[u8]) -> Result<()> {
+            Ok(())
+        }
+    }
+
+    #[test]
+    fn test_voting() {
+        let storage = InMemoryStorage {
+            persisted_state: SafeKeeperState::new(),
+        };
+        let mut sk = SafeKeeper::new(Lsn(0), 0, storage, SafeKeeperState::new());
+
+        // check voting for 1 is ok
+        let vote_request = ProposerAcceptorMessage::VoteRequest(VoteRequest { term: 1 });
+        let mut vote_resp = sk.process_msg(&vote_request);
+        match vote_resp.unwrap() {
+            AcceptorProposerMessage::VoteResponse(resp) => assert!(resp.vote_given != 0),
+            _ => assert!(false),
+        }
+
+        // reboot...
+        let state = sk.storage.persisted_state.clone();
+        let storage = InMemoryStorage {
+            persisted_state: state.clone(),
+        };
+        sk = SafeKeeper::new(Lsn(0), 0, storage, state);
+
+        // and ensure voting second time for 1 is not ok
+        vote_resp = sk.process_msg(&vote_request);
+        match vote_resp.unwrap() {
+            AcceptorProposerMessage::VoteResponse(resp) => assert!(resp.vote_given == 0),
+            _ => assert!(false),
+        }
+    }
+
+    #[test]
+    fn test_epoch_switch() {
+        let storage = InMemoryStorage {
+            persisted_state: SafeKeeperState::new(),
+        };
+        let mut sk = SafeKeeper::new(Lsn(0), 0, storage, SafeKeeperState::new());
+
+        let mut ar_hdr = AppendRequestHeader {
+            term: 1,
+            epoch_start_lsn: Lsn(2),
+            begin_lsn: Lsn(1),
+            end_lsn: Lsn(2),
+            commit_lsn: Lsn(0),
+            restart_lsn: Lsn(0),
+            proposer_uuid: [0; 16],
+        };
+        let mut append_request = AppendRequest {
+            h: ar_hdr.clone(),
+            wal_data: Bytes::from_static(b"b"),
+        };
+
+        // check that AppendRequest before VCL doesn't switch epoch
+        let resp = sk.process_msg(&ProposerAcceptorMessage::AppendRequest(append_request));
+        assert!(resp.is_ok());
+        assert!(sk.storage.persisted_state.acceptor_state.epoch == 0);
+
+        // but record after VCL does the switch
+        ar_hdr.begin_lsn = Lsn(2);
+        ar_hdr.end_lsn = Lsn(3);
+        append_request = AppendRequest {
+            h: ar_hdr.clone(),
+            wal_data: Bytes::from_static(b"b"),
+        };
+        let resp = sk.process_msg(&ProposerAcceptorMessage::AppendRequest(append_request));
+        assert!(resp.is_ok());
+        assert!(sk.storage.persisted_state.acceptor_state.epoch == 1);
+    }
+}
--- a/walkeeper/src/send_wal.rs
+++ b/walkeeper/src/send_wal.rs
@@ -15,12 +15,15 @@ use zenith_utils::postgres_backend::PostgresBackend;
 use zenith_utils::pq_proto::{BeMessage, FeStartupMessage, RowDescriptor};
 use zenith_utils::zid::ZTimelineId;

+use crate::timeline::CreateControlFile;
+
 /// Handler for streaming WAL from acceptor
 pub struct SendWalHandler {
    /// wal acceptor configuration
    pub conf: WalAcceptorConf,
    /// assigned application name
    pub appname: Option<String>,
+    pub timelineid: Option<ZTimelineId>,
    pub timeline: Option<Arc<Timeline>>,
 }

@@ -29,7 +32,7 @@ impl postgres_backend::Handler for SendWalHandler {
        match sm.params.get("ztimelineid") {
            Some(ref ztimelineid) => {
                let ztlid = ZTimelineId::from_str(ztimelineid)?;
-                self.timeline.set(ztlid)?;
+                self.timelineid = Some(ztlid);
            }
            _ => bail!("timelineid is required"),
        }
@@ -40,6 +43,22 @@ impl postgres_backend::Handler for SendWalHandler {
    }

    fn process_query(&mut self, pgb: &mut PostgresBackend, query_string: Bytes) -> Result<()> {
+        // START_WAL_PUSH is the only command that initializes the timeline
+        if self.timeline.is_none() {
+            if query_string.starts_with(b"START_WAL_PUSH") {
+                self.timeline.set(
+                    &self.conf,
+                    self.timelineid.unwrap(),
+                    CreateControlFile::True,
+                )?;
+            } else {
+                self.timeline.set(
+                    &self.conf,
+                    self.timelineid.unwrap(),
+                    CreateControlFile::False,
+                )?;
+            }
+        }
        if query_string.starts_with(b"IDENTIFY_SYSTEM") {
            self.handle_identify_system(pgb)?;
            Ok(())
@@ -60,6 +79,7 @@ impl SendWalHandler {
        SendWalHandler {
            conf,
            appname: None,
+            timelineid: None,
            timeline: None,
        }
    }
@@ -68,7 +88,7 @@ impl SendWalHandler {
    /// Handle IDENTIFY_SYSTEM replication command
    ///
    fn handle_identify_system(&mut self, pgb: &mut PostgresBackend) -> Result<()> {
-        let (start_pos, timeline) = self.timeline.find_end_of_wal(&self.conf.data_dir, false);
+        let (start_pos, timeline) = self.timeline.get().get_end_of_wal();
        let lsn = start_pos.to_string();
        let tli = timeline.to_string();
        let sysid = self.timeline.get().get_info().server.system_id.to_string();
--- a/walkeeper/src/timeline.rs
+++ b/walkeeper/src/timeline.rs
@@ -1,50 +1,84 @@
-//! This module contains tools for managing timelines.
-//!
+//! This module contains timeline id -> safekeeper state map with file-backed
+//! persistence and support for interaction between sending and receiving wal.

 use anyhow::{bail, Result};
 use fs2::FileExt;
 use lazy_static::lazy_static;
 use log::*;
-use postgres_ffi::xlog_utils::{find_end_of_wal, TimeLineID};
+use postgres_ffi::xlog_utils::find_end_of_wal;
 use std::cmp::{max, min};
 use std::collections::HashMap;
 use std::fs::{self, File, OpenOptions};
-use std::io::{Seek, SeekFrom};
-use std::path::Path;
+use std::io::{Seek, SeekFrom, Write};
 use std::sync::{Arc, Condvar, Mutex};
 use zenith_utils::bin_ser::LeSer;
 use zenith_utils::lsn::Lsn;
+
 use zenith_utils::zid::ZTimelineId;

-use crate::receive_wal::{SafeKeeperInfo, CONTROL_FILE_NAME, SK_FORMAT_VERSION, SK_MAGIC};
 use crate::replication::{HotStandbyFeedback, END_REPLICATION_MARKER};
+use crate::safekeeper::{
+    AcceptorProposerMessage, ProposerAcceptorMessage, SafeKeeper, SafeKeeperState, Storage,
+    SK_FORMAT_VERSION, SK_MAGIC,
+};
 use crate::WalAcceptorConf;
+use postgres_ffi::xlog_utils::{XLogFileName, XLOG_BLCKSZ};
+
+const CONTROL_FILE_NAME: &str = "safekeeper.control";

 /// Shared state associated with database instance (tenant)
 #[derive(Debug)]
 struct SharedState {
-    /// quorum commit LSN
+    /// Safekeeper object
+    sk: SafeKeeper<FileStorage>,
+    /// opened file control file handle (needed to hold exlusive file lock)
+    control_file: File,
+    /// For receiving-sending wal cooperation
+    /// quorum commit LSN we've notified walsenders about
    commit_lsn: Lsn,
-    /// information about this safekeeper
-    info: SafeKeeperInfo,
-    /// opened file control file handle (needed to hold exlusive file lock
-    control_file: Option<File>,
    /// combined hot standby feedback from all replicas
    hs_feedback: HotStandbyFeedback,
 }

+// A named boolean.
+#[derive(Debug)]
+pub enum CreateControlFile {
+    True,
+    False,
+}
+
 impl SharedState {
-    fn new() -> Self {
-        Self {
+    /// Restore SharedState from control file. Locks the control file along the
+    /// way to prevent running more than one instance of safekeeper on the same
+    /// data dir.
+    /// If create=false and file doesn't exist, bails out.
+    fn create_restore(
+        conf: &WalAcceptorConf,
+        timelineid: ZTimelineId,
+        create: CreateControlFile,
+    ) -> Result<Self> {
+        let (cf, state) = SharedState::load_control_file(conf, timelineid, create)?;
+        let storage = FileStorage {
+            control_file: cf.try_clone()?,
+            conf: conf.clone(),
+        };
+        let (flush_lsn, tli) = if state.server.wal_seg_size != 0 {
+            let wal_dir = conf.data_dir.join(format!("{}", timelineid));
+            find_end_of_wal(&wal_dir, state.server.wal_seg_size as usize, true)
+        } else {
+            (0, 0)
+        };
+
+        Ok(Self {
            commit_lsn: Lsn(0),
-            info: SafeKeeperInfo::new(),
-            control_file: None,
+            sk: SafeKeeper::new(Lsn(flush_lsn), tli, storage, state),
+            control_file: cf,
            hs_feedback: HotStandbyFeedback {
                ts: 0,
                xmin: u64::MAX,
                catalog_xmin: u64::MAX,
            },
-        }
+        })
    }

    /// Accumulate hot standby feedbacks from replicas
@@ -54,80 +88,75 @@ impl SharedState {
        self.hs_feedback.ts = max(self.hs_feedback.ts, feedback.ts);
    }

-    /// Load and lock control file (prevent running more than one instance of safekeeper)
-    pub fn load_control_file(
-        &mut self,
+    /// Fetch and lock control file (prevent running more than one instance of safekeeper)
+    /// If create=false and file doesn't exist, bails out.
+    fn load_control_file(
        conf: &WalAcceptorConf,
        timelineid: ZTimelineId,
-    ) -> Result<()> {
-        if self.control_file.is_some() {
-            info!("control file for timeline {} is already open", timelineid);
-            return Ok(());
-        }
-
+        create: CreateControlFile,
+    ) -> Result<(File, SafeKeeperState)> {
        let control_file_path = conf
            .data_dir
            .join(timelineid.to_string())
            .join(CONTROL_FILE_NAME);
-        info!("loading control file {}", control_file_path.display());
-        match OpenOptions::new()
-            .read(true)
-            .write(true)
-            .create(true)
-            .open(&control_file_path)
-        {
-            Ok(file) => {
+        info!(
+            "loading control file {}, create={:?}",
+            control_file_path.display(),
+            create
+        );
+        let mut opts = OpenOptions::new();
+        opts.read(true).write(true);
+        if let CreateControlFile::True = create {
+            opts.create(true);
+        }
+        match opts.open(&control_file_path) {
+            Ok(mut file) => {
                // Lock file to prevent two or more active wal_acceptors
                match file.try_lock_exclusive() {
                    Ok(()) => {}
                    Err(e) => {
                        bail!(
-                            "Control file {:?} is locked by some other process: {}",
+                            "control file {:?} is locked by some other process: {}",
                            &control_file_path,
                            e
                        );
                    }
                }
-                self.control_file = Some(file);
-
-                let cfile_ref = self.control_file.as_mut().unwrap();
-                match SafeKeeperInfo::des_from(cfile_ref) {
-                    Err(e) => {
-                        warn!("read from {:?} failed: {}", control_file_path, e);
+                // Empty file is legit on 'create', don't try to deser from it.
+                if file.metadata().unwrap().len() == 0 {
+                    if let CreateControlFile::False = create {
+                        bail!("control file is empty");
                    }
-                    Ok(info) => {
-                        if info.magic != SK_MAGIC {
-                            bail!("Invalid control file magic: {}", info.magic);
+                    return Ok((file, SafeKeeperState::new()));
+                } else {
+                    match SafeKeeperState::des_from(&mut file) {
+                        Err(e) => {
+                            bail!("failed to read control file {:?}: {}", control_file_path, e);
                        }
-                        if info.format_version != SK_FORMAT_VERSION {
-                            bail!(
-                                "Incompatible format version: {} vs. {}",
-                                info.format_version,
-                                SK_FORMAT_VERSION
-                            );
+                        Ok(s) => {
+                            if s.magic != SK_MAGIC {
+                                bail!("bad control file magic: {}", s.magic);
+                            }
+                            if s.format_version != SK_FORMAT_VERSION {
+                                bail!(
+                                    "incompatible format version: {} vs. {}",
+                                    s.format_version,
+                                    SK_FORMAT_VERSION
+                                );
+                            }
+                            return Ok((file, s));
                        }
-                        self.info = info;
                    }
                }
            }
            Err(e) => {
-                panic!(
-                    "Failed to open control file {:?}: {}",
-                    &control_file_path, e
+                bail!(
+                    "failed to open control file {:?}: {}",
+                    &control_file_path,
+                    e
                );
            }
        }
-        Ok(())
-    }
-
-    pub fn save_control_file(&mut self, sync: bool) -> Result<()> {
-        let file = self.control_file.as_mut().unwrap();
-        file.seek(SeekFrom::Start(0))?;
-        self.info.ser_into(file)?;
-        if sync {
-            file.sync_all()?;
-        }
-        Ok(())
    }
 }

@@ -179,12 +208,32 @@ impl Timeline {
        self.notify_wal_senders(END_REPLICATION_MARKER);
    }

-    pub fn get_info(&self) -> SafeKeeperInfo {
-        return self.mutex.lock().unwrap().info.clone();
+    /// Pass arrived message to the safekeeper.
+    pub fn process_msg(&self, msg: &ProposerAcceptorMessage) -> Result<AcceptorProposerMessage> {
+        let mut rmsg: AcceptorProposerMessage;
+        let commit_lsn: Lsn;
+        {
+            let mut shared_state = self.mutex.lock().unwrap();
+            rmsg = shared_state.sk.process_msg(msg)?;
+            // locally available commit lsn. flush_lsn can be smaller than
+            // commit_lsn if we are catching up safekeeper.
+            commit_lsn = min(shared_state.sk.flush_lsn, shared_state.sk.s.commit_lsn);
+
+            // if this is AppendResponse, fill in proper hot standby feedback
+            match rmsg {
+                AcceptorProposerMessage::AppendResponse(ref mut resp) => {
+                    resp.hs_feedback = shared_state.hs_feedback.clone();
+                }
+                _ => (),
+            }
+        }
+        // Ping wal sender that new data might be available.
+        self.notify_wal_senders(commit_lsn);
+        Ok(rmsg)
    }

-    pub fn set_info(&self, info: &SafeKeeperInfo) {
-        self.mutex.lock().unwrap().info = info.clone();
+    pub fn get_info(&self) -> SafeKeeperState {
+        self.mutex.lock().unwrap().sk.s.clone()
    }

    // Accumulate hot standby feedbacks from replicas
@@ -198,44 +247,40 @@ impl Timeline {
        shared_state.hs_feedback.clone()
    }

-    pub fn load_control_file(&self, conf: &WalAcceptorConf) -> Result<()> {
-        let mut shared_state = self.mutex.lock().unwrap();
-        shared_state.load_control_file(conf, self.timelineid)
-    }
-
-    pub fn save_control_file(&self, sync: bool) -> Result<()> {
-        let mut shared_state = self.mutex.lock().unwrap();
-        shared_state.save_control_file(sync)
+    pub fn get_end_of_wal(&self) -> (Lsn, u32) {
+        let shared_state = self.mutex.lock().unwrap();
+        (shared_state.sk.flush_lsn, shared_state.sk.tli)
    }
 }

 // Utilities needed by various Connection-like objects
 pub trait TimelineTools {
-    fn set(&mut self, timeline_id: ZTimelineId) -> Result<()>;
+    fn set(
+        &mut self,
+        conf: &WalAcceptorConf,
+        timeline_id: ZTimelineId,
+        create: CreateControlFile,
+    ) -> Result<()>;
    fn get(&self) -> &Arc<Timeline>;
-    fn find_end_of_wal(&self, data_dir: &Path, precise: bool) -> (Lsn, TimeLineID);
 }

 impl TimelineTools for Option<Arc<Timeline>> {
-    fn set(&mut self, timeline_id: ZTimelineId) -> Result<()> {
+    fn set(
+        &mut self,
+        conf: &WalAcceptorConf,
+        timeline_id: ZTimelineId,
+        create: CreateControlFile,
+    ) -> Result<()> {
        // We will only set the timeline once. If it were to ever change,
        // anyone who cloned the Arc would be out of date.
        assert!(self.is_none());
-        *self = Some(GlobalTimelines::store(timeline_id)?);
+        *self = Some(GlobalTimelines::get(conf, timeline_id, create)?);
        Ok(())
    }

    fn get(&self) -> &Arc<Timeline> {
        self.as_ref().unwrap()
    }
-
-    /// Find last WAL record. If "precise" is false then just locate last partial segment
-    fn find_end_of_wal(&self, data_dir: &Path, precise: bool) -> (Lsn, TimeLineID) {
-        let seg_size = self.get().get_info().server.wal_seg_size as usize;
-        let wal_dir = data_dir.join(format!("{}", self.get().timelineid));
-        let (lsn, timeline) = find_end_of_wal(&wal_dir, seg_size, precise);
-        (Lsn(lsn), timeline)
-    }
 }

 lazy_static! {
@@ -247,22 +292,143 @@ lazy_static! {
 struct GlobalTimelines;

 impl GlobalTimelines {
-    /// Store a new timeline into the global TIMELINES map.
-    fn store(timeline_id: ZTimelineId) -> Result<Arc<Timeline>> {
+    /// Get a timeline with control file loaded from the global TIMELINES map.
+    /// If control file doesn't exist and create=false, bails out.
+    pub fn get(
+        conf: &WalAcceptorConf,
+        timeline_id: ZTimelineId,
+        create: CreateControlFile,
+    ) -> Result<Arc<Timeline>> {
        let mut timelines = TIMELINES.lock().unwrap();

        match timelines.get(&timeline_id) {
            Some(result) => Ok(Arc::clone(result)),
            None => {
-                info!("creating timeline dir {}", timeline_id);
+                info!(
+                    "creating timeline dir {}, create is {:?}",
+                    timeline_id, create
+                );
                fs::create_dir_all(timeline_id.to_string())?;

-                let shared_state = SharedState::new();
+                let shared_state = SharedState::create_restore(conf, timeline_id, create)?;

-                let new_tid = Arc::new(Timeline::new(timeline_id, shared_state));
-                timelines.insert(timeline_id, Arc::clone(&new_tid));
-                Ok(new_tid)
+                let new_tli = Arc::new(Timeline::new(timeline_id, shared_state));
+                timelines.insert(timeline_id, Arc::clone(&new_tli));
+                Ok(new_tli)
            }
        }
    }
 }
+
+#[derive(Debug)]
+struct FileStorage {
+    control_file: File,
+    conf: WalAcceptorConf,
+}
+
+impl Storage for FileStorage {
+    fn persist(&mut self, s: &SafeKeeperState, sync: bool) -> Result<()> {
+        self.control_file.seek(SeekFrom::Start(0))?;
+        s.ser_into(&mut self.control_file)?;
+        if sync {
+            self.control_file.sync_all()?;
+        }
+        Ok(())
+    }
+
+    fn write_wal(&mut self, s: &SafeKeeperState, startpos: Lsn, buf: &[u8]) -> Result<()> {
+        let mut bytes_left: usize = buf.len();
+        let mut bytes_written: usize = 0;
+        let mut partial;
+        let mut start_pos = startpos;
+        const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ];
+        let wal_seg_size = s.server.wal_seg_size as usize;
+        let ztli = s.server.ztli;
+
+        /* Extract WAL location for this block */
+        let mut xlogoff = start_pos.segment_offset(wal_seg_size) as usize;
+
+        while bytes_left != 0 {
+            let bytes_to_write;
+
+            /*
+             * If crossing a WAL boundary, only write up until we reach wal
+             * segment size.
+             */
+            if xlogoff + bytes_left > wal_seg_size {
+                bytes_to_write = wal_seg_size - xlogoff;
+            } else {
+                bytes_to_write = bytes_left;
+            }
+
+            /* Open file */
+            let segno = start_pos.segment_number(wal_seg_size);
+            // note: we basically don't support changing pg timeline
+            let wal_file_name = XLogFileName(s.server.tli, segno, wal_seg_size);
+            let wal_file_path = self
+                .conf
+                .data_dir
+                .join(ztli.to_string())
+                .join(wal_file_name.clone());
+            let wal_file_partial_path = self
+                .conf
+                .data_dir
+                .join(ztli.to_string())
+                .join(wal_file_name.clone() + ".partial");
+
+            {
+                let mut wal_file: File;
+                /* Try to open already completed segment */
+                if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_path) {
+                    wal_file = file;
+                    partial = false;
+                } else if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_partial_path)
+                {
+                    /* Try to open existed partial file */
+                    wal_file = file;
+                    partial = true;
+                } else {
+                    /* Create and fill new partial file */
+                    partial = true;
+                    match OpenOptions::new()
+                        .create(true)
+                        .write(true)
+                        .open(&wal_file_partial_path)
+                    {
+                        Ok(mut file) => {
+                            for _ in 0..(wal_seg_size / XLOG_BLCKSZ) {
+                                file.write_all(&ZERO_BLOCK)?;
+                            }
+                            wal_file = file;
+                        }
+                        Err(e) => {
+                            error!("Failed to open log file {:?}: {}", &wal_file_path, e);
+                            return Err(e.into());
+                        }
+                    }
+                }
+                wal_file.seek(SeekFrom::Start(xlogoff as u64))?;
+                wal_file.write_all(&buf[bytes_written..(bytes_written + bytes_to_write)])?;
+
+                // Flush file, if not said otherwise
+                if !self.conf.no_sync {
+                    wal_file.sync_all()?;
+                }
+            }
+            /* Write was successful, advance our position */
+            bytes_written += bytes_to_write;
+            bytes_left -= bytes_to_write;
+            start_pos += bytes_to_write as u64;
+            xlogoff += bytes_to_write;
+
+            /* Did we reach the end of a WAL segment? */
+            if start_pos.segment_offset(wal_seg_size) == 0 {
+                xlogoff = 0;
+                if partial {
+                    fs::rename(&wal_file_partial_path, &wal_file_path)?;
+                }
+            }
+        }
+        Ok(())
+    }
+}
--- a/zenith/src/main.rs
+++ b/zenith/src/main.rs
@@ -61,13 +61,6 @@ fn main() -> Result<()> {
                        .long("enable-auth")
                        .takes_value(false)
                        .help("Enable authentication using ZenithJWT")
-                )
-                .arg(
-                    Arg::with_name("repository-format")
-                        .long("repository-format")
-                        .takes_value(false)
-                        .value_name("repository-format")
-                        .help("Choose repository format, 'layered' or 'rocksdb'")
                ),
        )
        .subcommand(
@@ -131,12 +124,6 @@ fn main() -> Result<()> {
                        ),
                ),
        )
-        .subcommand(
-            SubCommand::with_name("push")
-                .about("Push timeline to remote pageserver")
-                .arg(Arg::with_name("timeline").required(true))
-                .arg(Arg::with_name("remote").required(true)),
-        )
        .get_matches();

    // Create config file
@@ -148,8 +135,7 @@ fn main() -> Result<()> {
        } else {
            AuthType::Trust
        };
-        let repository_format = init_match.value_of("repository-format");
-        local_env::init(pageserver_uri, tenantid, auth_type, repository_format)
+        local_env::init(pageserver_uri, tenantid, auth_type)
            .with_context(|| "Failed to create config file")?;
    }

@@ -168,7 +154,6 @@ fn main() -> Result<()> {
            if let Err(e) = pageserver.init(
                Some(&env.tenantid.to_string()),
                init_match.is_present("enable-auth"),
-                init_match.value_of("repository-format"),
            ) {
                eprintln!("pageserver init failed: {}", e);
                exit(1);
@@ -224,7 +209,7 @@ fn main() -> Result<()> {

        ("pg", Some(pg_match)) => {
            if let Err(e) = handle_pg(pg_match, &env) {
-                eprintln!("pg operation failed: {}", e);
+                eprintln!("pg operation failed: {:?}", e);
                exit(1);
            }
        }
@@ -236,13 +221,6 @@ fn main() -> Result<()> {
            }
        }

-        ("push", Some(push_match)) => {
-            if let Err(e) = handle_push(push_match, &env) {
-                eprintln!("push operation failed: {}", e);
-                exit(1);
-            }
-        }
-
        _ => {}
    };

@@ -371,7 +349,7 @@ fn get_branch_infos(
    tenantid: &ZTenantId,
 ) -> Result<HashMap<ZTimelineId, BranchInfo>> {
    let page_server = PageServerNode::from_env(env);
-    let branch_infos: Vec<BranchInfo> = page_server.branches_list(tenantid)?;
+    let branch_infos: Vec<BranchInfo> = page_server.branch_list(tenantid)?;
    let branch_infos: HashMap<ZTimelineId, BranchInfo> = branch_infos
        .into_iter()
        .map(|branch_info| (branch_info.timeline_id, branch_info))
@@ -384,7 +362,7 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &local_env::LocalEnv) -> Result
    let pageserver = PageServerNode::from_env(&env);
    match tenant_match.subcommand() {
        ("list", Some(_)) => {
-            for tenant in pageserver.tenants_list()? {
+            for tenant in pageserver.tenant_list()? {
                println!("{}", tenant);
            }
        }
@@ -394,7 +372,7 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &local_env::LocalEnv) -> Result
                None => ZTenantId::generate(),
            };
            println!("using tenant id {}", tenantid);
-            pageserver.tenant_create(&tenantid)?;
+            pageserver.tenant_create(tenantid)?;
            println!("tenant successfully created on the pageserver");
        }
        _ => {}
@@ -424,7 +402,7 @@ fn handle_branch(branch_match: &ArgMatches, env: &local_env::LocalEnv) -> Result
            .value_of("tenantid")
            .map_or(Ok(env.tenantid), |value| value.parse())?;
        // No arguments, list branches for tenant
-        let branches = pageserver.branches_list(&tenantid)?;
+        let branches = pageserver.branch_list(&tenantid)?;
        print_branches_tree(branches)?;
    }

@@ -541,20 +519,3 @@ fn handle_remote(remote_match: &ArgMatches, local_env: &LocalEnv) -> Result<()>

    Ok(())
 }
-
-fn handle_push(push_match: &ArgMatches, local_env: &LocalEnv) -> Result<()> {
-    let timeline_id_str = push_match.value_of("timeline").unwrap();
-    ZTimelineId::from_str(timeline_id_str)?;
-
-    let remote_name = push_match.value_of("remote").unwrap();
-    let remote = local_env
-        .remotes
-        .get(remote_name)
-        .ok_or_else(|| anyhow!("remote {} not found", remote_name))?;
-
-    let page_server = PageServerNode::from_env(local_env);
-    let mut client = page_server.page_server_psql_client()?;
-    client.simple_query(&format!("request_push {} {}", timeline_id_str, remote))?;
-
-    Ok(())
-}
--- a/zenith_utils/Cargo.toml
+++ b/zenith_utils/Cargo.toml
@@ -13,7 +13,9 @@ hyper = { version = "0.14.7", features = ["full"] }
 lazy_static = "1.4.0"
 log = "0.4.14"
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
+routerify = "2"
 serde = { version = "1.0", features = ["derive"] }
+serde_json = "1"
 thiserror = "1.0"
 tokio = { version = "1.5.0", features = ["full"] }

--- a/zenith_utils/src/auth.rs
+++ b/zenith_utils/src/auth.rs
@@ -10,7 +10,7 @@ use serde::de::Error;
 use serde::{self, Deserializer, Serializer};
 use std::{fs, path::PathBuf};

-use anyhow::Result;
+use anyhow::{bail, Result};
 use jsonwebtoken::{
    decode, encode, Algorithm, DecodingKey, EncodingKey, Header, TokenData, Validation,
 };
@@ -20,7 +20,7 @@ use crate::zid::ZTenantId;

 const JWT_ALGORITHM: Algorithm = Algorithm::RS256;

-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 #[serde(rename_all = "lowercase")]
 pub enum Scope {
    Tenant,
@@ -48,7 +48,7 @@ where
    }
 }

-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct Claims {
    // this custom serialize/deserialize_with is needed because Option is not transparent to serde
    // so clearest option is serde(with = "hex") but it is not working, for details see https://github.com/serde-rs/serde/issues/1301
@@ -68,6 +68,22 @@ impl Claims {
    }
 }

+pub fn check_permission(claims: &Claims, tenantid: Option<ZTenantId>) -> Result<()> {
+    match (&claims.scope, tenantid) {
+        (Scope::Tenant, None) => {
+            bail!("Attempt to access management api with tenant scope. Permission denied")
+        }
+        (Scope::Tenant, Some(tenantid)) => {
+            if claims.tenant_id.unwrap() != tenantid {
+                bail!("Tenant id mismatch. Permission denied")
+            }
+            Ok(())
+        }
+        (Scope::PageServerApi, None) => Ok(()), // access to management api for PageServerApi scope
+        (Scope::PageServerApi, Some(_)) => Ok(()), // access to tenant api using PageServerApi scope
+    }
+}
+
 #[derive(Debug)]
 pub struct JwtAuth {
    decoding_key: DecodingKey<'static>,
--- a/zenith_utils/src/http/endpoint.rs
+++ b/zenith_utils/src/http/endpoint.rs
@@ -0,0 +1,164 @@
+use crate::auth::{self, Claims, JwtAuth};
+use crate::http::error;
+use crate::zid::ZTenantId;
+use anyhow::anyhow;
+use hyper::header::AUTHORIZATION;
+use hyper::{header::CONTENT_TYPE, Body, Request, Response, Server};
+use lazy_static::lazy_static;
+use routerify::ext::RequestExt;
+use routerify::RequestInfo;
+use routerify::{Middleware, Router, RouterBuilder, RouterService};
+use zenith_metrics::{register_int_counter, IntCounter};
+use zenith_metrics::{Encoder, TextEncoder};
+
+use super::error::ApiError;
+
+lazy_static! {
+    static ref SERVE_METRICS_COUNT: IntCounter = register_int_counter!(
+        "pageserver_serve_metrics_count",
+        "Number of metric requests made"
+    )
+    .expect("failed to define a metric");
+}
+
+async fn logger(res: Response<Body>, info: RequestInfo) -> Result<Response<Body>, ApiError> {
+    log::info!("{} {} {}", info.method(), info.uri().path(), res.status(),);
+    Ok(res)
+}
+
+async fn prometheus_metrics_handler(_req: Request<Body>) -> Result<Response<Body>, ApiError> {
+    SERVE_METRICS_COUNT.inc();
+
+    let mut buffer = vec![];
+    let encoder = TextEncoder::new();
+    let metrics = zenith_metrics::gather();
+    encoder.encode(&metrics, &mut buffer).unwrap();
+
+    let response = Response::builder()
+        .status(200)
+        .header(CONTENT_TYPE, encoder.format_type())
+        .body(Body::from(buffer))
+        .unwrap();
+
+    Ok(response)
+}
+
+pub fn make_router() -> RouterBuilder<hyper::Body, ApiError> {
+    Router::builder()
+        .middleware(Middleware::post_with_info(logger))
+        .get("/metrics", prometheus_metrics_handler)
+        .err_handler(error::handler)
+}
+
+pub fn attach_openapi_ui(
+    router_builder: RouterBuilder<hyper::Body, ApiError>,
+    spec: &'static [u8],
+    spec_mount_path: &'static str,
+    ui_mount_path: &'static str,
+) -> RouterBuilder<hyper::Body, ApiError> {
+    router_builder.get(spec_mount_path, move |_| async move {
+        Ok(Response::builder().body(Body::from(spec)).unwrap())
+    }).get(ui_mount_path, move |_| async move {
+        Ok(Response::builder().body(Body::from(format!(r#"
+            <!DOCTYPE html>
+            <html lang="en">
+            <head>
+            <title>rweb</title>
+            <link href="https://cdn.jsdelivr.net/npm/swagger-ui-dist@3/swagger-ui.css" rel="stylesheet">
+            </head>
+            <body>
+                <div id="swagger-ui"></div>
+                <script src="https://cdn.jsdelivr.net/npm/swagger-ui-dist@3/swagger-ui-bundle.js" charset="UTF-8"> </script>
+                <script>
+                    window.onload = function() {{
+                    const ui = SwaggerUIBundle({{
+                        "dom_id": "\#swagger-ui",
+                        presets: [
+                        SwaggerUIBundle.presets.apis,
+                        SwaggerUIBundle.SwaggerUIStandalonePreset
+                        ],
+                        layout: "BaseLayout",
+                        deepLinking: true,
+                        showExtensions: true,
+                        showCommonExtensions: true,
+                        url: "{}",
+                    }})
+                    window.ui = ui;
+                }};
+            </script>
+            </body>
+            </html>
+        "#, spec_mount_path))).unwrap())
+    })
+}
+
+fn parse_token(header_value: &str) -> Result<&str, ApiError> {
+    // header must be in form Bearer <token>
+    let (prefix, token) = header_value.split_once(' ').ok_or(ApiError::Unauthorized(
+        "malformed authorization header".to_string(),
+    ))?;
+    if prefix != "Bearer" {
+        Err(ApiError::Unauthorized(
+            "malformed authorization header".to_string(),
+        ))?
+    }
+    Ok(token)
+}
+
+pub fn auth_middleware<B: hyper::body::HttpBody + Send + Sync + 'static>(
+    provide_auth: fn(&Request<Body>) -> Option<&JwtAuth>,
+) -> Middleware<B, ApiError> {
+    Middleware::pre(move |req| async move {
+        if let Some(auth) = provide_auth(&req) {
+            match req.headers().get(AUTHORIZATION) {
+                Some(value) => {
+                    let header_value = value.to_str().map_err(|_| {
+                        ApiError::Unauthorized("malformed authorization header".to_string())
+                    })?;
+                    let token = parse_token(header_value)?;
+
+                    let data = auth
+                        .decode(token)
+                        .map_err(|_| ApiError::Unauthorized("malformed jwt token".to_string()))?;
+                    req.set_context(data.claims);
+                }
+                None => Err(ApiError::Unauthorized(
+                    "missing authorization header".to_string(),
+                ))?,
+            }
+        }
+        Ok(req)
+    })
+}
+
+pub fn check_permission(req: &Request<Body>, tenantid: Option<ZTenantId>) -> Result<(), ApiError> {
+    match req.context::<Claims>() {
+        Some(claims) => Ok(auth::check_permission(&claims, tenantid)
+            .map_err(|err| ApiError::Forbidden(err.to_string()))?),
+        None => Ok(()), // claims is None because auth is disabled
+    }
+}
+
+pub fn serve_thread_main(
+    router_builder: RouterBuilder<hyper::Body, ApiError>,
+    addr: String,
+) -> anyhow::Result<()> {
+    let addr = addr.parse()?;
+    log::info!("Starting a http endoint at {}", addr);
+
+    // Create a Service from the router above to handle incoming requests.
+    let service = RouterService::new(router_builder.build().map_err(|err| anyhow!(err))?).unwrap();
+
+    // Enter a single-threaded tokio runtime bound to the current thread
+    let runtime = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()?;
+
+    let _guard = runtime.enter();
+
+    let server = Server::bind(&addr).serve(service);
+
+    runtime.block_on(server)?;
+
+    Ok(())
+}
--- a/zenith_utils/src/http/error.rs
+++ b/zenith_utils/src/http/error.rs
@@ -0,0 +1,77 @@
+use anyhow::anyhow;
+use hyper::{header, Body, Response, StatusCode};
+use serde::Serialize;
+use serde_json;
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum ApiError {
+    #[error("Bad request: {0}")]
+    BadRequest(String),
+
+    #[error("Forbidden: {0}")]
+    Forbidden(String),
+
+    #[error("Unauthorized: {0}")]
+    Unauthorized(String),
+
+    #[error(transparent)]
+    InternalServerError(#[from] anyhow::Error),
+}
+
+impl ApiError {
+    pub fn from_err<E: Into<anyhow::Error>>(err: E) -> Self {
+        Self::InternalServerError(anyhow!(err))
+    }
+
+    pub fn into_response(self) -> Response<Body> {
+        match self {
+            ApiError::BadRequest(_) => HttpErrorBody::response_from_msg_and_status(
+                self.to_string(),
+                StatusCode::BAD_REQUEST,
+            ),
+            ApiError::Forbidden(_) => {
+                HttpErrorBody::response_from_msg_and_status(self.to_string(), StatusCode::FORBIDDEN)
+            }
+            ApiError::Unauthorized(_) => HttpErrorBody::response_from_msg_and_status(
+                self.to_string(),
+                StatusCode::UNAUTHORIZED,
+            ),
+            ApiError::InternalServerError(err) => HttpErrorBody::response_from_msg_and_status(
+                err.to_string(),
+                StatusCode::INTERNAL_SERVER_ERROR,
+            ),
+        }
+    }
+}
+
+#[derive(Serialize)]
+pub struct HttpErrorBody {
+    pub msg: String,
+}
+
+impl HttpErrorBody {
+    pub fn from_msg(msg: String) -> Self {
+        HttpErrorBody { msg }
+    }
+
+    pub fn response_from_msg_and_status(msg: String, status: StatusCode) -> Response<Body> {
+        HttpErrorBody { msg }.into_response(status)
+    }
+
+    pub fn into_response(&self, status: StatusCode) -> Response<Body> {
+        Response::builder()
+            .status(status)
+            .header(header::CONTENT_TYPE, "application/json")
+            // we do not have nested maps with non string keys so serialization shouldn't fail
+            .body(Body::from(serde_json::to_string(self).unwrap()))
+            .unwrap()
+    }
+}
+
+pub async fn handler(err: routerify::RouteError) -> Response<Body> {
+    log::error!("{}", err);
+    err.downcast::<ApiError>()
+        .expect("handler should always return api error")
+        .into_response()
+}
--- a/zenith_utils/src/http/json.rs
+++ b/zenith_utils/src/http/json.rs
@@ -0,0 +1,29 @@
+use bytes::Buf;
+use hyper::{header, Body, Request, Response, StatusCode};
+use serde::{Deserialize, Serialize};
+use serde_json;
+
+use super::error::ApiError;
+
+pub async fn json_request<T: for<'de> Deserialize<'de>>(
+    request: &mut Request<Body>,
+) -> Result<T, ApiError> {
+    let whole_body = hyper::body::aggregate(request.body_mut())
+        .await
+        .map_err(ApiError::from_err)?;
+    Ok(serde_json::from_reader(whole_body.reader())
+        .map_err(|err| ApiError::BadRequest(format!("Failed to parse json request {}", err)))?)
+}
+
+pub fn json_response<T: Serialize>(
+    status: StatusCode,
+    data: T,
+) -> Result<Response<Body>, ApiError> {
+    let json = serde_json::to_string(&data).map_err(ApiError::from_err)?;
+    let response = Response::builder()
+        .status(status)
+        .header(header::CONTENT_TYPE, "application/json")
+        .body(Body::from(json))
+        .map_err(ApiError::from_err)?;
+    Ok(response)
+}
--- a/zenith_utils/src/http/mod.rs
+++ b/zenith_utils/src/http/mod.rs
@@ -0,0 +1,3 @@
+pub mod endpoint;
+pub mod error;
+pub mod json;
--- a/zenith_utils/src/http_endpoint.rs
+++ b/zenith_utils/src/http_endpoint.rs
@@ -1,53 +0,0 @@
-use hyper::{
-    header::CONTENT_TYPE,
-    service::{make_service_fn, service_fn},
-    Body, Request, Response, Server,
-};
-use lazy_static::lazy_static;
-use zenith_metrics::{register_int_counter, IntCounter};
-use zenith_metrics::{Encoder, TextEncoder};
-
-lazy_static! {
-    static ref SERVE_METRICS_COUNT: IntCounter = register_int_counter!(
-        "pageserver_serve_metrics_count",
-        "Number of metric requests made"
-    )
-    .expect("failed to define a metric");
-}
-
-async fn serve_prometheus_metrics(_req: Request<Body>) -> anyhow::Result<Response<Body>> {
-    SERVE_METRICS_COUNT.inc();
-
-    let mut buffer = vec![];
-    let encoder = TextEncoder::new();
-    let metrics = zenith_metrics::gather();
-    encoder.encode(&metrics, &mut buffer).unwrap();
-
-    let response = Response::builder()
-        .status(200)
-        .header(CONTENT_TYPE, encoder.format_type())
-        .body(Body::from(buffer))
-        .unwrap();
-
-    Ok(response)
-}
-
-pub fn thread_main(addr: String) -> anyhow::Result<()> {
-    let addr = addr.parse()?;
-    log::info!("Starting a prometheus endoint at {}", addr);
-
-    // Enter a single-threaded tokio runtime bound to the current thread
-    let runtime = tokio::runtime::Builder::new_current_thread()
-        .enable_all()
-        .build()?;
-    let _guard = runtime.enter();
-
-    // TODO: use hyper_router/routerify/etc when we have more methods
-    let server = Server::bind(&addr).serve(make_service_fn(|_| async {
-        Ok::<_, anyhow::Error>(service_fn(serve_prometheus_metrics))
-    }));
-
-    runtime.block_on(server)?;
-
-    Ok(())
-}
--- a/zenith_utils/src/lib.rs
+++ b/zenith_utils/src/lib.rs
@@ -10,7 +10,6 @@ pub mod seqwait;
 // pub mod seqwait_async;

 pub mod bin_ser;
-pub mod http_endpoint;
 pub mod postgres_backend;
 pub mod pq_proto;

@@ -22,3 +21,5 @@ pub mod auth;

 // utility functions and helper traits for unified unique id generation/serialization etc.
 pub mod zid;
+// http endpoint utils
+pub mod http;
--- a/zenith_utils/src/postgres_backend.rs
+++ b/zenith_utils/src/postgres_backend.rs
@@ -301,8 +301,9 @@ impl PostgresBackend {
            FeMessage::Query(m) => {
                trace!("got query {:?}", m.body);
                // xxx distinguish fatal and recoverable errors?
-                if let Err(e) = handler.process_query(self, m.body) {
+                if let Err(e) = handler.process_query(self, m.body.clone()) {
                    let errmsg = format!("{}", e);
+                    warn!("query handler for {:?} failed: {}", m.body, errmsg);
                    self.write_message_noflush(&BeMessage::ErrorResponse(errmsg))?;
                }
                self.write_message(&BeMessage::ReadyForQuery)?;
@@ -340,7 +341,7 @@ impl PostgresBackend {

            // We prefer explicit pattern matching to wildcards, because
            // this helps us spot the places where new variants are missing
-            FeMessage::CopyData(_) | FeMessage::CopyDone => {
+            FeMessage::CopyData(_) | FeMessage::CopyDone | FeMessage::CopyFail => {
                bail!("unexpected message type: {:?}", msg);
            }
        }
--- a/zenith_utils/src/pq_proto.rs
+++ b/zenith_utils/src/pq_proto.rs
@@ -31,6 +31,7 @@ pub enum FeMessage {
    Terminate,
    CopyData(Bytes),
    CopyDone,
+    CopyFail,
    PasswordMessage(Bytes),
 }

@@ -138,6 +139,7 @@ impl FeMessage {
            b'X' => Ok(Some(FeMessage::Terminate)),
            b'd' => Ok(Some(FeMessage::CopyData(body))),
            b'c' => Ok(Some(FeMessage::CopyDone)),
+            b'f' => Ok(Some(FeMessage::CopyFail)),
            b'p' => Ok(Some(FeMessage::PasswordMessage(body))),
            tag => Err(anyhow!("unknown message tag: {},'{:?}'", tag, body)),
        }
@@ -338,6 +340,7 @@ pub enum BeMessage<'a> {
    ControlFile,
    CopyData(&'a [u8]),
    CopyDone,
+    CopyFail,
    CopyInResponse,
    CopyOutResponse,
    CopyBothResponse,
@@ -546,6 +549,11 @@ impl<'a> BeMessage<'a> {
                write_body(buf, |_| Ok::<(), io::Error>(())).unwrap();
            }

+            BeMessage::CopyFail => {
+                buf.put_u8(b'f');
+                write_body(buf, |_| Ok::<(), io::Error>(())).unwrap();
+            }
+
            BeMessage::CopyInResponse => {
                buf.put_u8(b'G');
                write_body(buf, |buf| {
--- a/zenith_utils/src/zid.rs
+++ b/zenith_utils/src/zid.rs
@@ -96,6 +96,20 @@ macro_rules! zid_newtype {
            }
        }

+        impl FromHex for $t {
+            type Error = hex::FromHexError;
+
+            fn from_hex<T: AsRef<[u8]>>(hex: T) -> Result<Self, Self::Error> {
+                Ok($t(ZId::from_hex(hex)?))
+            }
+        }
+
+        impl AsRef<[u8]> for $t {
+            fn as_ref(&self) -> &[u8] {
+                &self.0 .0
+            }
+        }
+
        impl fmt::Display for $t {
            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
                self.0.fmt(f)
@@ -137,19 +151,3 @@ zid_newtype!(ZTimelineId);
 pub struct ZTenantId(ZId);

 zid_newtype!(ZTenantId);
-
-// for now the following impls are used only with ZTenantId,
-// if this impls become useful in other newtypes they can be moved under zid_newtype macro too
-impl FromHex for ZTenantId {
-    type Error = hex::FromHexError;
-
-    fn from_hex<T: AsRef<[u8]>>(hex: T) -> Result<Self, Self::Error> {
-        Ok(ZTenantId(ZId::from_hex(hex)?))
-    }
-}
-
-impl AsRef<[u8]> for ZTenantId {
-    fn as_ref(&self) -> &[u8] {
-        &self.0 .0
-    }
-}
Author	SHA1	Message	Date
anastasia	01e2e88135	Fixes to checkpoint.nextXid accounting: - Advance next_xid only when incoming xlog record has one. - Advance next_xid by one. Remove XID_CHECKPOINT_INTERVAL.	2021-09-01 12:26:57 +03:00
anastasia	d64c083dad	Add batch_wraparound/test_xid_wraparound.py	2021-09-01 12:26:57 +03:00
Heikki Linnakangas	b45d5368b0	Don't create an image layer for dropped relations. I noticed that the timeline directory contained files like this: pg_xact_0000_0_000000000169C3C2_00000000016BB399 pg_xact_0000_0_00000000016BB399 pg_xact_0000_0_00000000016BB399_00000000016BDD06 pg_xact_0000_0_00000000016BDD06 pg_xact_0000_0_00000000016BDD06_00000000016C63AA pg_xact_0000_0_00000000016C63AA pg_xact_0000_0_00000000016C63AA_0000000001765226_DROPPED pg_xact_0000_0_0000000001765226 pg_xact_0001_0_00000000016BB77E_00000000016BDD06 pg_xact_0001_0_00000000016BDD06 pg_xact_0001_0_00000000016BDD06_0000000001765226_DROPPED pg_xact_0001_0_0000000001765226 Note how there is an image file after each DROPPED file. It's a waste of time and space to materialize an image of the file at the point where it's dropped, no one is going to request pages on a dropped relation. And it's a correctness issue too: list_rels() and list_nonrels() will not consider the relation as unlinked, unless the latest layer indicates so, and there is no concept of a dropped image layer. That was causing test_clog_truncate test to fail, when I adjusted the checkpointer to force a checkpoint more aggressively. There are a bunch more issues related to dropped rels and branching, see https://github.com/zenithdb/zenith/issues/502. Hence this doesn't completely fix the issue I saw with test_clog_truncate either. But it's a start.	2021-09-01 09:42:18 +03:00
Max Sharnoff	625abf3c52	Bump vendor/postgres for walproposer cleanup ref zenithdb/postgres#69	2021-08-31 13:09:16 -07:00
anastasia	c0ace1efff	Bump vendor/postgres to use relsize cache.	2021-08-31 14:10:50 +03:00
Kirill Bulatov	03a09b7827	Replace old git urls with the current ones	2021-08-30 23:51:47 +03:00
Heikki Linnakangas	63d0a865f4	Update and move comment. The comment talked about the WAL redo thread, but commit `6e22a8f709` refactored that away. The problem the comment describes probably still exists, so keep the comment, but update the wording.	2021-08-30 20:35:08 +03:00
Patrick Insinger	5ac4a27042	image_layer - read images directly from disk Avoid slurping entire image files into memory. For blocky segments, we write the bytes directly to a bookfile chapter. The blocks are a fixed size, which allows for random access.	2021-08-30 10:34:36 -07:00
Patrick Insinger	7c7e89e2ea	layered_repo - atomic last/prev record_lsn Make a new type that stores both Lsns. Use an RwLock for thread safety.	2021-08-30 09:40:13 -07:00
Patrick Insinger	561bf2c510	circleci - fix test summary	2021-08-30 09:18:49 -07:00
Patrick Insinger	98f49671c1	delta_layer - read page versions from disk split the page versions into two chapters: PAGE_VERSION_METAS - a rust BTreeMap from (block #, lsn) -> page & WAL byte ranges in PAGE_VERSIONS_CHAPTER PAGE_VERSIONS_CHAPTER - raw page images and serialized WAL records	2021-08-30 09:12:38 -07:00
anastasia	78963ad104	Issue #411 . Support drop database in pageserver. Use put_unlink for FileNodeMap relishes. Always store FileNodeMap as materialized page images (no wal records).	2021-08-30 17:29:29 +03:00
anastasia	27442c3daa	Add test for DROP DATABASE command	2021-08-30 17:29:29 +03:00
anastasia	e29bfa09b2	Fix list_rels and list_nonrels in layeredRepository - return only visible objects	2021-08-30 17:29:29 +03:00
Heikki Linnakangas	b949127b06	Rename page_cache.rs to tenant_mgr.rs. Once upon a time, 'page_cache.rs' contained an actual page cache, but it hasn't for a very long time. Rename to reflect what it actually does these days.	2021-08-30 15:17:30 +03:00
Heikki Linnakangas	a3f3d46016	Misc doc updates	2021-08-30 14:29:21 +03:00
Heikki Linnakangas	c5fc4e6905	Fix instructions in README.md on how to start psql Commit `c4b2bf7ebd` changed the bootstrap superuser name.	2021-08-30 14:29:21 +03:00
Heikki Linnakangas	9dfee8a3b5	Add Gauge for # of layers Seems like a useful metric	2021-08-30 12:58:15 +03:00
sharnoff	263e03f4b8	Improve code & text formatting in proxy welcome Adds some named formatting variables to make things a little more clear. Also adds some words & commas to the message itself.	2021-08-30 12:41:47 +03:00
Heikki Linnakangas	074bd3bb12	Add basic performance test framework. This provides a pytest fixture to record metrics from pytest tests. The The recorded metrics are printed out at the end of the tests. As a starter, this includes on small test, using pgbench. It prints out three metrics: the initialization time, runtime of 5000 xacts, and the repository size after the tests.	2021-08-27 21:00:45 +03:00
Alexey Kondratov	e1d8f97b9e	Mention `pipenv run` as an option to run pytest	2021-08-27 19:46:51 +03:00
Alexey Kondratov	7e7b31a626	Extract basebackup directly from the CopyOutReader Do not fetch it into the intermediate buffer.	2021-08-27 19:46:51 +03:00
Heikki Linnakangas	787806285d	Remove unused 'update_meta' argument. It was used by the object repository code, but now that that's gone, it's dead.	2021-08-27 15:45:45 +03:00
Arseny Sher	7474cfac08	Rename VCL to epochStartLsn and restart_lsn to truncate_lsn. epochStartLsn is the LSN since which new proposer writes its WAL in its epoch, let's be more explicit here. truncate_lsn is LSN still needed by the most lagging safekeeper. restart_lsn is terminology from pg_replicaton_slots, but here we don't really have 'restart'; hopefully truncate word makes it clearer.	2021-08-27 15:22:10 +03:00
Arseny Sher	6cbc08f1fb	bump pg version	2021-08-27 15:22:10 +03:00
Arseny Sher	8d3450f4c6	Basic safekeeper refactoring and bug fixing. 1) Extract consensus logic to safekeeper.rs. 2) Change the voting flow so that acceptor tells his epoch along with giving the vote, not before it; otherwise it might get immediately stale. #294 3) Process messages from compute atomically and sync state properly. #270 4) Use separate structs for disk and network. ref #315	2021-08-27 15:22:10 +03:00
Heikki Linnakangas	4902d1daa8	Store base images in separate ImageLayers Previously, a SnapshotLayer and corresponding file on disk contained the base image of every page in the segment at the start LSN, and all the changes (= WAL records) in the range between start and end LSN. That was a bit awkward, because we had to keep the base image of every page in memory until we had accumulated enough WAL after the base image to write out the layer. When it's time to write out a layer, we would really want to replay the WAL to reconstruct the most recent version of each page, to save the effort later. That's on the assumption that the client will usually request the most recent version, not some older one. Split the SnapshotLayer into two structs: ImageLayer and DeltaLayer. An image layer contains a "snapshot" of the segment at one specific LSN, and no WAL records, whereas a delta layer contains WAL records in a range of LSNs. In order to reconstruct a page version in the delta layer, by performing WAL redo, you also need the previous image layer. So the delta layers are "incremental" against the previous layer. So where previously we would create snapshot files like this: rel_100_200 rel_200_300 rel_300_400 We now create image and delta files like this: rel_100 # image rel_100_200 # delta rel_200 rel_200_300 rel_300 rel_300_400 rel_400 That's more files, but as discussed above, this allows storing more up-to-date page versions on disk, which should reduce the latency of responding to a GetPage request. It also allows more fine-grained garbage collection. In the above example, after the old page version are no longer needed and if the relation is not modified anymore, we only need to keep the latest image file, 'rel_400', and everything else can be removed. Implements https://github.com/zenithdb/zenith/issues/339	2021-08-27 02:35:16 +03:00
Heikki Linnakangas	40c79988a8	Move code to handle snapshot filenames This isn't very useful yet, but the next commit will add more code related to handling the filenames.	2021-08-27 02:35:16 +03:00
Patrick Insinger	d265b4cdd3	waldecoder - check for trailing bytes When we parse the main data in a WAL record, ensure we consume all bytes.	2021-08-26 10:24:33 -07:00
Konstantin Knizhnik	beaa2cd0a2	Handle COPY error	2021-08-26 13:53:10 +03:00
Arseny Sher	c4450907e5	Don't hide exact error of get_timeline. ref #470	2021-08-25 20:46:31 +03:00
Heikki Linnakangas	de9d5e0aa4	Remove unnecessary dependencies. Found by "cargo udeps"	2021-08-25 18:51:15 +03:00
Heikki Linnakangas	4046530160	Remove remnants of choosing between repository formats. Now that we only have one Repository implementation, no need for the command-line options to choose it either. I'm removing these as a separate commit to show what we will need to do if we add another Repository implementation in the future (even though I don't foresee us doing that any time soon)	2021-08-25 18:37:22 +03:00
Heikki Linnakangas	5998744bcc	Remove rocksdb implementation. The layered storage format is good enough that we don't need the rocksdb implementation anymore. There are a lot of known issues but we'll keep working on them.	2021-08-25 18:37:22 +03:00
Heikki Linnakangas	250ae643a8	Remove 'zenith push' feature. Now that the new storage format is based on immutable files, we want to implement push/pull in terms of these immutable files as well. Similarly to how those files will be transferred between S3 and the page server. The implementation we had was fairly tightly coupled with the object repository implementation, but I'm about to remove the object / rocksdb storage format soon. That would leave the current "zenith push" command completely broken. It seemed like a good idea at the time, but in hindsight, it was premature to implement push/pull yet. It's a nice feature and I'd like to see it reimplemented in the future, but in the meanwhile, let's remove the code we had. We can dig the parts of it that might be useful in the future from the git history.	2021-08-25 18:37:22 +03:00
Dmitry Ivanov	3edad463fb	Adjust docker container for console's CI pipeline	2021-08-25 17:28:42 +03:00
Heikki Linnakangas	19fcea99da	If too much memory is being used for in-memory layers, flush oldest one. The old policy was to flush all in-memory layers to disk every 10 seconds. That was a pretty dumb policy, unnecessarily aggressive. This commit changes the policy so that we only flush layers where the oldest WAL record is older than 16 MB from the last valid LSN on the timeline. That's still pretty aggressive, but it's a step in the right direction. We do need a limit on how old the oldest in-memory layer is allowed to be, because that determines how much WAL the safekeepers need to hold onto, and how much WAL we need to reprocess in case of a page server crash. 16 MB is surely still too aggressive for that, but it's easy to change the setting later. To support that, keep all in-memory layers in a binary heap, so that we can easily find the one with the oldest LSN. This tracks and a new LSN value in the metadata file: 'disk_consistent_lsn'. Before, on page server restart we restarted the WAL processing from the 'last_record_lsn' value, but now that we don't flush everything to disk in one go, the 'last_record_lsn' tracked in memory is usually ahead of the last record that's been flushed to disk. Even though we track that oldest LSN now, the crash recovery story isn't really complete. We don't do fsync()s anywhere, and thing will break if a snapshot file isn't complete, as there's no CRC on them. That's not new, and it's a TODO.	2021-08-25 11:20:47 +03:00
Dmitry Rodionov	f2f02a8af0	apply transformation (Arc<Option> -> Option<Arc>) suggested by @funbringer	2021-08-24 19:05:00 +03:00
Dmitry Rodionov	b135723994	review adjustments	2021-08-24 19:05:00 +03:00
Dmitry Rodionov	23b5249512	translate pageserver api to http	2021-08-24 19:05:00 +03:00
Eric Seppanen	41fa02f82b	Replace transmute with serde Upgrade to bindgen 0.59, which has two new abilities: - specify arbitrary #[derive] attributes to attach to generated structs - request explicit padding fields These two features are enough to replace transmute with serde/bincode.	2021-08-24 16:32:37 +03:00
Heikki Linnakangas	81dd4bc41e	Fix decoding XLOG_HEAP_DELETE and XLOG_HEAP_UPDATE records. Because the t_cid field was missing from the XlHeapDelete struct that corresponds to the PostgreSQL xl_heap_delete struct, the check for the XLH_DELETE_ALL_VISIBLE_CLEARED flag did not work correctly. Decoding XlHeapUpdate struct was also missing the t_cid field, but that didn't cause any immediate problems because in that struct, the t_cid field is after all the fields that the page server cares about. But fix that too, as it was an accident waiting to happen. The bug was mostly hidden by the VM page handling in zenith_wallog_page, where it forcibly generates a FPW record whenever a VM page is evicted: else if (forknum == VISIBILITYMAP_FORKNUM && !RecoveryInProgress()) { /* * Always WAL-log vm. * We should never miss clearing visibility map bits. * * TODO Is it too bad for performance? * Hopefully we do not evict actively used vm too often. */ XLogRecPtr recptr; recptr = log_newpage_copy(&reln->smgr_rnode.node, forknum, blocknum, buffer, false); XLogFlush(recptr); lsn = recptr; But that was just hiding the issue: it's still visible if you had a read-only node relying on the data in the page server, or you killed and restarted the primary node, or you started a branch. In the included test case, I used a new branch to expose this. Fixes https://github.com/zenithdb/zenith/issues/461	2021-08-24 15:59:25 +03:00
anastasia	ad8b5c3845	use updated vendor/postgres	2021-08-23 18:19:59 +03:00
Dmitry Rodionov	dcaa2126f1	fix code format after main rebase	2021-08-23 18:01:59 +03:00
Dmitry Rodionov	b29ca232d6	add ability to disable colors, use argparse for arguments	2021-08-23 17:28:45 +03:00
Dmitry Rodionov	8c62b11bd5	adjust for review	2021-08-23 17:28:45 +03:00
Dmitry Rodionov	35b60d509f	Add support for code format checking using rustfmt in optional pre-commit hook and in ci pipeline. Found issues can be fixed automatically via make fmt.	2021-08-23 17:28:45 +03:00
Dmitry Rodionov	d989580c1c	remove small code duplication involving InMemoryLayer::get_seg_size, and remove redundant Option around new snapshot layer in InMemoryLayer::freeze	2021-08-23 13:00:05 +03:00
anastasia	798160544c	Update zenith readmes: - Move source tree overview into separate docs/sourcetree.md and update it. - Add glossary: docs/glossary.md - Add a draft of Architecture overview to main Readme.md	2021-08-23 10:21:10 +03:00
Max Sharnoff	39bb6fb19c	Marginally improve walkeeper error visibility (#440 ) Adds a warning if a postgres query fails, and some additional context to errors generated inside `ReceiveWalConn::run`	2021-08-19 08:46:18 -07:00
Dmitry Rodionov	82725725fd	update README to match required Rust version and new python package installation process	2021-08-19 17:42:52 +03:00
Alexey Kondratov	1c3d51ed92	Add Docker images building doc and refactor the overall docs reference	2021-08-19 15:12:35 +03:00
Alexey Kondratov	04a309f562	Build zenithdb/zenith:latest in CI (zenithdb/console#18 )	2021-08-19 15:12:35 +03:00
anastasia	20e6cd7724	Update test_twophase - check that we correctly restore files at compute node start.	2021-08-19 12:15:09 +03:00
Heikki Linnakangas	9fed5c8fb7	Add test for page server restart.	2021-08-18 20:19:07 +03:00