REVERT ME. Add unlink test and a bunch of debug printlns

Rename put_unlink() to drop_segment() in Layer trait.

.circleci/config.yml

@@ -7,9 +7,19 @@ executors:
   zenith-build-executor:
     resource_class: xlarge
     docker:
-      - image: cimg/rust:1.51.0
+      - image: cimg/rust:1.52.1
 
 jobs:
+  check-codestyle:
+    executor: zenith-build-executor
+    steps:
+      - checkout
+
+      - run:
+          name: rustfmt
+          when: always
+          command: |
+            cargo fmt --all -- --check
 
   # A job to build postgres
   build-postgres:
@@ -37,7 +47,7 @@ jobs:
           command: |
             if [ ! -e tmp_install/bin/postgres ]; then
               sudo apt update
-              sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libcurl4-openssl-dev
+              sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev
             fi
 
         # Build postgres if the restore_cache didn't find a build.
@@ -119,8 +129,7 @@ jobs:
             - target
 
         # Run rust unit tests
-        # FIXME: remove -p zenith_utils once integration tests are moved to python
-      - run: cargo test -p zenith_utils
+      - run: cargo test
 
         # Install the rust binaries, for use by test jobs
         # `--locked` is required; otherwise, `cargo install` will ignore Cargo.lock.
@@ -192,7 +201,12 @@ jobs:
           condition: << parameters.needs_postgres_source >>
           steps:
             - run: git submodule update --init --depth 1
-      - run: pip install pytest psycopg2
+      - run:
+          name: Install pipenv & deps
+          working_directory: test_runner
+          command: |
+            pip install pipenv
+            pipenv install
       - run:
           name: Run pytest
           working_directory: test_runner
@@ -211,25 +225,21 @@ jobs:
             #
             # The junit.xml file allows CircleCI to display more fine-grained test information
             # in its "Tests" tab in the results page.
-            pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short $TEST_SELECTION $EXTRA_PARAMS
+            # -s prevents pytest from capturing output, which helps to see
+            # what's going on if the test hangs
+            # --verbose prints name of each test (helpful when there are
+            # multiple tests in one file)
+            # -rA prints summary in the end
+            pipenv run pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short -s --verbose -rA $TEST_SELECTION $EXTRA_PARAMS
       - run:
           # CircleCI artifacts are preserved one file at a time, so skipping
           # this step isn't a good idea. If you want to extract the
           # pageserver state, perhaps a tarball would be a better idea.
-          name: Delete pageserver data
+          name: Delete all data but logs
           when: always
           command: |
             du -sh /tmp/test_output/*
-            for DIR in /tmp/test_output/*; do
-              mv $DIR/repo/pageserver.log $DIR/ || true # ignore errors
-              for PGDIR in $DIR/repo/pgdatadirs/pg?; do
-                echo "PGDIR: $PGDIR"
-                NEW_LOG="${PGDIR##*/}_log"
-                mv $PGDIR/log "$DIR/$NEW_LOG" || true # ignore errors
-              done
-              echo "rm $DIR/repo"
-              rm -rf $DIR/repo
-            done
+            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "wal_acceptor.log" ! -name "regression.diffs" ! -name "junit.xml" -delete
             du -sh /tmp/test_output/*
       - store_artifacts:
           path: /tmp/test_output
@@ -237,9 +247,27 @@ jobs:
       - store_test_results:
           path: /tmp/test_output
 
+  # Build zenithdb/zenith:latest image and push it to Docker hub
+  docker-image:
+    docker:
+      - image: cimg/base:2021.04
+    steps:
+      - checkout
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - run:
+          name: Init postgres submodule
+          command: git submodule update --init --depth 1
+      - run:
+          name: Build and push Docker image
+          command: |
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
+            docker build -t zenithdb/zenith:latest . && docker push zenithdb/zenith:latest
+
 workflows:
   build_and_test:
     jobs:
+      - check-codestyle
       - build-postgres
       - build-zenith:
           name: build-zenith-<< matrix.build_type >>
@@ -265,3 +293,20 @@ workflows:
           test_selection: batch_others
           requires:
             - build-zenith-<< matrix.build_type >>
+      - run-pytest:
+          name: benchmarks
+          build_type: release
+          test_selection: performance
+          requires:
+            - build-zenith-release
+      - docker-image:
+          # Context gives an ability to login
+          context: Docker Hub
+          # Build image only for commits to main
+          filters:
+            branches:
+              only:
+                - main
+          requires:
+            - pg_regress tests release
+            - other tests release

.dockerignore

@@ -0,0 +1,13 @@
+**/.git/
+**/__pycache__
+**/.pytest_cache
+
+/target
+/tmp_check
+/tmp_install
+/tmp_check_cli
+/test_output
+/.vscode
+/.zenith
+/integration_tests/.zenith
+/Dockerfile

.github/workflows/testing.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Install postgres dependencies
         run: |
           sudo apt update
-          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libcurl4-openssl-dev
+          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev
 
       - name: Set pg revision for caching
         id: pg_ver

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "vendor/postgres"]
 	path = vendor/postgres
-	url = https://github.com/libzenith/postgres
+	url = https://github.com/zenithdb/postgres
 	branch = main

CONTRIBUTING.md

@@ -0,0 +1,31 @@
+# How to contribute
+
+Howdy! Usual good software engineering practices apply. Write
+tests. Write comments. Follow standard Rust coding practices where
+possible. Use 'cargo fmt' and 'clippy' to tidy up formatting.
+
+There are soft spots in the code, which could use cleanup,
+refactoring, additional comments, and so forth. Let's try to raise the
+bar, and clean things up as we go. Try to leave code in a better shape
+than it was before.
+
+## Submitting changes
+
+1. Make a PR for every change.
+
+   Even seemingly trivial patches can break things in surprising ways.
+Use of common sense is OK. If you're only fixing a typo in a comment,
+it's probably fine to just push it. But if in doubt, open a PR.
+
+2. Get at least one +1 on your PR before you push.
+
+   For simple patches, it will only take a minute for someone to review
+it.
+
+3. Always keep the CI green.
+
+   Do not push, if the CI failed on your PR. Even if you think it's not
+your patch's fault. Help to fix the root cause if something else has
+broken the CI, before pushing.
+
+*Happy Hacking!*

Cargo.toml

@@ -1,11 +1,17 @@
 [workspace]
 members = [
-    "integration_tests",
-    "pageserver",
-    "walkeeper",
-    "zenith",
     "control_plane",
+    "pageserver",
     "postgres_ffi",
-    "zenith_utils",
+    "proxy",
+    "walkeeper",
     "workspace_hack",
+    "zenith",
+    "zenith_metrics",
+    "zenith_utils",
 ]
+
+[profile.release]
+# This is useful for profiling and, to some extent, debug.
+# Besides, debug info should not affect the performance.
+debug = true

Dockerfile

@@ -0,0 +1,78 @@
+#
+# Docker image for console integration testing.
+#
+
+#
+# Build Postgres separately --- this layer will be rebuilt only if one of
+# mentioned paths will get any changes.
+#
+FROM zenithdb/build:buster AS pg-build
+WORKDIR /zenith
+COPY ./vendor/postgres vendor/postgres
+COPY ./Makefile Makefile
+RUN make -j $(getconf _NPROCESSORS_ONLN) -s postgres
+
+#
+# Calculate cargo dependencies.
+# This will always run, but only generate recipe.json with list of dependencies without
+# installing them.
+#
+FROM zenithdb/build:buster AS cargo-deps-inspect
+WORKDIR /zenith
+COPY . .
+RUN cargo chef prepare --recipe-path /zenith/recipe.json
+
+#
+# Build cargo dependencies.
+# This temp cantainner should be rebuilt only if recipe.json was changed.
+#
+FROM zenithdb/build:buster AS deps-build
+WORKDIR /zenith
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=cargo-deps-inspect /usr/local/cargo/bin/cargo-chef /usr/local/cargo/bin/
+COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
+RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json
+
+#
+# Build zenith binaries
+#
+FROM zenithdb/build:buster AS build
+WORKDIR /zenith
+COPY . .
+# Copy cached dependencies
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=deps-build /zenith/target target
+COPY --from=deps-build /usr/local/cargo/ /usr/local/cargo/
+RUN cargo build --release
+
+#
+# Copy binaries to resulting image.
+#
+FROM debian:buster-slim
+WORKDIR /data
+
+RUN apt-get update && apt-get -yq install librocksdb-dev libseccomp-dev openssl && \
+    mkdir zenith_install
+
+COPY --from=build /zenith/target/release/pageserver /usr/local/bin
+COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin
+COPY --from=build /zenith/target/release/proxy /usr/local/bin
+COPY --from=pg-build /zenith/tmp_install postgres_install
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+
+# Remove build artifacts (~ 500 MB)
+RUN rm -rf postgres_install/build && \
+    # 'Install' Postgres binaries locally
+    cp -r postgres_install/* /usr/local/ && \
+    # Prepare an archive of Postgres binaries (should be around 11 MB)
+    # and keep it inside container for an ease of deploy pipeline.
+    cd postgres_install && tar -czf /data/postgres_install.tar.gz . && cd .. && \
+    rm -rf postgres_install
+
+RUN useradd -d /data zenith && chown -R zenith:zenith /data
+
+VOLUME ["/data"]
+USER zenith
+EXPOSE 6400
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["pageserver"]

Dockerfile.alpine

@@ -0,0 +1,95 @@
+#
+# Docker image for console integration testing.
+#
+# We may also reuse it in CI to unify installation process and as a general binaries building
+# tool for production servers.
+#
+# Dynamic linking is used for librocksdb and libstdc++ bacause librocksdb-sys calls
+# bindgen with "dynamic" feature flag. This also prevents usage of dockerhub alpine-rust
+# images which are statically linked and have guards against any dlopen. I would rather
+# prefer all static binaries so we may change the way librocksdb-sys builds or wait until
+# we will have our own storage and drop rockdb dependency.
+#
+# Cargo-chef is used to separate dependencies building from main binaries building. This
+# way `docker build` will download and install dependencies only of there are changes to
+# out Cargo.toml files.
+#
+
+
+#
+# build postgres separately -- this layer will be rebuilt only if one of
+# mentioned paths will get any changes
+#
+FROM alpine:3.13 as pg-build
+RUN apk add --update clang llvm compiler-rt compiler-rt-static lld musl-dev binutils \
+                     make bison flex readline-dev zlib-dev perl linux-headers libseccomp-dev
+WORKDIR zenith
+COPY ./vendor/postgres vendor/postgres
+COPY ./Makefile Makefile
+# Build using clang and lld
+RUN CC='clang' LD='lld' CFLAGS='-fuse-ld=lld --rtlib=compiler-rt' make postgres -j4
+
+#
+# Calculate cargo dependencies.
+# This will always run, but only generate recipe.json with list of dependencies without
+# installing them.
+#
+FROM alpine:20210212 as cargo-deps-inspect
+RUN apk add --update rust cargo
+RUN cargo install cargo-chef
+WORKDIR zenith
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+#
+# Build cargo dependencies.
+# This temp cantainner would be build only if recipe.json was changed.
+#
+FROM alpine:20210212 as deps-build
+RUN apk add --update rust cargo openssl-dev clang build-base
+# rust-rocksdb can be built against system-wide rocksdb -- that saves about
+# 10 minutes during build. Rocksdb apk package is in testing now, but use it
+# anyway. In case of any troubles we can download and build rocksdb here manually
+# (to cache it as a docker layer).
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
+WORKDIR zenith
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=cargo-deps-inspect /root/.cargo/bin/cargo-chef /root/.cargo/bin/
+COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
+RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json
+
+#
+# Build zenith binaries
+#
+FROM alpine:20210212 as build
+RUN apk add --update rust cargo openssl-dev clang build-base
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
+WORKDIR zenith
+COPY . .
+# Copy cached dependencies
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=deps-build /zenith/target target
+COPY --from=deps-build /root/.cargo /root/.cargo
+RUN cargo build --release
+
+#
+# Copy binaries to resulting image.
+# build-base hare to provide libstdc++ (it will also bring gcc, but leave it this way until we figure
+# out how to statically link rocksdb or avoid it at all).
+#
+FROM alpine:3.13
+RUN apk add --update openssl build-base libseccomp-dev
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb
+COPY --from=build /zenith/target/release/pageserver /usr/local/bin
+COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin
+COPY --from=build /zenith/target/release/proxy /usr/local/bin
+COPY --from=pg-build /zenith/tmp_install /usr/local
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+
+RUN addgroup zenith && adduser -h /data -D -G zenith zenith
+VOLUME ["/data"]
+WORKDIR /data
+USER zenith
+EXPOSE 6400
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["pageserver"]

Dockerfile.build

@@ -0,0 +1,15 @@
+#
+# Image with all the required dependencies to build https://github.com/zenithdb/zenith
+# and Postgres from https://github.com/zenithdb/postgres
+# Also includes some rust development and build tools.
+#
+FROM rust:slim-buster
+WORKDIR /zenith
+
+# Install postgres and zenith build dependencies
+# clang is for rocksdb
+RUN apt-get update && apt-get -yq install automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \
+                                          libseccomp-dev pkg-config libssl-dev librocksdb-dev clang
+
+# Install rust tools
+RUN rustup component add clippy && cargo install cargo-chef cargo-audit

Makefile

@@ -1,3 +1,11 @@
+# Seccomp BPF is only available for Linux
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Linux)
+	SECCOMP = --with-libseccomp
+else
+	SECCOMP =
+endif
+
 #
 # Top level Makefile to build Zenith and PostgreSQL
 #
@@ -13,6 +21,7 @@ all: zenith postgres
 ### Zenith Rust bits
 #
 # The 'postgres_ffi' depends on the Postgres headers.
+.PHONY: zenith
 zenith: postgres-headers
 	cargo build
 
@@ -21,25 +30,33 @@ tmp_install/build/config.status:
 	+@echo "Configuring postgres build"
 	mkdir -p tmp_install/build
 	(cd tmp_install/build && \
-	../../vendor/postgres/configure CFLAGS='-O0' --enable-debug --enable-cassert \
-	    --enable-depend --with-libxml --prefix=$(abspath tmp_install) > configure.log)
+	../../vendor/postgres/configure CFLAGS='-O0 -g3 $(CFLAGS)' \
+		--enable-cassert \
+		--enable-debug \
+		--enable-depend \
+		$(SECCOMP) \
+		--prefix=$(abspath tmp_install) > configure.log)
 
 # nicer alias for running 'configure'
+.PHONY: postgres-configure
 postgres-configure: tmp_install/build/config.status
 
 # Install the PostgreSQL header files into tmp_install/include
+.PHONY: postgres-headers
 postgres-headers: postgres-configure
 	+@echo "Installing PostgreSQL headers"
 	$(MAKE) -C tmp_install/build/src/include MAKELEVEL=0 install
 
 
 # Compile and install PostgreSQL and contrib/zenith
+.PHONY: postgres
 postgres: postgres-configure
 	+@echo "Compiling PostgreSQL"
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 install
 	+@echo "Compiling contrib/zenith"
-	(cd vendor/postgres/contrib/zenith && \
-	$(MAKE) PG_CONFIG=$(abspath tmp_install)/bin/pg_config install USE_PGXS=1)
+	$(MAKE) -C tmp_install/build/contrib/zenith install
+	+@echo "Compiling contrib/zenith_test_utils"
+	$(MAKE) -C tmp_install/build/contrib/zenith_test_utils install
 
 postgres-clean:
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 clean
@@ -54,4 +71,10 @@ distclean:
 	rm -rf tmp_install
 	cargo clean
 
-.PHONY: postgres-configure postgres postgres-headers zenith
+.PHONY: fmt
+fmt:
+	./pre-commit.py --fix-inplace
+
+.PHONY: setup-pre-commit-hook
+setup-pre-commit-hook:
+	ln -s -f ../../pre-commit.py .git/hooks/pre-commit

Pipfile

@@ -0,0 +1 @@
+./test_runner/Pipfile

Pipfile.lock

@@ -0,0 +1 @@
+./test_runner/Pipfile.lock

README.md

@@ -2,29 +2,42 @@
 
 Zenith substitutes PostgreSQL storage layer and redistributes data across a cluster of nodes
 
+## Architecture overview
+
+A Zenith installation consists of Compute nodes and Storage engine.
+
+Compute nodes are stateles PostgreSQL nodes, backed by zenith storage.
+
+Zenith storage engine consists of two major components:
+- Pageserver. Scalable storage backend for compute nodes.
+- WAL service. The service that receives WAL from compute node and ensures that it is stored durably.
+
+Pageserver consists of:
+- Repository - Zenith storage implementation.
+- WAL receiver - service that receives WAL from WAL service and stores it in the repository.
+- Page service - service that communicates with compute nodes and responds with pages from the repository.
+- WAL redo - service that builds pages from base images and WAL records on Page service request.
+
 ## Running local installation
 
 1. Install build dependencies and other useful packages
 
 On Ubuntu or Debian this set of packages should be sufficient to build the code:
 ```text
-apt install build-essential libtool libreadline-dev zlib1g-dev flex bison \
-libxml2-dev libcurl4-openssl-dev libssl-dev clang
+apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libseccomp-dev \
+libssl-dev clang
 ```
 
-[Rust] 1.48 or later is also required.
+[Rust] 1.52 or later is also required.
 
 To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively.
 
 To run the integration tests (not required to use the code), install
-Python (3.6 or higher), and install python3 packages with `pip` (called `pip3` on some systems):
-```
-pip install pytest psycopg2
-```
+Python (3.6 or higher), and install python3 packages with `pipenv` using `pipenv install` in the project directory.
 
 2. Build zenith and patched postgres
 ```sh
-git clone --recursive https://github.com/libzenith/zenith.git
+git clone --recursive https://github.com/zenithdb/zenith.git
 cd zenith
 make -j5
 ```
@@ -34,8 +47,7 @@ make -j5
 # Create repository in .zenith with proper paths to binaries and data
 # Later that would be responsibility of a package install script
 > ./target/debug/zenith init
-<...>
-new zenith repository was created in .zenith
+pageserver init succeeded
 
 # start pageserver
 > ./target/debug/zenith start
@@ -55,7 +67,7 @@ main	127.0.0.1:55432	0/1609610	running
 
 4. Now it is possible to connect to postgres and run some queries:
 ```text
-> psql -p55432 -h 127.0.0.1 postgres
+> psql -p55432 -h 127.0.0.1 -U zenith_admin postgres
 postgres=# CREATE TABLE t(key int primary key, value text);
 CREATE TABLE
 postgres=# insert into t values(1,1);
@@ -85,9 +97,9 @@ waiting for server to start.... done
 
 # this new postgres instance will have all the data from 'main' postgres,
 # but all modifications would not affect data in original postgres
-> psql -p55433 -h 127.0.0.1 postgres
+> psql -p55433 -h 127.0.0.1 -U zenith_admin postgres
 postgres=# select * from t;
- key | value 
+ key | value
 -----+-------
    1 | 1
 (1 row)
@@ -99,36 +111,23 @@ INSERT 0 1
 ## Running tests
 
 ```sh
-git clone --recursive https://github.com/libzenith/zenith.git
+git clone --recursive https://github.com/zenithdb/zenith.git
 make # builds also postgres and installs it to ./tmp_install
 cd test_runner
 pytest
 ```
 
-## Source tree layout
+## Documentation
 
-`/walkeeper`:
+Now we use README files to cover design ideas and overall architecture for each module and `rustdoc` style documentation comments. See also [/docs/](/docs/) a top-level overview of all available markdown documentation.
 
-WAL safekeeper. Written in Rust.
+- [/docs/sourcetree.md](/docs/sourcetree.md) contains overview of source tree layout.
 
-`/pageserver`:
+To view your `rustdoc` documentation in a browser, try running `cargo doc --no-deps --open`
 
-Page Server. Written in Rust.
+## Join the development
 
-Depends on the modified 'postgres' binary for WAL redo.
-
-`/vendor/postgres`:
-
-PostgreSQL source tree, with the modifications needed for Zenith.
-
-`/vendor/postgres/src/bin/safekeeper`:
-
-Extension (safekeeper_proxy) that runs in the compute node, and connects to the WAL safekeepers
-and streams the WAL
-
-`/test_runner`:
-
-Integration tests, written in Python using the `pytest` framework.
-
-
-[Rust]: https://www.rust-lang.org/learn/get-started
+- Read `CONTRIBUTING.md` to learn about project code style and practices.
+- Use glossary in [/docs/glossary.md](/docs/glossary.md)
+- To get familiar with a source tree layout, use [/docs/sourcetree.md](/docs/sourcetree.md).
+- To learn more about PostgreSQL internals, check http://www.interdb.jp/pg/index.html

control_plane/Cargo.toml

@@ -16,12 +16,11 @@ toml = "0.5"
 lazy_static = "1.4"
 regex = "1"
 anyhow = "1.0"
-# hex = "0.4.3"
 bytes = "1.0.1"
-# fs_extra = "1.2.0"
 nix = "0.20"
-# thiserror = "1"
 url = "2.2.2"
+hex = { version = "0.4.3", features = ["serde"] }
+reqwest = { version = "0.11", features = ["blocking", "json"] }
 
 pageserver = { path = "../pageserver" }
 walkeeper = { path = "../walkeeper" }

control_plane/src/compute.rs

@@ -1,3 +1,4 @@
+use std::fs::{self, File, OpenOptions};
 use std::io::Write;
 use std::net::SocketAddr;
 use std::net::TcpStream;
@@ -6,18 +7,16 @@ use std::process::Command;
 use std::sync::Arc;
 use std::time::Duration;
 use std::{collections::BTreeMap, path::PathBuf};
-use std::{
-    fs::{self, OpenOptions},
-    io::Read,
-};
 
 use anyhow::{Context, Result};
 use lazy_static::lazy_static;
 use regex::Regex;
+use zenith_utils::connstring::connection_host_port;
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::ZTenantId;
+use zenith_utils::zid::ZTimelineId;
 
 use crate::local_env::LocalEnv;
-use pageserver::ZTimelineId;
-
 use crate::storage::PageServerNode;
 
 //
@@ -26,27 +25,36 @@ use crate::storage::PageServerNode;
 pub struct ComputeControlPlane {
     base_port: u16,
     pageserver: Arc<PageServerNode>,
-    pub nodes: BTreeMap<String, Arc<PostgresNode>>,
+    pub nodes: BTreeMap<(ZTenantId, String), Arc<PostgresNode>>,
     env: LocalEnv,
 }
 
 impl ComputeControlPlane {
     // Load current nodes with ports from data directories on disk
+    // Directory structure has the following layout:
+    // pgdatadirs
+    // |- tenants
+    // |  |- <tenant_id>
+    // |  |   |- <branch name>
     pub fn load(env: LocalEnv) -> Result<ComputeControlPlane> {
         // TODO: since pageserver do not have config file yet we believe here that
         // it is running on default port. Change that when pageserver will have config.
         let pageserver = Arc::new(PageServerNode::from_env(&env));
 
+        let mut nodes = BTreeMap::default();
         let pgdatadirspath = &env.pg_data_dirs_path();
-        let nodes: Result<BTreeMap<_, _>> = fs::read_dir(&pgdatadirspath)
+
+        for tenant_dir in fs::read_dir(&pgdatadirspath)
             .with_context(|| format!("failed to list {}", pgdatadirspath.display()))?
-            .into_iter()
-            .map(|f| {
-                PostgresNode::from_dir_entry(f?, &env, &pageserver)
-                    .map(|node| (node.name.clone(), Arc::new(node)))
-            })
-            .collect();
-        let nodes = nodes?;
+        {
+            let tenant_dir = tenant_dir?;
+            for timeline_dir in fs::read_dir(tenant_dir.path())
+                .with_context(|| format!("failed to list {}", tenant_dir.path().display()))?
+            {
+                let node = PostgresNode::from_dir_entry(timeline_dir?, &env, &pageserver)?;
+                nodes.insert((node.tenantid, node.name.clone()), Arc::new(node));
+            }
+        }
 
         Ok(ComputeControlPlane {
             base_port: 55431,
@@ -74,86 +82,40 @@ impl ComputeControlPlane {
         }
     }
 
-    /// Connect to a page server, get base backup, and untar it to initialize a
-    /// new data directory
-    pub fn new_from_page_server(
+    pub fn new_node(
         &mut self,
-        is_test: bool,
-        timelineid: ZTimelineId,
-        name: &str,
+        tenantid: ZTenantId,
+        branch_name: &str,
+        config_only: bool,
     ) -> Result<Arc<PostgresNode>> {
+        let timeline_id = self
+            .pageserver
+            .branch_get_by_name(&tenantid, branch_name)?
+            .timeline_id;
+
         let node = Arc::new(PostgresNode {
-            name: name.to_owned(),
+            name: branch_name.to_owned(),
             address: SocketAddr::new("127.0.0.1".parse().unwrap(), self.get_port()),
             env: self.env.clone(),
             pageserver: Arc::clone(&self.pageserver),
-            is_test,
-            timelineid,
+            is_test: false,
+            timelineid: timeline_id,
+            tenantid,
         });
 
-        node.init_from_page_server()?;
-        self.nodes.insert(node.name.clone(), Arc::clone(&node));
-
-        Ok(node)
-    }
-
-    pub fn new_test_node(&mut self, branch_name: &str) -> Arc<PostgresNode> {
-        let timeline_id = self
-            .pageserver
-            .branch_get_by_name(branch_name)
-            .expect("failed to get timeline_id")
-            .timeline_id;
-
-        let node = self.new_from_page_server(true, timeline_id, branch_name);
-        let node = node.unwrap();
+        node.init_from_page_server(self.env.auth_type, config_only)?;
+        self.nodes
+            .insert((tenantid, node.name.clone()), Arc::clone(&node));
 
         // Configure the node to stream WAL directly to the pageserver
         node.append_conf(
             "postgresql.conf",
             format!(
-                "shared_preload_libraries = zenith\n\
-                zenith.callmemaybe_connstring = '{}'\n", // FIXME escaping
-                node.connstr()
-            )
-            .as_str(),
-        )
-        .unwrap();
-
-        node
-    }
-
-    pub fn new_test_master_node(&mut self, branch_name: &str) -> Arc<PostgresNode> {
-        let timeline_id = self
-            .pageserver
-            .branch_get_by_name(branch_name)
-            .expect("failed to get timeline_id")
-            .timeline_id;
-
-        let node = self
-            .new_from_page_server(true, timeline_id, branch_name)
-            .unwrap();
-
-        node.append_conf(
-            "postgresql.conf",
-            "synchronous_standby_names = 'safekeeper_proxy'\n",
-        )
-        .unwrap();
-
-        node
-    }
-
-    pub fn new_node(&mut self, branch_name: &str) -> Result<Arc<PostgresNode>> {
-        let timeline_id = self.pageserver.branch_get_by_name(branch_name)?.timeline_id;
-
-        let node = self.new_from_page_server(false, timeline_id, branch_name)?;
-
-        // Configure the node to stream WAL directly to the pageserver
-        node.append_conf(
-            "postgresql.conf",
-            format!(
-                "shared_preload_libraries = zenith\n\
-                zenith.callmemaybe_connstring = '{}'\n", // FIXME escaping
-                node.connstr()
+                concat!(
+                    "synchronous_standby_names = 'pageserver'\n", // TODO: add a new function arg?
+                    "zenith.callmemaybe_connstring = '{}'\n",     // FIXME escaping
+                ),
+                node.connstr(),
             )
             .as_str(),
         )?;
@@ -164,6 +126,7 @@ impl ComputeControlPlane {
 
 ///////////////////////////////////////////////////////////////////////////////
 
+#[derive(Debug)]
 pub struct PostgresNode {
     pub address: SocketAddr,
     name: String,
@@ -171,6 +134,7 @@ pub struct PostgresNode {
     pageserver: Arc<PageServerNode>,
     is_test: bool,
     pub timelineid: ZTimelineId,
+    pub tenantid: ZTenantId,
 }
 
 impl PostgresNode {
@@ -190,6 +154,8 @@ impl PostgresNode {
             static ref CONF_PORT_RE: Regex = Regex::new(r"(?m)^\s*port\s*=\s*(\d+)\s*$").unwrap();
             static ref CONF_TIMELINE_RE: Regex =
                 Regex::new(r"(?m)^\s*zenith.zenith_timeline\s*=\s*'(\w+)'\s*$").unwrap();
+            static ref CONF_TENANT_RE: Regex =
+                Regex::new(r"(?m)^\s*zenith.zenith_tenant\s*=\s*'(\w+)'\s*$").unwrap();
         }
 
         // parse data directory name
@@ -237,6 +203,22 @@ impl PostgresNode {
             .parse()
             .with_context(|| err_msg)?;
 
+        // parse tenant
+        let err_msg = format!(
+            "failed to find tenant definition in config file {}",
+            cfg_path.to_str().unwrap()
+        );
+        let tenantid = CONF_TENANT_RE
+            .captures(config.as_str())
+            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 1"))?
+            .iter()
+            .last()
+            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 2"))?
+            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 3"))?
+            .as_str()
+            .parse()
+            .with_context(|| err_msg)?;
+
         // ok now
         Ok(PostgresNode {
             address: SocketAddr::new("127.0.0.1".parse().unwrap(), port),
@@ -245,12 +227,36 @@ impl PostgresNode {
             pageserver: Arc::clone(pageserver),
             is_test: false,
             timelineid,
+            tenantid,
         })
     }
 
-    // Connect to a page server, get base backup, and untar it to initialize a
-    // new data directory
-    pub fn init_from_page_server(&self) -> Result<()> {
+    /// Get basebackup from the pageserver as a tar archive and extract it
+    /// to the `self.pgdata()` directory.
+    pub fn do_basebackup(&self) -> Result<()> {
+        let pgdata = self.pgdata();
+
+        let sql = format!("basebackup {} {}", self.tenantid, self.timelineid);
+        let mut client = self
+            .pageserver
+            .page_server_psql_client()
+            .with_context(|| "connecting to page server failed")?;
+
+        let copyreader = client
+            .copy_out(sql.as_str())
+            .with_context(|| "page server 'basebackup' command failed")?;
+
+        // Read the archive directly from the `CopyOutReader`
+        tar::Archive::new(copyreader)
+            .unpack(&pgdata)
+            .with_context(|| "extracting page backup failed")?;
+
+        Ok(())
+    }
+
+    /// Connect to a pageserver, get basebackup, and untar it to initialize a
+    /// new data directory
+    pub fn init_from_page_server(&self, auth_type: AuthType, config_only: bool) -> Result<()> {
         let pgdata = self.pgdata();
 
         println!(
@@ -264,12 +270,6 @@ impl PostgresNode {
             fs::remove_dir_all(&pgdata).ok();
         }
 
-        let sql = format!("basebackup {}", self.timelineid);
-        let mut client = self
-            .pageserver
-            .page_server_psql_client()
-            .with_context(|| "connecting to page server failed")?;
-
         fs::create_dir_all(&pgdata)
             .with_context(|| format!("could not create data directory {}", pgdata.display()))?;
         fs::set_permissions(pgdata.as_path(), fs::Permissions::from_mode(0o700)).with_context(
@@ -281,31 +281,22 @@ impl PostgresNode {
             },
         )?;
 
-        // FIXME: The compute node should be able to stream the WAL it needs from the WAL safekeepers or archive.
-        // But that's not implemented yet. For now, 'pg_wal' is included in the base backup tarball that
-        // we receive from the Page Server, so we don't need to create the empty 'pg_wal' directory here.
-        //fs::create_dir_all(pgdata.join("pg_wal"))?;
+        if config_only {
+            //Just create an empty config file
+            File::create(self.pgdata().join("postgresql.conf").to_str().unwrap())?;
+        } else {
+            self.do_basebackup()?;
+            fs::create_dir_all(self.pgdata().join("pg_wal"))?;
+            fs::create_dir_all(self.pgdata().join("pg_wal").join("archive_status"))?;
+        }
 
-        let mut copyreader = client
-            .copy_out(sql.as_str())
-            .with_context(|| "page server 'basebackup' command failed")?;
-
-        // FIXME: Currently, we slurp the whole tarball into memory, and then extract it,
-        // but we really should do this:
-        //let mut ar = tar::Archive::new(copyreader);
-        let mut buf = vec![];
-        copyreader
-            .read_to_end(&mut buf)
-            .with_context(|| "reading base backup from page server failed")?;
-        let mut ar = tar::Archive::new(buf.as_slice());
-        ar.unpack(&pgdata)
-            .with_context(|| "extracting page backup failed")?;
-
-        // listen for selected port
+        // wal_log_hints is mandatory when running against pageserver (see gh issue#192)
+        // TODO: is it possible to check wal_log_hints at pageserver side via XLOG_PARAMETER_CHANGE?
         self.append_conf(
             "postgresql.conf",
             &format!(
                 "max_wal_senders = 10\n\
+                 wal_log_hints = on\n\
                  max_replication_slots = 10\n\
                  hot_standby = on\n\
                  shared_buffers = 1MB\n\
@@ -322,30 +313,40 @@ impl PostgresNode {
 
         // Never clean up old WAL. TODO: We should use a replication
         // slot or something proper, to prevent the compute node
-        // from removing WAL that hasn't been streamed to the safekeepr or
-        // page server yet. But this will do for now.
+        // from removing WAL that hasn't been streamed to the safekeeper or
+        // page server yet. (gh issue #349)
         self.append_conf("postgresql.conf", "wal_keep_size='10TB'\n")?;
 
-        // Connect it to the page server.
+        // set up authentication
+        let password = if let AuthType::ZenithJWT = auth_type {
+            "$ZENITH_AUTH_TOKEN"
+        } else {
+            ""
+        };
 
         // Configure that node to take pages from pageserver
+        let (host, port) = connection_host_port(&self.pageserver.pg_connection_config);
         self.append_conf(
             "postgresql.conf",
-            &format!(
-                "shared_preload_libraries = zenith \n\
-                 zenith.page_server_connstring = 'host={} port={}'\n\
-                 zenith.zenith_timeline='{}'\n",
-                self.pageserver.address().ip(),
-                self.pageserver.address().port(),
-                self.timelineid
-            ),
+            format!(
+                concat!(
+                    "shared_preload_libraries = zenith\n",
+                    // $ZENITH_AUTH_TOKEN will be replaced with value from environment variable during compute pg startup
+                    // it is done this way because otherwise user will be able to retrieve the value using SHOW command or pg_settings
+                    "zenith.page_server_connstring = 'host={} port={} password={}'\n",
+                    "zenith.zenith_timeline='{}'\n",
+                    "zenith.zenith_tenant='{}'\n",
+                ),
+                host, port, password, self.timelineid, self.tenantid,
+            )
+            .as_str(),
         )?;
 
         Ok(())
     }
 
     pub fn pgdata(&self) -> PathBuf {
-        self.env.pg_data_dir(&self.name)
+        self.env.pg_data_dir(&self.tenantid, &self.name)
     }
 
     pub fn status(&self) -> &str {
@@ -369,44 +370,88 @@ impl PostgresNode {
         Ok(())
     }
 
-    fn pg_ctl(&self, args: &[&str]) -> Result<()> {
+    fn pg_ctl(&self, args: &[&str], auth_token: &Option<String>) -> Result<()> {
         let pg_ctl_path = self.env.pg_bin_dir().join("pg_ctl");
+        let mut cmd = Command::new(pg_ctl_path);
+        cmd.args(
+            [
+                &[
+                    "-D",
+                    self.pgdata().to_str().unwrap(),
+                    "-l",
+                    self.pgdata().join("pg.log").to_str().unwrap(),
+                    "-w", //wait till pg_ctl actually does what was asked
+                ],
+                args,
+            ]
+            .concat(),
+        )
+        .env_clear()
+        .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
+        .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap());
+
+        if let Some(token) = auth_token {
+            cmd.env("ZENITH_AUTH_TOKEN", token);
+        }
+        let pg_ctl = cmd.status().with_context(|| "pg_ctl failed")?;
 
-        let pg_ctl = Command::new(pg_ctl_path)
-            .args(
-                [
-                    &[
-                        "-D",
-                        self.pgdata().to_str().unwrap(),
-                        "-l",
-                        self.pgdata().join("log").to_str().unwrap(),
-                    ],
-                    args,
-                ]
-                .concat(),
-            )
-            .env_clear()
-            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .status()
-            .with_context(|| "pg_ctl failed")?;
         if !pg_ctl.success() {
             anyhow::bail!("pg_ctl failed");
         }
         Ok(())
     }
 
-    pub fn start(&self) -> Result<()> {
+    pub fn start(&self, auth_token: &Option<String>) -> Result<()> {
+        // Bail if the node already running.
+        if self.status() == "running" {
+            anyhow::bail!("The node is already running");
+        }
+
+        // 1. We always start compute node from scratch, so
+        // if old dir exists, preserve config files and drop the directory
+
+        // XXX Now we only use 'postgresql.conf'.
+        // If we will need 'pg_hba.conf', support it here too
+
+        let postgresql_conf_path = self.pgdata().join("postgresql.conf");
+        let postgresql_conf = fs::read(postgresql_conf_path.clone()).with_context(|| {
+            format!(
+                "failed to read config file in {}",
+                postgresql_conf_path.to_str().unwrap()
+            )
+        })?;
+
+        println!(
+            "Destroying postgres data directory '{}'",
+            self.pgdata().to_str().unwrap()
+        );
+        fs::remove_dir_all(&self.pgdata())?;
+
+        // 2. Create new node
+        self.init_from_page_server(self.env.auth_type, false)?;
+
+        // 3. Bring back config files
+
+        if let Ok(mut file) = OpenOptions::new()
+            .append(false)
+            .write(true)
+            .open(&postgresql_conf_path)
+        {
+            file.write_all(&postgresql_conf)?;
+            file.sync_all()?;
+        }
+
+        // 4. Finally start the compute node postgres
         println!("Starting postgres node at '{}'", self.connstr());
-        self.pg_ctl(&["start"])
+        self.pg_ctl(&["start"], auth_token)
     }
 
-    pub fn restart(&self) -> Result<()> {
-        self.pg_ctl(&["restart"])
+    pub fn restart(&self, auth_token: &Option<String>) -> Result<()> {
+        self.pg_ctl(&["restart"], auth_token)
     }
 
     pub fn stop(&self, destroy: bool) -> Result<()> {
-        self.pg_ctl(&["-m", "immediate", "stop"])?;
+        self.pg_ctl(&["-m", "immediate", "stop"], &None)?;
         if destroy {
             println!(
                 "Destroying postgres data directory '{}'",
@@ -419,10 +464,11 @@ impl PostgresNode {
 
     pub fn connstr(&self) -> String {
         format!(
-            "host={} port={} user={}",
+            "host={} port={} user={} dbname={}",
             self.address.ip(),
             self.address.port(),
-            self.whoami()
+            "zenith_admin",
+            "postgres"
         )
     }
 

control_plane/src/lib.rs

@@ -1,7 +1,7 @@
 //
 // Local control plane.
 //
-// Can start, cofigure and stop postgres instances running as a local processes.
+// Can start, configure and stop postgres instances running as a local processes.
 //
 // Intended to be used in integration tests and in CLI tools for
 // local installations.

control_plane/src/local_env.rs

@@ -4,19 +4,24 @@
 // Now it also provides init method which acts like a stub for proper installation
 // script which will use local paths.
 //
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, Context, Result};
+use hex;
 use serde::{Deserialize, Serialize};
 use std::fs;
 use std::path::PathBuf;
+use std::process::{Command, Stdio};
 use std::{collections::BTreeMap, env};
 use url::Url;
+use zenith_utils::auth::{encode_from_key_path, Claims, Scope};
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::ZTenantId;
 
 pub type Remotes = BTreeMap<String, String>;
 
 //
 // This data structures represent deserialized zenith CLI config
 //
-#[derive(Serialize, Deserialize, Clone)]
+#[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct LocalEnv {
     // Pageserver connection strings
     pub pageserver_connstring: String,
@@ -33,6 +38,19 @@ pub struct LocalEnv {
     // Path to pageserver binary. Empty for remote pageserver.
     pub zenith_distrib_dir: Option<PathBuf>,
 
+    // keeping tenant id in config to reduce copy paste when running zenith locally with single tenant
+    #[serde(with = "hex")]
+    pub tenantid: ZTenantId,
+
+    // jwt auth token used for communication with pageserver
+    pub auth_token: String,
+
+    // used to determine which auth type is used
+    pub auth_type: AuthType,
+
+    // used to issue tokens during e.g pg start
+    pub private_key_path: PathBuf,
+
     pub remotes: Remotes,
 }
 
@@ -54,11 +72,13 @@ impl LocalEnv {
     }
 
     pub fn pg_data_dirs_path(&self) -> PathBuf {
-        self.base_data_dir.join("pgdatadirs")
+        self.base_data_dir.join("pgdatadirs").join("tenants")
     }
 
-    pub fn pg_data_dir(&self, name: &str) -> PathBuf {
-        self.pg_data_dirs_path().join(name)
+    pub fn pg_data_dir(&self, tenantid: &ZTenantId, branch_name: &str) -> PathBuf {
+        self.pg_data_dirs_path()
+            .join(tenantid.to_string())
+            .join(branch_name)
     }
 
     // TODO: move pageserver files into ./pageserver
@@ -77,7 +97,11 @@ fn base_path() -> PathBuf {
 //
 // Initialize a new Zenith repository
 //
-pub fn init(remote_pageserver: Option<&str>) -> Result<()> {
+pub fn init(
+    remote_pageserver: Option<&str>,
+    tenantid: ZTenantId,
+    auth_type: AuthType,
+) -> Result<()> {
     // check if config already exists
     let base_path = base_path();
     if base_path.exists() {
@@ -86,6 +110,7 @@ pub fn init(remote_pageserver: Option<&str>) -> Result<()> {
             base_path.to_str().unwrap()
         );
     }
+    fs::create_dir(&base_path)?;
 
     // ok, now check that expected binaries are present
 
@@ -102,8 +127,43 @@ pub fn init(remote_pageserver: Option<&str>) -> Result<()> {
         anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
     }
 
-    fs::create_dir(&base_path)?;
-    fs::create_dir(base_path.join("pgdatadirs"))?;
+    // generate keys for jwt
+    // openssl genrsa -out private_key.pem 2048
+    let private_key_path = base_path.join("auth_private_key.pem");
+    let keygen_output = Command::new("openssl")
+        .arg("genrsa")
+        .args(&["-out", private_key_path.to_str().unwrap()])
+        .arg("2048")
+        .stdout(Stdio::null())
+        .output()
+        .with_context(|| "failed to generate auth private key")?;
+    if !keygen_output.status.success() {
+        anyhow::bail!(
+            "openssl failed: '{}'",
+            String::from_utf8_lossy(&keygen_output.stderr)
+        );
+    }
+
+    let public_key_path = base_path.join("auth_public_key.pem");
+    // openssl rsa -in private_key.pem -pubout -outform PEM -out public_key.pem
+    let keygen_output = Command::new("openssl")
+        .arg("rsa")
+        .args(&["-in", private_key_path.to_str().unwrap()])
+        .arg("-pubout")
+        .args(&["-outform", "PEM"])
+        .args(&["-out", public_key_path.to_str().unwrap()])
+        .stdout(Stdio::null())
+        .output()
+        .with_context(|| "failed to generate auth private key")?;
+    if !keygen_output.status.success() {
+        anyhow::bail!(
+            "openssl failed: '{}'",
+            String::from_utf8_lossy(&keygen_output.stderr)
+        );
+    }
+
+    let auth_token =
+        encode_from_key_path(&Claims::new(None, Scope::PageServerApi), &private_key_path)?;
 
     let conf = if let Some(addr) = remote_pageserver {
         // check that addr is parsable
@@ -115,6 +175,10 @@ pub fn init(remote_pageserver: Option<&str>) -> Result<()> {
             zenith_distrib_dir: None,
             base_data_dir: base_path,
             remotes: BTreeMap::default(),
+            tenantid,
+            auth_token,
+            auth_type,
+            private_key_path,
         }
     } else {
         // Find zenith binaries.
@@ -129,9 +193,15 @@ pub fn init(remote_pageserver: Option<&str>) -> Result<()> {
             zenith_distrib_dir: Some(zenith_distrib_dir),
             base_data_dir: base_path,
             remotes: BTreeMap::default(),
+            tenantid,
+            auth_token,
+            auth_type,
+            private_key_path,
         }
     };
 
+    fs::create_dir_all(conf.pg_data_dirs_path())?;
+
     let toml = toml::to_string_pretty(&conf)?;
     fs::write(conf.base_data_dir.join("config"), toml)?;
 

control_plane/src/storage.rs

@@ -1,60 +1,84 @@
 use std::collections::HashMap;
-use std::net::{SocketAddr, TcpStream};
+use std::net::TcpStream;
 use std::path::PathBuf;
 use std::process::Command;
 use std::thread;
 use std::time::Duration;
 
-use anyhow::{anyhow, bail, Result};
+use anyhow::{anyhow, bail, ensure, Result};
 use nix::sys::signal::{kill, Signal};
 use nix::unistd::Pid;
-use postgres::{Client, NoTls};
+use pageserver::http::models::{BranchCreateRequest, TenantCreateRequest};
+use postgres::{Config, NoTls};
+use reqwest::blocking::{Client, RequestBuilder};
+use reqwest::{IntoUrl, Method, StatusCode};
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::ZTenantId;
 
 use crate::local_env::LocalEnv;
 use crate::read_pidfile;
 use pageserver::branches::BranchInfo;
+use zenith_utils::connstring::connection_address;
+
+const HTTP_BASE_URL: &str = "http://127.0.0.1:9898/v1";
 
 //
 // Control routines for pageserver.
 //
 // Used in CLI and tests.
 //
+#[derive(Debug)]
 pub struct PageServerNode {
     pub kill_on_exit: bool,
-    pub listen_address: Option<SocketAddr>,
+    pub pg_connection_config: Config,
     pub env: LocalEnv,
+    pub http_client: Client,
+    pub http_base_url: String,
 }
 
 impl PageServerNode {
     pub fn from_env(env: &LocalEnv) -> PageServerNode {
+        let password = if env.auth_type == AuthType::ZenithJWT {
+            &env.auth_token
+        } else {
+            ""
+        };
+
         PageServerNode {
             kill_on_exit: false,
-            listen_address: None, // default
+            pg_connection_config: Self::default_config(password), // default
             env: env.clone(),
+            http_client: Client::new(),
+            http_base_url: HTTP_BASE_URL.to_owned(),
         }
     }
 
-    pub fn address(&self) -> SocketAddr {
-        match self.listen_address {
-            Some(addr) => addr,
-            None => "127.0.0.1:64000".parse().unwrap(),
-        }
+    fn default_config(password: &str) -> Config {
+        format!("postgresql://no_user:{}@localhost:64000/no_db", password)
+            .parse()
+            .unwrap()
     }
 
-    pub fn init(&self) -> Result<()> {
+    pub fn init(&self, create_tenant: Option<&str>, enable_auth: bool) -> Result<()> {
         let mut cmd = Command::new(self.env.pageserver_bin()?);
+        let mut args = vec![
+            "--init",
+            "-D",
+            self.env.base_data_dir.to_str().unwrap(),
+            "--postgres-distrib",
+            self.env.pg_distrib_dir.to_str().unwrap(),
+        ];
+
+        if enable_auth {
+            args.extend(&["--auth-validation-public-key-path", "auth_public_key.pem"]);
+            args.extend(&["--auth-type", "ZenithJWT"]);
+        }
+
+        create_tenant.map(|tenantid| args.extend(&["--create-tenant", tenantid]));
         let status = cmd
-            .args(&["--init", "-D", self.env.base_data_dir.to_str().unwrap()])
+            .args(args)
             .env_clear()
             .env("RUST_BACKTRACE", "1")
-            .env(
-                "POSTGRES_DISTRIB_DIR",
-                self.env.pg_distrib_dir.to_str().unwrap(),
-            )
-            .env("ZENITH_REPO_DIR", self.repo_path())
-            .env("PATH", self.env.pg_bin_dir().to_str().unwrap()) // needs postres-wal-redo binary
-            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
             .status()
             .expect("pageserver init failed");
 
@@ -76,28 +100,15 @@ impl PageServerNode {
     pub fn start(&self) -> Result<()> {
         println!(
             "Starting pageserver at '{}' in {}",
-            self.address(),
+            connection_address(&self.pg_connection_config),
             self.repo_path().display()
         );
 
         let mut cmd = Command::new(self.env.pageserver_bin()?);
-        cmd.args(&[
-            "-l",
-            self.address().to_string().as_str(),
-            "-D",
-            self.repo_path().to_str().unwrap(),
-        ])
-        .arg("-d")
-        .env_clear()
-        .env("RUST_BACKTRACE", "1")
-        .env(
-            "POSTGRES_DISTRIB_DIR",
-            self.env.pg_distrib_dir.to_str().unwrap(),
-        )
-        .env("ZENITH_REPO_DIR", self.repo_path())
-        .env("PATH", self.env.pg_bin_dir().to_str().unwrap()) // needs postres-wal-redo binary
-        .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-        .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap());
+        cmd.args(&["-D", self.repo_path().to_str().unwrap()])
+            .arg("-d")
+            .env_clear()
+            .env("RUST_BACKTRACE", "1");
 
         if !cmd.status()?.success() {
             bail!(
@@ -109,18 +120,21 @@ impl PageServerNode {
         // It takes a while for the page server to start up. Wait until it is
         // open for business.
         for retries in 1..15 {
-            let client = self.page_server_psql_client();
-            if client.is_ok() {
-                break;
-            } else {
-                println!("Pageserver not responding yet, retrying ({})...", retries);
-                thread::sleep(Duration::from_secs(1));
+            match self.check_status() {
+                Ok(_) => {
+                    println!("Pageserver started");
+                    return Ok(());
+                }
+                Err(err) => {
+                    println!(
+                        "Pageserver not responding yet, err {} retrying ({})...",
+                        err, retries
+                    );
+                    thread::sleep(Duration::from_secs(1));
+                }
             }
         }
-
-        println!("Pageserver started");
-
-        Ok(())
+        bail!("pageserver failed to start");
     }
 
     pub fn stop(&self) -> Result<()> {
@@ -131,116 +145,118 @@ impl PageServerNode {
         }
 
         // wait for pageserver stop
+        let address = connection_address(&self.pg_connection_config);
         for _ in 0..5 {
-            let stream = TcpStream::connect(self.address());
+            let stream = TcpStream::connect(&address);
             thread::sleep(Duration::from_secs(1));
             if let Err(_e) = stream {
                 println!("Pageserver stopped");
                 return Ok(());
             }
-            println!("Stopping pageserver on {}", self.address());
+            println!("Stopping pageserver on {}", address);
         }
 
         bail!("Failed to stop pageserver with pid {}", pid);
     }
 
     pub fn page_server_psql(&self, sql: &str) -> Vec<postgres::SimpleQueryMessage> {
-        let connstring = format!(
-            "host={} port={} dbname={} user={}",
-            self.address().ip(),
-            self.address().port(),
-            "no_db",
-            "no_user",
-        );
-        let mut client = Client::connect(connstring.as_str(), NoTls).unwrap();
+        let mut client = self.pg_connection_config.connect(NoTls).unwrap();
 
         println!("Pageserver query: '{}'", sql);
         client.simple_query(sql).unwrap()
     }
 
     pub fn page_server_psql_client(&self) -> Result<postgres::Client, postgres::Error> {
-        let connstring = format!(
-            "host={} port={} dbname={} user={}",
-            self.address().ip(),
-            self.address().port(),
-            "no_db",
-            "no_user",
+        self.pg_connection_config.connect(NoTls)
+    }
+
+    fn http_request<U: IntoUrl>(&self, method: Method, url: U) -> RequestBuilder {
+        let mut builder = self.http_client.request(method, url);
+        if self.env.auth_type == AuthType::ZenithJWT {
+            builder = builder.bearer_auth(&self.env.auth_token)
+        }
+        builder
+    }
+
+    pub fn check_status(&self) -> Result<()> {
+        let status = self
+            .http_request(Method::GET, format!("{}/{}", self.http_base_url, "status"))
+            .send()?
+            .status();
+        ensure!(
+            status == StatusCode::OK,
+            format!("got unexpected response status {}", status)
         );
-        Client::connect(connstring.as_str(), NoTls)
+        Ok(())
     }
 
-    pub fn branches_list(&self) -> Result<Vec<BranchInfo>> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query("branch_list")?;
-        let branches_json = query_result
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
-            })
-            .flatten()
-            .ok_or_else(|| anyhow!("missing branches"))?;
-
-        let res: Vec<BranchInfo> = serde_json::from_str(branches_json)?;
-        Ok(res)
+    pub fn tenant_list(&self) -> Result<Vec<String>> {
+        Ok(self
+            .http_request(Method::GET, format!("{}/{}", self.http_base_url, "tenant"))
+            .send()?
+            .error_for_status()?
+            .json()?)
     }
 
-    pub fn branch_create(&self, name: &str, startpoint: &str) -> Result<BranchInfo> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result =
-            client.simple_query(format!("branch_create {} {}", name, startpoint).as_str())?;
-
-        let branch_json = query_result
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
+    pub fn tenant_create(&self, tenantid: ZTenantId) -> Result<()> {
+        Ok(self
+            .http_request(Method::POST, format!("{}/{}", self.http_base_url, "tenant"))
+            .json(&TenantCreateRequest {
+                tenant_id: tenantid,
             })
-            .flatten()
-            .ok_or_else(|| anyhow!("missing branch"))?;
+            .send()?
+            .error_for_status()?
+            .json()?)
+    }
 
-        let res: BranchInfo = serde_json::from_str(branch_json).map_err(|e| {
-            anyhow!(
-                "failed to parse branch_create response: {}: {}",
-                branch_json,
-                e
+    pub fn branch_list(&self, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
+        Ok(self
+            .http_request(
+                Method::GET,
+                format!("{}/branch/{}", self.http_base_url, tenantid),
             )
-        })?;
+            .send()?
+            .error_for_status()?
+            .json()?)
+    }
 
-        Ok(res)
+    pub fn branch_create(
+        &self,
+        branch_name: &str,
+        startpoint: &str,
+        tenantid: &ZTenantId,
+    ) -> Result<BranchInfo> {
+        Ok(self
+            .http_request(Method::POST, format!("{}/{}", self.http_base_url, "branch"))
+            .json(&BranchCreateRequest {
+                tenant_id: tenantid.to_owned(),
+                name: branch_name.to_owned(),
+                start_point: startpoint.to_owned(),
+            })
+            .send()?
+            .error_for_status()?
+            .json()?)
     }
 
     // TODO: make this a separate request type and avoid loading all the branches
-    pub fn branch_get_by_name(&self, name: &str) -> Result<BranchInfo> {
-        let branch_infos = self.branches_list()?;
-        let branche_by_name: Result<HashMap<String, BranchInfo>> = branch_infos
+    pub fn branch_get_by_name(
+        &self,
+        tenantid: &ZTenantId,
+        branch_name: &str,
+    ) -> Result<BranchInfo> {
+        let branch_infos = self.branch_list(tenantid)?;
+        let branch_by_name: Result<HashMap<String, BranchInfo>> = branch_infos
             .into_iter()
             .map(|branch_info| Ok((branch_info.name.clone(), branch_info)))
             .collect();
-        let branche_by_name = branche_by_name?;
+        let branch_by_name = branch_by_name?;
 
-        let branch = branche_by_name
-            .get(name)
-            .ok_or_else(|| anyhow!("Branch {} not found", name))?;
+        let branch = branch_by_name
+            .get(branch_name)
+            .ok_or_else(|| anyhow!("Branch {} not found", branch_name))?;
 
         Ok(branch.clone())
     }
-
-    pub fn system_id_get(&self) -> Result<u64> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result = client
-            .simple_query("identify_system")?
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
-            })
-            .flatten()
-            .ok_or_else(|| anyhow!("failed to get system_id"))?
-            .parse::<u64>()?;
-
-        Ok(query_result)
-    }
 }
 
 impl Drop for PageServerNode {

docker-entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eux
+
+if [ "$1" = 'pageserver' ]; then
+    if [ ! -d "/data/tenants" ]; then
+        echo "Initializing pageserver data directory"
+        pageserver --init -D /data --postgres-distrib /usr/local
+    fi
+    echo "Staring pageserver at 0.0.0.0:6400"
+    pageserver -l 0.0.0.0:6400 -D /data
+else
+    "$@"
+fi

docs/README.md

@@ -0,0 +1,13 @@
+# Zenith documentation
+
+## Table of contents
+
+- [authentication.md](authentication.md) — pageserver JWT authentication.
+- [docker.md](docker.md) — Docker images and building pipeline.
+- [glossary.md](glossary.md) — Glossary of all the terms used in codebase.
+- [multitenancy.md](multitenancy.md) — how multitenancy is organized in the pageserver and Zenith CLI.
+- [sourcetree.md](sourcetree.md) — Overview of the source tree layeout.
+- [pageserver/README](/pageserver/README) — pageserver overview.
+- [postgres_ffi/README](/postgres_ffi/README) — Postgres FFI overview.
+- [test_runner/README.md](/test_runner/README.md) — tests infrastructure overview.
+- [walkeeper/README](/walkeeper/README.md) — WAL service overview.

docs/authentication.md

@@ -0,0 +1,30 @@
+## Authentication
+
+### Overview
+
+Current state of authentication includes usage of JWT tokens in communication between compute and pageserver and between CLI and pageserver. JWT token is signed using RSA keys. CLI generates a key pair during call to `zenith init`. Using following openssl commands:
+
+```bash
+openssl genrsa -out private_key.pem 2048
+openssl rsa -in private_key.pem -pubout -outform PEM -out public_key.pem
+```
+
+CLI also generates signed token and saves it in the config for later access to pageserver. Now authentication is optional. Pageserver has two variables in config: `auth_validation_public_key_path` and `auth_type`, so when auth type present and set to `ZenithJWT` pageserver will require authentication for connections. Actual JWT is passed in password field of connection string. There is a caveat for psql, it silently truncates passwords to 100 symbols, so to correctly pass JWT via psql you have to either use PGPASSWORD environment variable, or store password in psql config file.
+
+Currently there is no authentication between compute and safekeepers, because this communication layer is under heavy refactoring. After this refactoring support for authentication will be added there too. Now safekeeper supports "hardcoded" token passed via environment variable to be able to use callmemaybe command in pageserver.
+
+Compute uses token passed via environment variable to communicate to pageserver and in the future to the safekeeper too.
+
+JWT authentication now supports two scopes: tenant and pageserverapi. Tenant scope is intended for use in tenant related api calls, e.g. create_branch. Compute launched for particular tenant also uses this scope. Scope pageserver api is intended to be used by console to manage pageserver. For now we have only one management operation - create tenant.
+
+Examples for token generation in python:
+
+```python
+# generate pageserverapi token
+management_token = jwt.encode({"scope": "pageserverapi"}, auth_keys.priv, algorithm="RS256")
+
+# generate tenant token
+tenant_token = jwt.encode({"scope": "tenant", "tenant_id": ps.initial_tenant}, auth_keys.priv, algorithm="RS256")
+```
+
+Utility functions to work with jwts in rust are located in zenith_utils/src/auth.rs

docs/docker.md

@@ -0,0 +1,38 @@
+# Docker images of Zenith
+
+## Images
+
+Currently we build two main images:
+
+- [zenithdb/zenith](https://hub.docker.com/repository/docker/zenithdb/zenith) — image with pre-built `pageserver`, `wal_acceptor` and `proxy` binaries and all the required runtime dependencies. Built from [/Dockerfile](/Dockerfile).
+- [zenithdb/compute-node](https://hub.docker.com/repository/docker/zenithdb/compute-node) — compute node image with pre-built Postgres binaries from [zenithdb/postgres](https://github.com/zenithdb/postgres).
+
+And two intermediate images used either to reduce build time or to deliver some additional binary tools from other repos:
+
+- [zenithdb/build](https://hub.docker.com/repository/docker/zenithdb/build) — image with all the dependencies required to build Zenith and compute node images. This image is based on `rust:slim-buster`, so it also has a proper `rust` environment. Built from [/Dockerfile.build](/Dockerfile.build).
+- [zenithdb/compute-tools](https://hub.docker.com/repository/docker/zenithdb/compute-tools) — compute node configuration management tools.
+
+## Building pipeline
+
+1. Image `zenithdb/compute-tools` is re-built automatically.
+
+2. Image `zenithdb/build` is built manually. If you want to introduce any new compile time dependencies to Zenith or compute node you have to update this image as well, build it and push to Docker Hub.
+
+Build:
+```sh
+docker build -t zenithdb/build:buster -f Dockerfile.build .
+```
+
+Login:
+```sh
+docker login
+```
+
+Push to Docker Hub:
+```sh
+docker push zenithdb/build:buster
+```
+
+3. Image `zenithdb/compute-node` is built independently in the [zenithdb/postgres](https://github.com/zenithdb/postgres) repo.
+
+4. Image `zenithdb/zenith` is built in this repo after a successful `release` tests run and pushed to Docker Hub automatically.

docs/glossary.md

@@ -0,0 +1,196 @@
+# Glossary
+
+### Authentication
+
+### Base image (page image)
+
+### Basebackup
+
+A tarball with files needed to bootstrap a compute node[] and a corresponding command to create it.
+NOTE:It has nothing to do with PostgreSQL pg_basebackup.
+
+### Branch
+
+We can create branch at certain LSN using `zenith branch` command.
+Each Branch lives in a corresponding timeline[] and has an ancestor[].
+
+
+### Checkpoint (PostgreSQL)
+
+NOTE: This is an overloaded term.
+
+A checkpoint record in the WAL marks a point in the WAL sequence at which it is guaranteed that all data files have been updated with all information from shared memory modified before that checkpoint; 
+
+### Checkpoint (Layered repository)
+
+NOTE: This is an overloaded term.
+
+Whenever enough WAL has been accumulated in memory, the page server []
+writes out the changes from in-memory layers into new layer files[]. This process
+is called "checkpointing". The page server only creates layer files for
+relations that have been modified since the last checkpoint. 
+
+### Compute node
+
+Stateless Postgres node that stores data in pageserver.
+
+### Garbage collection
+
+### Fork
+
+Each of the separate segmented file sets in which a relation is stored. The main fork is where the actual data resides. There also exist two secondary forks for metadata: the free space map and the visibility map.
+Each PostgreSQL fork is considered a separate relish.
+
+### Layer
+
+Each layer corresponds to one RELISH_SEG_SIZE slice of a relish in a range of LSNs.
+There are two kinds of layers, in-memory and on-disk layers. In-memory
+layers are used to ingest incoming WAL, and provide fast access
+to the recent page versions. On-disk layers are stored as files on disk, and
+are immutable.
+### Layer file (on-disk layer)
+
+Layered repository on-disk format is based on immutable files.  The
+files are called "layer files". Each file corresponds to one RELISH_SEG_SIZE
+segment of a PostgreSQL relation fork. There are two kinds of layer
+files: image files and delta files. An image file contains a
+"snapshot" of the segment at a particular LSN, and a delta file
+contains WAL records applicable to the segment, in a range of LSNs.
+
+### Layer map
+
+The layer map tracks what layers exist for all the relishes in a timeline.
+### Layered repository
+
+Zenith repository implementation that keeps data in layers.
+### LSN
+
+
+### Page (block)
+
+The basic structure used to store relation data. All pages are of the same size.
+This is the unit of data exchange between compute node and pageserver.
+
+### Pageserver
+
+Zenith storage engine: repositories + wal receiver + page service + wal redo.
+
+### Page service
+
+The Page Service listens for GetPage@LSN requests from the Compute Nodes,
+and responds with pages from the repository.
+
+
+### PITR (Point-in-time-recovery)
+
+PostgreSQL's ability to restore up to a specified LSN.
+
+### Primary node
+
+
+### Proxy
+
+Postgres protocol proxy/router.
+This service listens psql port, can check auth via external service
+and create new databases and accounts (control plane API in our case).
+
+### Relation
+
+The generic term in PostgreSQL for all objects in a database that have a name and a list of attributes defined in a specific order.
+
+### Relish
+
+We call each relation and other file that is stored in the
+repository a "relish". It comes from "rel"-ish, as in "kind of a
+rel", because it covers relations as well as other things that are
+not relations, but are treated similarly for the purposes of the
+storage layer.
+
+### Replication slot
+
+
+### Replica node
+
+
+### Repository
+
+Repository stores multiple timelines, forked off from the same initial call to 'initdb'
+and has associated WAL redo service.
+One repository corresponds to one Tenant.
+
+### Retention policy
+
+How much history do we need to keep around for PITR and read-only nodes?
+
+### SLRU
+
+SLRUs include pg_clog, pg_multixact/members, and
+pg_multixact/offsets. There are other SLRUs in PostgreSQL, but
+they don't need to be stored permanently (e.g. pg_subtrans),
+or we do not support them in zenith yet (pg_commit_ts).
+Each SLRU segment is considered a separate relish[].
+
+### Tenant (Multitenancy)
+Tenant represents a single customer, interacting with Zenith.
+Wal redo[] activity, timelines[], layers[] are managed for each tenant independently.
+One pageserver[] can serve multiple tenants at once.
+One safekeeper 
+
+See `docs/multitenancy.md` for more.
+
+### Timeline
+
+Timeline accepts page changes and serves get_page_at_lsn() and
+get_rel_size() requests. The term "timeline" is used internally
+in the system, but to users they are exposed as "branches", with
+human-friendly names.
+
+NOTE: this has nothing to do with PostgreSQL WAL timelines.
+
+### XLOG
+
+PostgreSQL alias for WAL[].
+
+### WAL (Write-ahead log)
+
+The journal that keeps track of the changes in the database cluster as user- and system-invoked operations take place. It comprises many individual WAL records[] written sequentially to WAL files[].
+
+### WAL acceptor, WAL proposer
+
+In the context of the consensus algorithm, the Postgres
+compute node is also known as the WAL proposer, and the safekeeper is also known
+as the acceptor. Those are the standard terms in the Paxos algorithm.
+
+### WAL receiver (WAL decoder)
+
+The WAL receiver connects to the external WAL safekeeping service (or
+directly to the primary) using PostgreSQL physical streaming
+replication, and continuously receives WAL. It decodes the WAL records,
+and stores them to the repository.
+
+We keep one WAL receiver active per timeline.
+
+### WAL record
+
+A low-level description of an individual data change.
+
+### WAL redo
+
+A service that runs PostgreSQL in a special wal_redo mode
+to apply given WAL records over an old page image and return new page image.
+
+### WAL safekeeper
+
+One node that participates in the quorum. All the safekeepers
+together form the WAL service.
+
+### WAL segment (WAL file)
+
+Also known as WAL segment or WAL segment file. Each of the sequentially-numbered files that provide storage space for WAL. The files are all of the same predefined size and are written in sequential order, interspersing changes as they occur in multiple simultaneous sessions.
+
+### WAL service
+
+The service as whole that ensures that WAL is stored durably.
+
+### Web console
+

docs/multitenancy.md

@@ -0,0 +1,59 @@
+## Multitenancy
+
+### Overview
+
+Zenith supports multitenancy. One pageserver can serve multiple tenants at once. Tenants can be managed via zenith CLI. During page server setup tenant can be created using ```zenith init --create-tenant``` Also tenants can be added into the system on the fly without pageserver restart. This can be done using the following cli command: ```zenith tenant create``` Tenants use random identifiers which can be represented as a 32 symbols hexadecimal string. So zenith tenant create accepts desired tenant id as an optional argument. The concept of timelines/branches is working independently per tenant.
+
+### Tenants in other commands
+
+By default during `zenith init` new tenant is created on the pageserver. Newly created tenant's id is saved to cli config, so other commands can use it automatically if no direct arugment `--tenantid=<tenantid>` is provided. So generally tenantid more frequently appears in internal pageserver interface. Its commands take tenantid argument to distinguish to which tenant operation should be applied. CLI support creation of new tenants.
+
+Examples for cli:
+
+```sh
+zenith tenant list
+
+zenith tenant create // generates new id
+
+zenith tenant create ee6016ec31116c1b7c33dfdfca38892f
+
+zenith pg create main // default tenant from zenith init
+
+zenith pg create main --tenantid=ee6016ec31116c1b7c33dfdfca38892f
+
+zenith branch --tenantid=ee6016ec31116c1b7c33dfdfca38892f
+```
+
+### Data layout
+
+On the page server tenants introduce one level of indirection, so data directory structured the following way:
+```
+<pageserver working directory>
+├── pageserver.log
+├── pageserver.pid
+├── pageserver.toml
+└── tenants
+   ├── 537cffa58a4fa557e49e19951b5a9d6b
+   ├── de182bc61fb11a5a6b390a8aed3a804a
+   └── ee6016ec31116c1b7c33dfdfca38891f
+```
+Wal redo activity and timelines are managed for each tenant independently.
+
+For local environment used for example in tests there also new level of indirection for tenants. It touches `pgdatadirs` directory. Now it contains `tenants` subdirectory so the structure looks the following way:
+
+```
+pgdatadirs
+└── tenants
+   ├── de182bc61fb11a5a6b390a8aed3a804a
+   │  └── main
+   └── ee6016ec31116c1b7c33dfdfca38892f
+      └── main
+```
+
+### Changes to postgres
+
+Tenant id is passed to postgres via GUC the same way as the timeline. Tenant id is added to commands issued to pageserver, namely: pagestream, callmemaybe. Tenant id is also exists in ServerInfo structure, this is needed to pass the value to wal receiver to be able to forward it to the pageserver.
+
+### Safety
+
+For now particular tenant can only appear on a particular pageserver. Set of WAL acceptors are also pinned to particular (tenantid, timeline) pair so there can only be one writer for particular (tenantid, timeline).

docs/sourcetree.md

@@ -0,0 +1,81 @@
+## Source tree layout
+
+Below you will find a brief overview of each subdir in the source tree in alphabetical order.
+
+`/control_plane`:
+
+Local control plane.
+Functions to start, configure and stop pageserver and postgres instances running as a local processes.
+Intended to be used in integration tests and in CLI tools for local installations.
+
+`/docs`:
+
+Documentaion of the Zenith features and concepts.
+Now it is mostly dev documentation.
+
+`/monitoring`:
+
+TODO
+
+`/pageserver`:
+
+Zenith storage service.
+The pageserver has a few different duties:
+
+- Store and manage the data.
+- Generate a tarball with files needed to bootstrap ComputeNode.
+- Respond to GetPage@LSN requests from the Compute Nodes.
+- Receive WAL from the WAL service and decode it.
+- Replay WAL that's applicable to the chunks that the Page Server maintains
+
+For more detailed info, see `/pageserver/README`
+
+`/postgres_ffi`:
+
+Utility functions for interacting with PostgreSQL file formats.
+Misc constants, copied from PostgreSQL headers.
+
+`/proxy`:
+
+Postgres protocol proxy/router.
+This service listens psql port, can check auth via external service
+and create new databases and accounts (control plane API in our case).
+
+`/test_runner`:
+
+Integration tests, written in Python using the `pytest` framework.
+
+`/vendor/postgres`:
+
+PostgreSQL source tree, with the modifications needed for Zenith.
+
+`/vendor/postgres/contrib/zenith`:
+
+PostgreSQL extension that implements storage manager API and network communications with remote page server.
+
+`/vendor/postgres/contrib/zenith_test_utils`:
+
+PostgreSQL extension that contains functions needed for testing and debugging.
+
+`/walkeeper`:
+
+The zenith WAL service that receives WAL from a primary compute nodes and streams it to the pageserver.
+It acts as a holding area and redistribution center for recently generated WAL.
+
+For more detailed info, see `/walkeeper/README`
+
+`/workspace_hack`:
+The workspace_hack crate exists only to pin down some dependencies.
+
+`/zenith`
+
+Main entry point for the 'zenith' CLI utility.
+TODO: Doesn't it belong to control_plane?
+
+`/zenith_metrics`:
+
+Helpers for exposing Prometheus metrics from the server.
+
+`/zenith_utils`:
+
+Helpers that are shared between other crates in this repository.

integration_tests/.gitignore

@@ -1 +0,0 @@
-tmp_check/

integration_tests/Cargo.toml

@@ -1,18 +0,0 @@
-[package]
-name = "integration_tests"
-version = "0.1.0"
-authors = ["Stas Kelvich <stas@zenith.tech>"]
-edition = "2018"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-lazy_static = "1.4.0"
-rand = "0.8.3"
-anyhow = "1.0"
-nix = "0.20"
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-
-pageserver = { path = "../pageserver" }
-walkeeper = { path = "../walkeeper" }
-control_plane = { path = "../control_plane" }

integration_tests/src/lib.rs

@@ -1,414 +0,0 @@
-use anyhow::{bail, Result};
-use nix::sys::signal::{kill, Signal};
-use nix::unistd::Pid;
-use std::collections::BTreeMap;
-use std::convert::TryInto;
-use std::fs::{self, File, OpenOptions};
-use std::io::Read;
-use std::net::SocketAddr;
-use std::path::{Path, PathBuf};
-use std::process::{Command, ExitStatus};
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
-
-use control_plane::compute::PostgresNode;
-use control_plane::read_pidfile;
-use control_plane::{local_env::LocalEnv, storage::PageServerNode};
-
-// Find the directory where the binaries were put (i.e. target/debug/)
-fn cargo_bin_dir() -> PathBuf {
-    let mut pathbuf = std::env::current_exe().unwrap();
-
-    pathbuf.pop();
-    if pathbuf.ends_with("deps") {
-        pathbuf.pop();
-    }
-
-    pathbuf
-}
-
-// local compute env for tests
-pub fn create_test_env(testname: &str) -> LocalEnv {
-    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
-        .join("../tmp_check/")
-        .join(testname);
-
-    let base_path_str = base_path.to_str().unwrap();
-
-    // Remove remnants of old test repo
-    let _ = fs::remove_dir_all(&base_path);
-
-    fs::create_dir_all(&base_path)
-        .unwrap_or_else(|_| panic!("could not create directory for {}", base_path_str));
-
-    let pgdatadirs_path = base_path.join("pgdatadirs");
-    fs::create_dir(&pgdatadirs_path)
-        .unwrap_or_else(|_| panic!("could not create directory {:?}", pgdatadirs_path));
-
-    LocalEnv {
-        pageserver_connstring: "postgresql://127.0.0.1:64000".to_string(),
-        pg_distrib_dir: Path::new(env!("CARGO_MANIFEST_DIR")).join("../tmp_install"),
-        zenith_distrib_dir: Some(cargo_bin_dir()),
-        base_data_dir: base_path,
-        remotes: BTreeMap::default(),
-    }
-}
-
-//
-// Collection of several example deployments useful for tests.
-//
-// I'm intendedly modelling storage and compute control planes as a separate entities
-// as it is closer to the actual setup.
-//
-pub struct TestStorageControlPlane {
-    pub wal_acceptors: Vec<WalAcceptorNode>,
-    pub pageserver: Arc<PageServerNode>,
-    pub test_done: AtomicBool,
-}
-
-impl TestStorageControlPlane {
-    // postgres <-> page_server
-    //
-    // Initialize a new repository and configure a page server to run in it
-    //
-    pub fn one_page_server(local_env: &LocalEnv) -> TestStorageControlPlane {
-        let pserver = Arc::new(PageServerNode {
-            env: local_env.clone(),
-            kill_on_exit: true,
-            listen_address: None,
-        });
-        pserver.init().unwrap();
-        pserver.start().unwrap();
-
-        TestStorageControlPlane {
-            wal_acceptors: Vec::new(),
-            pageserver: pserver,
-            test_done: AtomicBool::new(false),
-        }
-    }
-
-    // postgres <-> {wal_acceptor1, wal_acceptor2, ...}
-    pub fn fault_tolerant(local_env: &LocalEnv, redundancy: usize) -> TestStorageControlPlane {
-        let mut cplane = TestStorageControlPlane {
-            wal_acceptors: Vec::new(),
-            pageserver: Arc::new(PageServerNode {
-                env: local_env.clone(),
-                kill_on_exit: true,
-                listen_address: None,
-            }),
-            test_done: AtomicBool::new(false),
-            // repopath,
-        };
-        cplane.pageserver.init().unwrap();
-        cplane.pageserver.start().unwrap();
-
-        let systemid = cplane.pageserver.system_id_get().unwrap();
-
-        const WAL_ACCEPTOR_PORT: usize = 54321;
-
-        let datadir_base = local_env.base_data_dir.join("safekeepers");
-        fs::create_dir_all(&datadir_base).unwrap();
-
-        for i in 0..redundancy {
-            let wal_acceptor = WalAcceptorNode {
-                listen: format!("127.0.0.1:{}", WAL_ACCEPTOR_PORT + i)
-                    .parse()
-                    .unwrap(),
-                data_dir: datadir_base.join(format!("wal_acceptor_{}", i)),
-                systemid,
-                env: local_env.clone(),
-                pass_to_pageserver: true,
-            };
-            wal_acceptor.init();
-            wal_acceptor.start();
-            cplane.wal_acceptors.push(wal_acceptor);
-        }
-        cplane
-    }
-
-    pub fn stop(&self) {
-        for wa in self.wal_acceptors.iter() {
-            let _ = wa.stop();
-        }
-        self.test_done.store(true, Ordering::Relaxed);
-    }
-
-    pub fn get_wal_acceptor_conn_info(&self) -> String {
-        self.wal_acceptors
-            .iter()
-            .map(|wa| wa.listen.to_string())
-            .collect::<Vec<String>>()
-            .join(",")
-    }
-
-    pub fn is_running(&self) -> bool {
-        self.test_done.load(Ordering::Relaxed)
-    }
-}
-
-impl Drop for TestStorageControlPlane {
-    fn drop(&mut self) {
-        self.stop();
-    }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-//
-// PostgresNodeExt
-//
-///////////////////////////////////////////////////////////////////////////////
-
-///
-/// Testing utilities for PostgresNode type
-///
-pub trait PostgresNodeExt {
-    fn pg_regress(&self) -> ExitStatus;
-    fn pg_bench(&self, clients: u32, seconds: u32) -> ExitStatus;
-    fn start_proxy(&self, wal_acceptors: &str) -> WalProposerNode;
-    fn open_psql(&self, db: &str) -> postgres::Client;
-    fn dump_log_file(&self);
-    fn safe_psql(&self, db: &str, sql: &str) -> Vec<postgres::Row>;
-}
-
-impl PostgresNodeExt for PostgresNode {
-    fn pg_regress(&self) -> ExitStatus {
-        self.safe_psql("postgres", "CREATE DATABASE regression");
-
-        let regress_run_path = self.env.base_data_dir.join("regress");
-        fs::create_dir_all(&regress_run_path).unwrap();
-        fs::create_dir_all(regress_run_path.join("testtablespace")).unwrap();
-        std::env::set_current_dir(regress_run_path).unwrap();
-
-        let regress_build_path =
-            Path::new(env!("CARGO_MANIFEST_DIR")).join("../tmp_install/build/src/test/regress");
-        let regress_src_path =
-            Path::new(env!("CARGO_MANIFEST_DIR")).join("../vendor/postgres/src/test/regress");
-
-        let regress_check = Command::new(regress_build_path.join("pg_regress"))
-            .args(&[
-                "--bindir=''",
-                "--use-existing",
-                format!("--bindir={}", self.env.pg_bin_dir().to_str().unwrap()).as_str(),
-                format!("--dlpath={}", regress_build_path.to_str().unwrap()).as_str(),
-                format!(
-                    "--schedule={}",
-                    regress_src_path.join("parallel_schedule").to_str().unwrap()
-                )
-                .as_str(),
-                format!("--inputdir={}", regress_src_path.to_str().unwrap()).as_str(),
-            ])
-            .env_clear()
-            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .env("PGPORT", self.address.port().to_string())
-            .env("PGUSER", self.whoami())
-            .env("PGHOST", self.address.ip().to_string())
-            .status()
-            .expect("pg_regress failed");
-        if !regress_check.success() {
-            if let Ok(mut file) = File::open("regression.diffs") {
-                let mut buffer = String::new();
-                file.read_to_string(&mut buffer).unwrap();
-                println!("--------------- regression.diffs:\n{}", buffer);
-            }
-            self.dump_log_file();
-        }
-        regress_check
-    }
-
-    fn pg_bench(&self, clients: u32, seconds: u32) -> ExitStatus {
-        let port = self.address.port().to_string();
-        let clients = clients.to_string();
-        let seconds = seconds.to_string();
-        let _pg_bench_init = Command::new(self.env.pg_bin_dir().join("pgbench"))
-            .args(&["-i", "-p", port.as_str(), "postgres"])
-            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .status()
-            .expect("pgbench -i");
-        let pg_bench_run = Command::new(self.env.pg_bin_dir().join("pgbench"))
-            .args(&[
-                "-p",
-                port.as_str(),
-                "-T",
-                seconds.as_str(),
-                "-P",
-                "1",
-                "-c",
-                clients.as_str(),
-                "-M",
-                "prepared",
-                "postgres",
-            ])
-            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .status()
-            .expect("pgbench run");
-        pg_bench_run
-    }
-
-    fn start_proxy(&self, wal_acceptors: &str) -> WalProposerNode {
-        let proxy_path = self.env.pg_bin_dir().join("safekeeper_proxy");
-        match Command::new(proxy_path.as_path())
-            .args(&["--ztimelineid", &self.timelineid.to_string()])
-            .args(&["-s", wal_acceptors])
-            .args(&["-h", &self.address.ip().to_string()])
-            .args(&["-p", &self.address.port().to_string()])
-            .arg("-v")
-            .stderr(
-                OpenOptions::new()
-                    .create(true)
-                    .append(true)
-                    .open(self.pgdata().join("safekeeper_proxy.log"))
-                    .unwrap(),
-            )
-            .spawn()
-        {
-            Ok(child) => WalProposerNode { pid: child.id() },
-            Err(e) => panic!("Failed to launch {:?}: {}", proxy_path, e),
-        }
-    }
-
-    fn dump_log_file(&self) {
-        if let Ok(mut file) = File::open(self.env.pageserver_data_dir().join("pageserver.log")) {
-            let mut buffer = String::new();
-            file.read_to_string(&mut buffer).unwrap();
-            println!("--------------- pageserver.log:\n{}", buffer);
-        }
-    }
-
-    fn safe_psql(&self, db: &str, sql: &str) -> Vec<postgres::Row> {
-        let connstring = format!(
-            "host={} port={} dbname={} user={}",
-            self.address.ip(),
-            self.address.port(),
-            db,
-            self.whoami()
-        );
-        let mut client = postgres::Client::connect(connstring.as_str(), postgres::NoTls).unwrap();
-
-        println!("Running {}", sql);
-        let result = client.query(sql, &[]);
-        if result.is_err() {
-            self.dump_log_file();
-        }
-        result.unwrap()
-    }
-
-    fn open_psql(&self, db: &str) -> postgres::Client {
-        let connstring = format!(
-            "host={} port={} dbname={} user={}",
-            self.address.ip(),
-            self.address.port(),
-            db,
-            self.whoami()
-        );
-        postgres::Client::connect(connstring.as_str(), postgres::NoTls).unwrap()
-    }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-//
-// WalAcceptorNode
-//
-///////////////////////////////////////////////////////////////////////////////
-
-//
-// Control routines for WalAcceptor.
-//
-// Now used only in test setups.
-//
-pub struct WalAcceptorNode {
-    listen: SocketAddr,
-    data_dir: PathBuf,
-    systemid: u64,
-    env: LocalEnv,
-    pass_to_pageserver: bool,
-}
-
-impl WalAcceptorNode {
-    pub fn init(&self) {
-        if self.data_dir.exists() {
-            fs::remove_dir_all(self.data_dir.clone()).unwrap();
-        }
-        fs::create_dir_all(self.data_dir.clone()).unwrap();
-    }
-
-    pub fn start(&self) {
-        println!(
-            "Starting wal_acceptor in {} listening '{}'",
-            self.data_dir.to_str().unwrap(),
-            self.listen
-        );
-
-        let ps_arg = if self.pass_to_pageserver {
-            // Tell page server it can receive WAL from this WAL safekeeper
-            ["--pageserver", "127.0.0.1:64000"].to_vec()
-        } else {
-            [].to_vec()
-        };
-
-        let status = Command::new(
-            self.env
-                .zenith_distrib_dir
-                .as_ref()
-                .unwrap()
-                .join("wal_acceptor"),
-        )
-        .args(&["-D", self.data_dir.to_str().unwrap()])
-        .args(&["-l", self.listen.to_string().as_str()])
-        .args(&["--systemid", self.systemid.to_string().as_str()])
-        .args(&ps_arg)
-        .arg("-d")
-        .arg("-n")
-        .status()
-        .expect("failed to start wal_acceptor");
-
-        if !status.success() {
-            panic!("wal_acceptor start failed");
-        }
-    }
-
-    pub fn stop(&self) -> Result<()> {
-        println!("Stopping wal acceptor on {}", self.listen);
-        let pidfile = self.data_dir.join("wal_acceptor.pid");
-        let pid = read_pidfile(&pidfile)?;
-        let pid = Pid::from_raw(pid);
-        if kill(pid, Signal::SIGTERM).is_err() {
-            bail!("Failed to kill wal_acceptor with pid {}", pid);
-        }
-        Ok(())
-    }
-}
-
-impl Drop for WalAcceptorNode {
-    fn drop(&mut self) {
-        // Ignore errors.
-        let _ = self.stop();
-    }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-//
-// WalProposerNode
-//
-///////////////////////////////////////////////////////////////////////////////
-
-pub struct WalProposerNode {
-    pub pid: u32,
-}
-
-impl WalProposerNode {
-    pub fn stop(&self) {
-        // std::process::Child::id() returns u32, we need i32.
-        let pid: i32 = self.pid.try_into().unwrap();
-        let pid = Pid::from_raw(pid);
-        kill(pid, Signal::SIGTERM).expect("failed to execute kill");
-    }
-}
-
-impl Drop for WalProposerNode {
-    fn drop(&mut self) {
-        self.stop();
-    }
-}

integration_tests/tests/test_wal_acceptor.rs

@@ -1,330 +0,0 @@
-use rand::Rng;
-use std::sync::Arc;
-use std::time::SystemTime;
-use std::{thread, time};
-
-use control_plane::compute::{ComputeControlPlane, PostgresNode};
-
-use integration_tests::PostgresNodeExt;
-use integration_tests::TestStorageControlPlane;
-
-const DOWNTIME: u64 = 2;
-
-fn start_node_with_wal_proposer(
-    timeline: &str,
-    compute_cplane: &mut ComputeControlPlane,
-    wal_acceptors: &str,
-) -> Arc<PostgresNode> {
-    let node = compute_cplane.new_test_master_node(timeline);
-    let _node = node.append_conf(
-        "postgresql.conf",
-        &format!("wal_acceptors='{}'\n", wal_acceptors),
-    );
-    node.start().unwrap();
-    node
-}
-
-#[test]
-fn test_embedded_wal_proposer() {
-    let local_env = integration_tests::create_test_env("test_embedded_wal_proposer");
-
-    const REDUNDANCY: usize = 3;
-    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
-    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
-    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
-
-    // start postgres
-    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
-
-    // check basic work with table
-    node.safe_psql(
-        "postgres",
-        "CREATE TABLE t(key int primary key, value text)",
-    );
-    node.safe_psql(
-        "postgres",
-        "INSERT INTO t SELECT generate_series(1,100000), 'payload'",
-    );
-    let count: i64 = node
-        .safe_psql("postgres", "SELECT sum(key) FROM t")
-        .first()
-        .unwrap()
-        .get(0);
-    println!("sum = {}", count);
-    assert_eq!(count, 5000050000);
-    // check wal files equality
-}
-
-#[test]
-fn test_acceptors_normal_work() {
-    let local_env = integration_tests::create_test_env("test_acceptors_normal_work");
-
-    const REDUNDANCY: usize = 3;
-    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
-    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
-    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
-
-    // start postgres
-    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
-
-    // check basic work with table
-    node.safe_psql(
-        "postgres",
-        "CREATE TABLE t(key int primary key, value text)",
-    );
-    node.safe_psql(
-        "postgres",
-        "INSERT INTO t SELECT generate_series(1,100000), 'payload'",
-    );
-    let count: i64 = node
-        .safe_psql("postgres", "SELECT sum(key) FROM t")
-        .first()
-        .unwrap()
-        .get(0);
-    println!("sum = {}", count);
-    assert_eq!(count, 5000050000);
-    // check wal files equality
-}
-
-// Run page server and multiple safekeepers, and multiple compute nodes running
-// against different timelines.
-#[test]
-fn test_many_timelines() {
-    // Initialize a new repository, and set up WAL safekeepers and page server.
-    const REDUNDANCY: usize = 3;
-    const N_TIMELINES: usize = 5;
-    let local_env = integration_tests::create_test_env("test_many_timelines");
-    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
-    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
-    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
-
-    // Create branches
-    let mut timelines: Vec<String> = vec!["main".to_string()];
-
-    for i in 1..N_TIMELINES {
-        let branchname = format!("experimental{}", i);
-        storage_cplane
-            .pageserver
-            .branch_create(&branchname, "main")
-            .unwrap();
-        timelines.push(branchname);
-    }
-
-    // start postgres on each timeline
-    let mut nodes = Vec::new();
-    for tli_name in timelines {
-        let node = start_node_with_wal_proposer(&tli_name, &mut compute_cplane, &wal_acceptors);
-        nodes.push(node.clone());
-    }
-
-    // create schema
-    for node in &nodes {
-        node.safe_psql(
-            "postgres",
-            "CREATE TABLE t(key int primary key, value text)",
-        );
-    }
-
-    // Populate data
-    for node in &nodes {
-        node.safe_psql(
-            "postgres",
-            "INSERT INTO t SELECT generate_series(1,100000), 'payload'",
-        );
-    }
-
-    // Check data
-    for node in &nodes {
-        let count: i64 = node
-            .safe_psql("postgres", "SELECT sum(key) FROM t")
-            .first()
-            .unwrap()
-            .get(0);
-        println!("sum = {}", count);
-        assert_eq!(count, 5000050000);
-    }
-}
-
-// Majority is always alive
-#[test]
-fn test_acceptors_restarts() {
-    let local_env = integration_tests::create_test_env("test_acceptors_restarts");
-
-    // Start pageserver that reads WAL directly from that postgres
-    const REDUNDANCY: usize = 3;
-    const FAULT_PROBABILITY: f32 = 0.01;
-
-    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
-    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
-    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
-    let mut rng = rand::thread_rng();
-
-    // start postgres
-    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
-
-    let mut failed_node: Option<usize> = None;
-
-    // check basic work with table
-    node.safe_psql(
-        "postgres",
-        "CREATE TABLE t(key int primary key, value text)",
-    );
-    let mut psql = node.open_psql("postgres");
-    for i in 1..=1000 {
-        psql.execute("INSERT INTO t values ($1, 'payload')", &[&i])
-            .unwrap();
-        let prob: f32 = rng.gen();
-        if prob <= FAULT_PROBABILITY {
-            if let Some(node) = failed_node {
-                storage_cplane.wal_acceptors[node].start();
-                failed_node = None;
-            } else {
-                let node: usize = rng.gen_range(0..REDUNDANCY);
-                failed_node = Some(node);
-                storage_cplane.wal_acceptors[node].stop().unwrap();
-            }
-        }
-    }
-    let count: i64 = node
-        .safe_psql("postgres", "SELECT sum(key) FROM t")
-        .first()
-        .unwrap()
-        .get(0);
-    println!("sum = {}", count);
-    assert_eq!(count, 500500);
-}
-
-fn start_acceptor(cplane: &Arc<TestStorageControlPlane>, no: usize) {
-    let cp = cplane.clone();
-    thread::spawn(move || {
-        thread::sleep(time::Duration::from_secs(DOWNTIME));
-        cp.wal_acceptors[no].start();
-    });
-}
-
-// Stop majority of acceptors while compute is under the load. Boot
-// them again and check that nothing was losed. Repeat.
-// N_CRASHES env var
-#[test]
-fn test_acceptors_unavailability() {
-    let local_env = integration_tests::create_test_env("test_acceptors_unavailability");
-
-    // Start pageserver that reads WAL directly from that postgres
-    const REDUNDANCY: usize = 2;
-
-    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
-    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
-    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
-
-    // start postgres
-    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
-
-    // check basic work with table
-    node.safe_psql(
-        "postgres",
-        "CREATE TABLE t(key int primary key, value text)",
-    );
-    let mut psql = node.open_psql("postgres");
-    psql.execute("INSERT INTO t values (1, 'payload')", &[])
-        .unwrap();
-
-    // Shut down all wal acceptors
-    storage_cplane.wal_acceptors[0].stop().unwrap();
-    let cp = Arc::new(storage_cplane);
-    start_acceptor(&cp, 0);
-    let now = SystemTime::now();
-    psql.execute("INSERT INTO t values (2, 'payload')", &[])
-        .unwrap();
-    // Here we check that the query above was hanging
-    // while wal_acceptor was unavailiable
-    assert!(now.elapsed().unwrap().as_secs() >= DOWNTIME);
-    psql.execute("INSERT INTO t values (3, 'payload')", &[])
-        .unwrap();
-
-    cp.wal_acceptors[1].stop().unwrap();
-    start_acceptor(&cp, 1);
-    psql.execute("INSERT INTO t values (4, 'payload')", &[])
-        .unwrap();
-    // Here we check that the query above was hanging
-    // while wal_acceptor was unavailiable
-    assert!(now.elapsed().unwrap().as_secs() >= 2 * DOWNTIME);
-
-    psql.execute("INSERT INTO t values (5, 'payload')", &[])
-        .unwrap();
-
-    let count: i64 = node
-        .safe_psql("postgres", "SELECT sum(key) FROM t")
-        .first()
-        .unwrap()
-        .get(0);
-    println!("sum = {}", count);
-    // Ensure that all inserts succeeded.
-    // Including ones that were waiting for wal acceptor restart.
-    assert_eq!(count, 15);
-}
-
-fn simulate_failures(cplane: Arc<TestStorageControlPlane>) {
-    let mut rng = rand::thread_rng();
-    let n_acceptors = cplane.wal_acceptors.len();
-    let failure_period = time::Duration::from_secs(1);
-    while cplane.is_running() {
-        thread::sleep(failure_period);
-        let mask: u32 = rng.gen_range(0..(1 << n_acceptors));
-        for i in 0..n_acceptors {
-            if (mask & (1 << i)) != 0 {
-                cplane.wal_acceptors[i].stop().unwrap();
-            }
-        }
-        thread::sleep(failure_period);
-        for i in 0..n_acceptors {
-            if (mask & (1 << i)) != 0 {
-                cplane.wal_acceptors[i].start();
-            }
-        }
-    }
-}
-
-// Race condition test
-#[test]
-fn test_race_conditions() {
-    let local_env = integration_tests::create_test_env("test_race_conditions");
-
-    // Start pageserver that reads WAL directly from that postgres
-    const REDUNDANCY: usize = 3;
-
-    let storage_cplane = Arc::new(TestStorageControlPlane::fault_tolerant(
-        &local_env, REDUNDANCY,
-    ));
-    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
-    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
-
-    // start postgres
-    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
-
-    // check basic work with table
-    node.safe_psql(
-        "postgres",
-        "CREATE TABLE t(key int primary key, value text)",
-    );
-
-    let cp = storage_cplane.clone();
-    let failures_thread = thread::spawn(move || {
-        simulate_failures(cp);
-    });
-
-    let mut psql = node.open_psql("postgres");
-    for i in 1..=1000 {
-        psql.execute("INSERT INTO t values ($1, 'payload')", &[&i])
-            .unwrap();
-    }
-    let count: i64 = node
-        .safe_psql("postgres", "SELECT sum(key) FROM t")
-        .first()
-        .unwrap()
-        .get(0);
-    println!("sum = {}", count);
-    assert_eq!(count, 500500);
-
-    storage_cplane.stop();
-    failures_thread.join().unwrap();
-}

monitoring/docker-compose.yml

@@ -0,0 +1,25 @@
+version: "3"
+services:
+
+  prometheus:
+    container_name: prometheus
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
+    # ports:
+    #   - "9090:9090"
+    # TODO: find a proper portable solution
+    network_mode: "host"
+
+  grafana:
+    image: grafana/grafana:latest
+    volumes:
+      - ./grafana.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+    # ports:
+    #   - "3000:3000"
+    # TODO: find a proper portable solution
+    network_mode: "host"

monitoring/grafana.yaml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+datasources:
+- name: Prometheus
+  type: prometheus
+  access: proxy
+  orgId: 1
+  url: http://localhost:9090
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: false

monitoring/prometheus.yaml

@@ -0,0 +1,5 @@
+scrape_configs:
+  - job_name: 'default'
+    scrape_interval: 10s
+    static_configs:
+      - targets: ['localhost:9898']

pageserver/Cargo.toml

@@ -7,41 +7,39 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+bookfile = "^0.3"
 chrono = "0.4.19"
 rand = "0.8.3"
 regex = "1.4.5"
 bytes = { version = "1.0.1", features = ['serde'] }
 byteorder = "1.4.3"
 futures = "0.3.13"
+hyper = "0.14"
 lazy_static = "1.4.0"
 slog-stdlog = "4.1.0"
-slog-async = "2.6.0"
 slog-scope = "4.4.0"
 slog-term = "2.8.0"
 slog = "2.7.0"
 log = "0.4.14"
 clap = "2.33.0"
-termion = "1.5.6"
-tui = "0.14.0"
 daemonize = "0.4.1"
-rust-s3 = { version = "0.27.0-rc4", features = ["no-verify-ssl"] }
-tokio = { version = "1.3.0", features = ["full"] }
-tokio-stream = { version = "0.1.4" }
+tokio = { version = "1.5.0", features = ["full"] }
 postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-rocksdb = "0.16.0"
+routerify = "2"
 anyhow = "1.0"
 crc32c = "0.6.0"
-walkdir = "2"
 thiserror = "1.0"
-hex = "0.4.3"
+hex = { version = "0.4.3", features = ["serde"] }
 tar = "0.4.33"
-parse_duration = "2.1.1"
+humantime = "2.1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
-fs_extra = "1.2.0"
+toml = "0.5"
+scopeguard = "1.1.0"
 
 postgres_ffi = { path = "../postgres_ffi" }
+zenith_metrics = { path = "../zenith_metrics" }
 zenith_utils = { path = "../zenith_utils" }
 workspace_hack = { path = "../workspace_hack" }

pageserver/README

@@ -1,91 +1,4 @@
-Page Server
-===========
-
-
-How to test
------------
-
-
-1. Compile and install Postgres from this repository (there are
-   modifications, so vanilla Postgres won't do)
-
-    ./configure --prefix=/home/heikki/zenith-install
-
-2. Compile the page server
-
-    cd pageserver
-    cargo build
-
-3. Create another "dummy" cluster that will be used by the page server when it applies
-   the WAL records. (shouldn't really need this, getting rid of it is a TODO):
-
-    /home/heikki/zenith-install/bin/initdb -D /data/zenith-dummy
-
-
-4. Initialize and start a new postgres cluster
-
-    /home/heikki/zenith-install/bin/initdb -D /data/zenith-test-db --username=postgres
-    /home/heikki/zenith-install/bin/postgres -D /data/zenith-test-db
-
-5. In another terminal, start the page server.
-
-    PGDATA=/data/zenith-dummy PATH=/home/heikki/zenith-install/bin:$PATH ./target/debug/pageserver
-
-   It should connect to the postgres instance using streaming replication, and print something
-   like this:
-
-    $ PGDATA=/data/zenith-dummy PATH=/home/heikki/zenith-install/bin:$PATH ./target/debug/pageserver
-    Starting WAL receiver
-    connecting...
-    Starting page server on 127.0.0.1:5430
-    connected!
-    page cache is empty
-
-6. You can now open another terminal and issue DDL commands. Generated WAL records will
-   be streamed to the page servers, and attached to blocks that they apply to in its
-   page cache
-
-    $ psql postgres -U postgres
-    psql (14devel)
-    Type "help" for help.
-    
-    postgres=# create table mydata (i int4);
-    CREATE TABLE
-    postgres=# insert into mydata select g from generate_series(1,100) g;
-    INSERT 0 100
-    postgres=# 
-
-7. The GetPage@LSN interface to the compute nodes isn't working yet, but to simulate
-   that, the page server generates a test GetPage@LSN call every 5 seconds on a random
-   block that's in the page cache. In a few seconds, you should see output from that:
-
-    testing GetPage@LSN for block 0
-    WAL record at LSN 23584576 initializes the page
-    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167DF40
-    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167DF80
-    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167DFC0
-    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E018
-    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E058
-    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E098
-    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E0D8
-    2021-03-19 11:03:13.792 EET [11439] LOG:  applied WAL record at 0/167E118
-    2021-03-19 11:03:13.792 EET [11439] LOG:  applied WAL record at 0/167E158
-    2021-03-19 11:03:13.792 EET [11439] LOG:  applied WAL record at 0/167E198
-    applied 10 WAL records to produce page image at LSN 18446744073709547246
-
-
-
-Architecture
-============
-
-The Page Server is responsible for all operations on a number of
-"chunks" of relation data. A chunk corresponds to a PostgreSQL
-relation segment (i.e. one max. 1 GB file in the data directory), but
-it holds all the different versions of every page in the segment that
-are still needed by the system.
-
-Determining which chunk each Page Server holds is handled elsewhere. (TODO:
-currently, there is only one Page Server which holds all chunks)
+## Page server architecture
 
 The Page Server has a few different duties:
 
@@ -95,10 +8,11 @@ The Page Server has a few different duties:
 - Backup to S3
 
 
-The Page Server consists of multiple threads that operate on a shared
-cache of page versions:
 
 
+The Page Server consists of multiple threads that operate on a shared
+repository of page versions:
+
                                            | WAL
                                            V
                                    +--------------+
@@ -110,16 +24,14 @@ cache of page versions:
                   +---------+                              ..........            |    |
                   |         |                              .        .            |    |
  GetPage@LSN      |         |                              . backup .  ------->  | S3 |
-------------->    |  Page   |         page cache           .        .            |    |
+------------->    |  Page   |         repository           .        .            |    |
                   | Service |                              ..........            |    |
    page           |         |                                                    +----+
 <-------------    |         |
-                  +---------+
-
-                             ...................................
-                             .                                 .
-                             . Garbage Collection / Compaction .
-                             ...................................
+                  +---------+      +--------------------+
+		                   |   Checkpointing /  |
+				   | Garbage collection |
+                                   +--------------------+
 
 Legend:
 
@@ -139,7 +51,7 @@ Page Service
 ------------
 
 The Page Service listens for GetPage@LSN requests from the Compute Nodes,
-and responds with pages from the page cache.
+and responds with pages from the repository.
 
 
 WAL Receiver
@@ -148,25 +60,59 @@ WAL Receiver
 The WAL receiver connects to the external WAL safekeeping service (or
 directly to the primary) using PostgreSQL physical streaming
 replication, and continuously receives WAL. It decodes the WAL records,
-and stores them to the page cache.
+and stores them to the repository.
 
 
-Page Cache
+Repository
 ----------
 
-The Page Cache is a data structure, to hold all the different page versions.
-It is accessed by all the other threads, to perform their duties.
+The repository stores all the page versions, or WAL records needed to
+reconstruct them. Each tenant has a separate Repository, which is
+stored in the .zenith/tenants/<tenantid> directory.
 
-Currently, the page cache is implemented fully in-memory. TODO: Store it
-on disk. Define a file format.
+Repository is an abstract trait, defined in `repository.rs`. It is
+implemented by the LayeredRepository object in
+`layered_repository.rs`. There is only that one implementation of the
+Repository trait, but it's still a useful abstraction that keeps the
+interface for the low-level storage functionality clean. The layered
+storage format is described in layered_repository/README.md.
+
+Each repository consists of multiple Timelines. Timeline is a
+workhorse that accepts page changes from the WAL, and serves
+get_page_at_lsn() and get_rel_size() requests. Note: this has nothing
+to do with PostgreSQL WAL timeline. The term "timeline" is mostly
+interchangeable with "branch", there is a one-to-one mapping from
+branch to timeline. A timeline has a unique ID within the tenant,
+represented as 16-byte hex string that never changes, whereas a
+branch is a user-given name for a timeline.
+
+Each repository also has a WAL redo manager associated with it, see
+`walredo.rs`. The WAL redo manager is used to replay PostgreSQL WAL
+records, whenever we need to reconstruct a page version from WAL to
+satisfy a GetPage@LSN request, or to avoid accumulating too much WAL
+for a page. The WAL redo manager uses a Postgres process running in
+special zenith wal-redo mode to do the actual WAL redo, and
+communicates with the process using a pipe.
 
 
-TODO: Garbage Collection / Compaction
--------------------------------------
+Checkpointing / Garbage Collection
+----------------------------------
 
-Periodically, the Garbage Collection / Compaction thread runs
-and applies pending WAL records, and removes old page versions that
-are no longer needed.
+Periodically, the checkpointer thread wakes up and performs housekeeping
+duties on the repository. It has two duties:
+
+### Checkpointing
+
+Flush WAL that has accumulated in memory to disk, so that the old WAL
+can be truncated away in the WAL safekeepers. Also, to free up memory
+for receiving new WAL. This process is called "checkpointing". It's
+similar to checkpointing in PostgreSQL or other DBMSs, but in the page
+server, checkpointing happens on a per-segment basis.
+
+### Garbage collection
+
+Remove old on-disk layer files that are no longer needed according to the
+PITR retention policy
 
 
 TODO: Backup service
@@ -177,3 +123,7 @@ The backup service is responsible for periodically pushing the chunks to S3.
 TODO: How/when do restore from S3? Whenever we get a GetPage@LSN request for
 a chunk we don't currently have? Or when an external Control Plane tells us?
 
+TODO: Sharding
+--------------------
+
+We should be able to run multiple Page Servers that handle sharded data.

pageserver/launch.sh

@@ -1,62 +0,0 @@
-#!/bin/sh
-#
-# Set up a simple Compute Node + Page Server combination locally.
-#
-# NOTE: This doesn't clean up between invocations. You'll need to manually:
-#
-# - Kill any previous 'postgres' and 'pageserver' processes
-# - Clear the S3 bucket
-# - Remove the 'zenith-pgdata' directory
-
-
-set -e
-
-# Set up some config.
-#
-# CHANGE THESE ACCORDING TO YOUR S3 INSTALLATION
-export S3_REGION=auto
-export S3_ENDPOINT=https://localhost:9000
-export S3_ACCESSKEY=minioadmin
-export S3_SECRET=pikkunen
-export S3_BUCKET=zenith-testbucket
-
-
-COMPUTE_NODE_PGDATA=zenith-pgdata
-
-
-# 1. Initialize a cluster.
-initdb -D $COMPUTE_NODE_PGDATA -U zenith
-
-echo "port=65432" >> $COMPUTE_NODE_PGDATA/postgresql.conf
-echo "log_connections=on" >> $COMPUTE_NODE_PGDATA/postgresql.conf
-
-# Use a small shared_buffers, so that we hit the Page Server more
-# easily.
-echo "shared_buffers = 1MB" >> $COMPUTE_NODE_PGDATA/postgresql.conf
-
-# TODO: page server should use a replication slot, or some other mechanism
-# to make sure that the primary doesn't lose data that the page server still
-# needs. (The WAL safekeepers should ensure that)
-echo "wal_keep_size=10GB" >> $COMPUTE_NODE_PGDATA/postgresql.conf
-
-# Tell the Postgres server how to connect to the Page Server
-echo "page_server_connstring='host=localhost port=5430'" >> $COMPUTE_NODE_PGDATA/postgresql.conf
-
-
-# 2. Run zenith_push to push a base backup fo the database to an S3 bucket. The
-# Page Server will read it from there
-zenith_push -D $COMPUTE_NODE_PGDATA
-
-
-# 3. Launch page server
-rm -rf /tmp/pgdata-dummy
-initdb -N -D /tmp/pgdata-dummy
-PGDATA=/tmp/pgdata-dummy ./target/debug/pageserver  &
-
-# 4. Start up the Postgres server
-postgres -D $COMPUTE_NODE_PGDATA &
-
-
-echo "ALL SET! You can now connect to Postgres with something like:"
-echo ""
-echo 'psql "dbname=postgres host=localhost user=zenith port=65432"'

pageserver/src/basebackup.rs

@@ -1,227 +1,313 @@
-use crate::ZTimelineId;
+//!
+//! Generate a tarball with files needed to bootstrap ComputeNode.
+//!
+//! TODO: this module has nothing to do with PostgreSQL pg_basebackup.
+//! It could use a better name.
+//!
+//! Stateless Postgres compute node is launched by sending a tarball
+//! which contains non-relational data (multixacts, clog, filenodemaps, twophase files),
+//! generated pg_control and dummy segment of WAL.
+//! This module is responsible for creation of such tarball
+//! from data stored in object storage.
+//!
+use bytes::{BufMut, BytesMut};
 use log::*;
+use std::io;
 use std::io::Write;
 use std::sync::Arc;
-use tar::Builder;
-use walkdir::WalkDir;
+use std::time::SystemTime;
+use tar::{Builder, EntryType, Header};
 
+use crate::relish::*;
 use crate::repository::Timeline;
-use postgres_ffi::relfile_utils::*;
-use zenith_utils::lsn::Lsn;
+use postgres_ffi::xlog_utils::*;
+use postgres_ffi::*;
+use zenith_utils::lsn::{Lsn, RecordLsn};
 
-///
-/// Generate tarball with non-relational files from repository
-///
-pub fn send_tarball_at_lsn(
-    write: &mut dyn Write,
-    timelineid: ZTimelineId,
-    _timeline: &Arc<dyn Timeline>,
-    _lsn: Lsn,
-    snapshot_lsn: Lsn,
-) -> anyhow::Result<()> {
-    let mut ar = Builder::new(write);
-
-    let snappath = format!("timelines/{}/snapshots/{:016X}", timelineid, snapshot_lsn.0);
-    let walpath = format!("timelines/{}/wal", timelineid);
-
-    debug!("sending tarball of snapshot in {}", snappath);
-    for entry in WalkDir::new(&snappath) {
-        let entry = entry?;
-        let fullpath = entry.path();
-        let relpath = entry.path().strip_prefix(&snappath).unwrap();
-
-        if relpath.to_str().unwrap() == "" {
-            continue;
-        }
-
-        if entry.file_type().is_dir() {
-            trace!(
-                "sending dir {} as {}",
-                fullpath.display(),
-                relpath.display()
-            );
-            ar.append_dir(relpath, fullpath)?;
-        } else if entry.file_type().is_symlink() {
-            error!("ignoring symlink in snapshot dir");
-        } else if entry.file_type().is_file() {
-            // Shared catalogs are exempt
-            if relpath.starts_with("global/") {
-                trace!("sending shared catalog {}", relpath.display());
-                ar.append_path_with_name(fullpath, relpath)?;
-            } else if !is_rel_file_path(relpath.to_str().unwrap()) {
-                trace!("sending {}", relpath.display());
-                ar.append_path_with_name(fullpath, relpath)?;
-            } else {
-                trace!("not sending {}", relpath.display());
-            }
-        } else {
-            error!("unknown file type: {}", fullpath.display());
-        }
-    }
-
-    // FIXME: Also send all the WAL. The compute node would only need
-    // the WAL that applies to non-relation files, because the page
-    // server handles all the relation files. But we don't have a
-    // mechanism for separating relation and non-relation WAL at the
-    // moment.
-    for entry in std::fs::read_dir(&walpath)? {
-        let entry = entry?;
-        let fullpath = &entry.path();
-        let relpath = fullpath.strip_prefix(&walpath).unwrap();
-
-        if !entry.path().is_file() {
-            continue;
-        }
-
-        let archive_fname = relpath.to_str().unwrap();
-        let archive_fname = archive_fname
-            .strip_suffix(".partial")
-            .unwrap_or(&archive_fname);
-        let archive_path = "pg_wal/".to_owned() + archive_fname;
-        ar.append_path_with_name(fullpath, archive_path)?;
-    }
-    ar.finish()?;
-    debug!("all tarred up!");
-    Ok(())
+/// This is short-living object only for the time of tarball creation,
+/// created mostly to avoid passing a lot of parameters between various functions
+/// used for constructing tarball.
+pub struct Basebackup<'a> {
+    ar: Builder<&'a mut dyn Write>,
+    timeline: &'a Arc<dyn Timeline>,
+    lsn: Lsn,
+    prev_record_lsn: Lsn,
 }
 
-///
-/// Send a tarball containing a snapshot of all non-relation files in the
-/// PostgreSQL data directory, at given LSN
-///
-/// There must be a snapshot at the given LSN in the snapshots directory, we cannot
-/// reconstruct the state at an arbitrary LSN at the moment.
-///
-pub fn send_snapshot_tarball(
-    write: &mut dyn Write,
-    timelineid: ZTimelineId,
-    snapshotlsn: Lsn,
-) -> Result<(), std::io::Error> {
-    let mut ar = Builder::new(write);
-
-    let snappath = format!("timelines/{}/snapshots/{:016X}", timelineid, snapshotlsn.0);
-    let walpath = format!("timelines/{}/wal", timelineid);
-
-    debug!("sending tarball of snapshot in {}", snappath);
-    //ar.append_dir_all("", &snappath)?;
-
-    for entry in WalkDir::new(&snappath) {
-        let entry = entry?;
-        let fullpath = entry.path();
-        let relpath = entry.path().strip_prefix(&snappath).unwrap();
-
-        if relpath.to_str().unwrap() == "" {
-            continue;
-        }
-
-        if entry.file_type().is_dir() {
-            trace!(
-                "sending dir {} as {}",
-                fullpath.display(),
-                relpath.display()
-            );
-            ar.append_dir(relpath, fullpath)?;
-        } else if entry.file_type().is_symlink() {
-            error!("ignoring symlink in snapshot dir");
-        } else if entry.file_type().is_file() {
-            // Shared catalogs are exempt
-            if relpath.starts_with("global/") {
-                trace!("sending shared catalog {}", relpath.display());
-                ar.append_path_with_name(fullpath, relpath)?;
-            } else if !is_rel_file_path(relpath.to_str().unwrap()) {
-                trace!("sending {}", relpath.display());
-                ar.append_path_with_name(fullpath, relpath)?;
-            } else {
-                trace!("not sending {}", relpath.display());
-
-                // FIXME: For now, also send all the relation files.
-                // This really shouldn't be necessary, and kind of
-                // defeats the point of having a page server in the
-                // first place. But it is useful at least when
-                // debugging with the DEBUG_COMPARE_LOCAL option (see
-                // vendor/postgres/src/backend/storage/smgr/pagestore_smgr.c)
-
-                ar.append_path_with_name(fullpath, relpath)?;
+// Create basebackup with non-rel data in it. Omit relational data.
+//
+// Currently we use empty lsn in two cases:
+//  * During the basebackup right after timeline creation
+//  * When working without safekeepers. In this situation it is important to match the lsn
+//    we are taking basebackup on with the lsn that is used in pageserver's walreceiver
+//    to start the replication.
+impl<'a> Basebackup<'a> {
+    pub fn new(
+        write: &'a mut dyn Write,
+        timeline: &'a Arc<dyn Timeline>,
+        req_lsn: Option<Lsn>,
+    ) -> Basebackup<'a> {
+        let RecordLsn {
+            last: lsn,
+            prev: prev_record_lsn,
+        } = if let Some(lsn) = req_lsn {
+            // FIXME: that wouldn't work since we don't know prev for old LSN's.
+            // Probably it is better to avoid using prev in compute node start
+            // at all and acept the fact that first WAL record in the timeline would
+            // have zero as prev. https://github.com/zenithdb/zenith/issues/506
+            RecordLsn {
+                last: lsn,
+                prev: lsn,
             }
         } else {
-            error!("unknown file type: {}", fullpath.display());
-        }
-    }
-
-    // FIXME: Also send all the WAL. The compute node would only need
-    // the WAL that applies to non-relation files, because the page
-    // server handles all the relation files. But we don't have a
-    // mechanism for separating relation and non-relation WAL at the
-    // moment.
-    for entry in std::fs::read_dir(&walpath)? {
-        let entry = entry?;
-        let fullpath = &entry.path();
-        let relpath = fullpath.strip_prefix(&walpath).unwrap();
-
-        if !entry.path().is_file() {
-            continue;
-        }
-
-        let archive_fname = relpath.to_str().unwrap();
-        let archive_fname = archive_fname
-            .strip_suffix(".partial")
-            .unwrap_or(&archive_fname);
-        let archive_path = "pg_wal/".to_owned() + archive_fname;
-        ar.append_path_with_name(fullpath, archive_path)?;
-    }
-
-    ar.finish()?;
-    debug!("all tarred up!");
-    Ok(())
-}
-
-///
-/// Parse a path, relative to the root of PostgreSQL data directory, as
-/// a PostgreSQL relation data file.
-///
-fn parse_rel_file_path(path: &str) -> Result<(), FilePathError> {
-    /*
-     * Relation data files can be in one of the following directories:
-     *
-     * global/
-     *		shared relations
-     *
-     * base/<db oid>/
-     *		regular relations, default tablespace
-     *
-     * pg_tblspc/<tblspc oid>/<tblspc version>/
-     *		within a non-default tablespace (the name of the directory
-     *		depends on version)
-     *
-     * And the relation data files themselves have a filename like:
-     *
-     * <oid>.<segment number>
-     */
-    if let Some(fname) = path.strip_prefix("global/") {
-        let (_relnode, _forknum, _segno) = parse_relfilename(fname)?;
-
-        Ok(())
-    } else if let Some(dbpath) = path.strip_prefix("base/") {
-        let mut s = dbpath.split('/');
-        let dbnode_str = s.next().ok_or(FilePathError::InvalidFileName)?;
-        let _dbnode = dbnode_str.parse::<u32>()?;
-        let fname = s.next().ok_or(FilePathError::InvalidFileName)?;
-        if s.next().is_some() {
-            return Err(FilePathError::InvalidFileName);
+            // Atomically get last and prev LSN's
+            timeline.get_last_record_rlsn()
         };
 
-        let (_relnode, _forknum, _segno) = parse_relfilename(fname)?;
+        Basebackup {
+            ar: Builder::new(write),
+            timeline,
+            lsn,
+            prev_record_lsn,
+        }
+    }
+
+    pub fn send_tarball(&mut self) -> anyhow::Result<()> {
+        // Create pgdata subdirs structure
+        for dir in pg_constants::PGDATA_SUBDIRS.iter() {
+            info!("send subdir {:?}", *dir);
+            let header = new_tar_header_dir(*dir)?;
+            self.ar.append(&header, &mut io::empty())?;
+        }
+
+        // Send empty config files.
+        for filepath in pg_constants::PGDATA_SPECIAL_FILES.iter() {
+            if *filepath == "pg_hba.conf" {
+                let data = pg_constants::PG_HBA.as_bytes();
+                let header = new_tar_header(&filepath, data.len() as u64)?;
+                self.ar.append(&header, &data[..])?;
+            } else {
+                let header = new_tar_header(&filepath, 0)?;
+                self.ar.append(&header, &mut io::empty())?;
+            }
+        }
+
+        // Gather non-relational files from object storage pages.
+        for obj in self.timeline.list_nonrels(self.lsn)? {
+            match obj {
+                RelishTag::Slru { slru, segno } => {
+                    self.add_slru_segment(slru, segno)?;
+                }
+                RelishTag::FileNodeMap { spcnode, dbnode } => {
+                    self.add_relmap_file(spcnode, dbnode)?;
+                }
+                RelishTag::TwoPhase { xid } => {
+                    self.add_twophase_file(xid)?;
+                }
+                _ => {}
+            }
+        }
+
+        // Generate pg_control and bootstrap WAL segment.
+        self.add_pgcontrol_file()?;
+        self.ar.finish()?;
+        debug!("all tarred up!");
+        Ok(())
+    }
+
+    //
+    // Generate SLRU segment files from repository.
+    //
+    fn add_slru_segment(&mut self, slru: SlruKind, segno: u32) -> anyhow::Result<()> {
+        let seg_size = self
+            .timeline
+            .get_relish_size(RelishTag::Slru { slru, segno }, self.lsn)?;
+
+        if seg_size == None {
+            trace!(
+                "SLRU segment {}/{:>04X} was truncated",
+                slru.to_str(),
+                segno
+            );
+            return Ok(());
+        }
+
+        let nblocks = seg_size.unwrap();
+
+        let mut slru_buf: Vec<u8> =
+            Vec::with_capacity(nblocks as usize * pg_constants::BLCKSZ as usize);
+        for blknum in 0..nblocks {
+            let img = self.timeline.get_page_at_lsn_nowait(
+                RelishTag::Slru { slru, segno },
+                blknum,
+                self.lsn,
+            )?;
+            assert!(img.len() == pg_constants::BLCKSZ as usize);
+
+            slru_buf.extend_from_slice(&img);
+        }
+
+        let segname = format!("{}/{:>04X}", slru.to_str(), segno);
+        let header = new_tar_header(&segname, slru_buf.len() as u64)?;
+        self.ar.append(&header, slru_buf.as_slice())?;
+
+        trace!("Added to basebackup slru {} relsize {}", segname, nblocks);
+        Ok(())
+    }
+
+    //
+    // Extract pg_filenode.map files from repository
+    // Along with them also send PG_VERSION for each database.
+    //
+    fn add_relmap_file(&mut self, spcnode: u32, dbnode: u32) -> anyhow::Result<()> {
+        let img = self.timeline.get_page_at_lsn_nowait(
+            RelishTag::FileNodeMap { spcnode, dbnode },
+            0,
+            self.lsn,
+        )?;
+        let path = if spcnode == pg_constants::GLOBALTABLESPACE_OID {
+            let dst_path = "PG_VERSION";
+            let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
+            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
+            self.ar.append(&header, &version_bytes[..])?;
+
+            let dst_path = format!("global/PG_VERSION");
+            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
+            self.ar.append(&header, &version_bytes[..])?;
+
+            String::from("global/pg_filenode.map") // filenode map for global tablespace
+        } else {
+            // User defined tablespaces are not supported
+            assert!(spcnode == pg_constants::DEFAULTTABLESPACE_OID);
+
+            // Append dir path for each database
+            let path = format!("base/{}", dbnode);
+            let header = new_tar_header_dir(&path)?;
+            self.ar.append(&header, &mut io::empty())?;
+
+            let dst_path = format!("base/{}/PG_VERSION", dbnode);
+            let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
+            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
+            self.ar.append(&header, &version_bytes[..])?;
+
+            format!("base/{}/pg_filenode.map", dbnode)
+        };
+        assert!(img.len() == 512);
+        let header = new_tar_header(&path, img.len() as u64)?;
+        self.ar.append(&header, &img[..])?;
+        Ok(())
+    }
+
+    //
+    // Extract twophase state files
+    //
+    fn add_twophase_file(&mut self, xid: TransactionId) -> anyhow::Result<()> {
+        let img = self
+            .timeline
+            .get_page_at_lsn_nowait(RelishTag::TwoPhase { xid }, 0, self.lsn)?;
+
+        let mut buf = BytesMut::new();
+        buf.extend_from_slice(&img[..]);
+        let crc = crc32c::crc32c(&img[..]);
+        buf.put_u32_le(crc);
+        let path = format!("pg_twophase/{:>08X}", xid);
+        let header = new_tar_header(&path, buf.len() as u64)?;
+        self.ar.append(&header, &buf[..])?;
 
         Ok(())
-    } else if path.strip_prefix("pg_tblspc/").is_some() {
-        // TODO
-        error!("tablespaces not implemented yet");
-        Err(FilePathError::InvalidFileName)
-    } else {
-        Err(FilePathError::InvalidFileName)
+    }
+
+    //
+    // Add generated pg_control file and bootstrap WAL segment.
+    // Also send zenith.signal file with extra bootstrap data.
+    //
+    fn add_pgcontrol_file(&mut self) -> anyhow::Result<()> {
+        let checkpoint_bytes =
+            self.timeline
+                .get_page_at_lsn_nowait(RelishTag::Checkpoint, 0, self.lsn)?;
+        let pg_control_bytes =
+            self.timeline
+                .get_page_at_lsn_nowait(RelishTag::ControlFile, 0, self.lsn)?;
+        let mut pg_control = ControlFileData::decode(&pg_control_bytes)?;
+        let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
+
+        // Generate new pg_control and WAL needed for bootstrap
+        let checkpoint_segno = self.lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE);
+        let checkpoint_lsn = XLogSegNoOffsetToRecPtr(
+            checkpoint_segno,
+            XLOG_SIZE_OF_XLOG_LONG_PHD as u32,
+            pg_constants::WAL_SEGMENT_SIZE,
+        );
+        checkpoint.redo = self.lsn.0 + self.lsn.calc_padding(8u32);
+
+        //reset some fields we don't want to preserve
+        //TODO Check this.
+        //We may need to determine the value from twophase data.
+        checkpoint.oldestActiveXid = 0;
+
+        //save new values in pg_control
+        pg_control.checkPoint = checkpoint_lsn;
+        pg_control.checkPointCopy = checkpoint;
+        pg_control.state = pg_constants::DB_SHUTDOWNED;
+
+        // add zenith.signal file
+        self.ar.append(
+            &new_tar_header("zenith.signal", 8)?,
+            &self.prev_record_lsn.0.to_le_bytes()[..],
+        )?;
+
+        //send pg_control
+        let pg_control_bytes = pg_control.encode();
+        let header = new_tar_header("global/pg_control", pg_control_bytes.len() as u64)?;
+        self.ar.append(&header, &pg_control_bytes[..])?;
+
+        //send wal segment
+        let wal_file_name = XLogFileName(
+            1, // FIXME: always use Postgres timeline 1
+            checkpoint_segno,
+            pg_constants::WAL_SEGMENT_SIZE,
+        );
+        let wal_file_path = format!("pg_wal/{}", wal_file_name);
+        let header = new_tar_header(&wal_file_path, pg_constants::WAL_SEGMENT_SIZE as u64)?;
+        let wal_seg = generate_wal_segment(&pg_control);
+        assert!(wal_seg.len() == pg_constants::WAL_SEGMENT_SIZE);
+        self.ar.append(&header, &wal_seg[..])?;
+        Ok(())
     }
 }
 
-fn is_rel_file_path(path: &str) -> bool {
-    parse_rel_file_path(path).is_ok()
+//
+// Create new tarball entry header
+//
+fn new_tar_header(path: &str, size: u64) -> anyhow::Result<Header> {
+    let mut header = Header::new_gnu();
+    header.set_size(size);
+    header.set_path(path)?;
+    header.set_mode(0b110000000); // -rw-------
+    header.set_mtime(
+        // use currenttime as last modified time
+        SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap()
+            .as_secs(),
+    );
+    header.set_cksum();
+    Ok(header)
+}
+
+fn new_tar_header_dir(path: &str) -> anyhow::Result<Header> {
+    let mut header = Header::new_gnu();
+    header.set_size(0);
+    header.set_path(path)?;
+    header.set_mode(0o755); // -rw-------
+    header.set_entry_type(EntryType::dir());
+    header.set_mtime(
+        // use currenttime as last modified time
+        SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap()
+            .as_secs(),
+    );
+    header.set_cksum();
+    Ok(header)
 }

pageserver/src/bin/dump_layerfile.rs

@@ -0,0 +1,25 @@
+//! Main entry point for the dump_layerfile executable
+//!
+//! A handy tool for debugging, that's all.
+use anyhow::Result;
+use clap::{App, Arg};
+use pageserver::layered_repository::dump_layerfile_from_path;
+use std::path::PathBuf;
+
+fn main() -> Result<()> {
+    let arg_matches = App::new("Zenith dump_layerfile utility")
+        .about("Dump contents of one layer file, for debugging")
+        .arg(
+            Arg::with_name("path")
+                .help("Path to file to dump")
+                .required(true)
+                .index(1),
+        )
+        .get_matches();
+
+    let path = PathBuf::from(arg_matches.value_of("path").unwrap());
+
+    dump_layerfile_from_path(&path)?;
+
+    Ok(())
+}

pageserver/src/bin/pageserver.rs

@@ -3,29 +3,154 @@
 //
 
 use log::*;
-use parse_duration::parse;
-use std::io;
-use std::process::exit;
-use std::thread;
-use std::time::Duration;
-use std::{env, path::PathBuf};
+use serde::{Deserialize, Serialize};
 use std::{
-    fs::{File, OpenOptions},
+    env,
     net::TcpListener,
+    path::{Path, PathBuf},
+    process::exit,
+    str::FromStr,
+    thread,
+    time::Duration,
 };
+use zenith_utils::{auth::JwtAuth, postgres_backend::AuthType};
 
-use anyhow::{Context, Result};
-use clap::{App, Arg};
+use anyhow::{ensure, Result};
+use clap::{App, Arg, ArgMatches};
 use daemonize::Daemonize;
 
-use slog::{Drain, FnValue};
+use pageserver::{branches, http, logger, page_service, tenant_mgr, PageServerConf};
+use zenith_utils::http::endpoint;
 
-use pageserver::{branches, page_cache, page_service, tui, PageServerConf};
+const DEFAULT_LISTEN_ADDR: &str = "127.0.0.1:64000";
+const DEFAULT_HTTP_ENDPOINT_ADDR: &str = "127.0.0.1:9898";
 
 const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
-const DEFAULT_GC_PERIOD_SEC: u64 = 10;
-//const DEFAULT_GC_HORIZON: u64 = 1024 * 1024 * 1024;
-//const DEFAULT_GC_PERIOD_SEC: u64 = 600;
+const DEFAULT_GC_PERIOD: Duration = Duration::from_secs(10);
+
+const DEFAULT_SUPERUSER: &str = "zenith_admin";
+
+/// String arguments that can be declared via CLI or config file
+#[derive(Serialize, Deserialize)]
+struct CfgFileParams {
+    listen_addr: Option<String>,
+    http_endpoint_addr: Option<String>,
+    gc_horizon: Option<String>,
+    gc_period: Option<String>,
+    pg_distrib_dir: Option<String>,
+    auth_validation_public_key_path: Option<String>,
+    auth_type: Option<String>,
+}
+
+impl CfgFileParams {
+    /// Extract string arguments from CLI
+    fn from_args(arg_matches: &ArgMatches) -> Self {
+        let get_arg = |arg_name: &str| -> Option<String> {
+            arg_matches.value_of(arg_name).map(str::to_owned)
+        };
+
+        Self {
+            listen_addr: get_arg("listen"),
+            http_endpoint_addr: get_arg("http_endpoint"),
+            gc_horizon: get_arg("gc_horizon"),
+            gc_period: get_arg("gc_period"),
+            pg_distrib_dir: get_arg("postgres-distrib"),
+            auth_validation_public_key_path: get_arg("auth-validation-public-key-path"),
+            auth_type: get_arg("auth-type"),
+        }
+    }
+
+    /// Fill missing values in `self` with `other`
+    fn or(self, other: CfgFileParams) -> Self {
+        // TODO cleaner way to do this
+        Self {
+            listen_addr: self.listen_addr.or(other.listen_addr),
+            http_endpoint_addr: self.http_endpoint_addr.or(other.http_endpoint_addr),
+            gc_horizon: self.gc_horizon.or(other.gc_horizon),
+            gc_period: self.gc_period.or(other.gc_period),
+            pg_distrib_dir: self.pg_distrib_dir.or(other.pg_distrib_dir),
+            auth_validation_public_key_path: self
+                .auth_validation_public_key_path
+                .or(other.auth_validation_public_key_path),
+            auth_type: self.auth_type.or(other.auth_type),
+        }
+    }
+
+    /// Create a PageServerConf from these string parameters
+    fn try_into_config(&self) -> Result<PageServerConf> {
+        let workdir = PathBuf::from(".");
+
+        let listen_addr = match self.listen_addr.as_ref() {
+            Some(addr) => addr.clone(),
+            None => DEFAULT_LISTEN_ADDR.to_owned(),
+        };
+
+        let http_endpoint_addr = match self.http_endpoint_addr.as_ref() {
+            Some(addr) => addr.clone(),
+            None => DEFAULT_HTTP_ENDPOINT_ADDR.to_owned(),
+        };
+
+        let gc_horizon: u64 = match self.gc_horizon.as_ref() {
+            Some(horizon_str) => horizon_str.parse()?,
+            None => DEFAULT_GC_HORIZON,
+        };
+        let gc_period = match self.gc_period.as_ref() {
+            Some(period_str) => humantime::parse_duration(period_str)?,
+            None => DEFAULT_GC_PERIOD,
+        };
+
+        let pg_distrib_dir = match self.pg_distrib_dir.as_ref() {
+            Some(pg_distrib_dir_str) => PathBuf::from(pg_distrib_dir_str),
+            None => env::current_dir()?.join("tmp_install"),
+        };
+
+        let auth_validation_public_key_path = self
+            .auth_validation_public_key_path
+            .as_ref()
+            .map(PathBuf::from);
+
+        let auth_type = self
+            .auth_type
+            .as_ref()
+            .map_or(Ok(AuthType::Trust), |auth_type| {
+                AuthType::from_str(&auth_type)
+            })?;
+
+        if !pg_distrib_dir.join("bin/postgres").exists() {
+            anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
+        }
+
+        if auth_type == AuthType::ZenithJWT {
+            ensure!(
+                auth_validation_public_key_path.is_some(),
+                "Missing auth_validation_public_key_path when auth_type is ZenithJWT"
+            );
+            let path_ref = auth_validation_public_key_path.as_ref().unwrap();
+            ensure!(
+                path_ref.exists(),
+                format!("Can't find auth_validation_public_key at {:?}", path_ref)
+            );
+        }
+
+        Ok(PageServerConf {
+            daemonize: false,
+
+            listen_addr,
+            http_endpoint_addr,
+            gc_horizon,
+            gc_period,
+
+            superuser: String::from(DEFAULT_SUPERUSER),
+
+            workdir,
+
+            pg_distrib_dir,
+
+            auth_validation_public_key_path,
+            auth_type,
+        })
+    }
+}
 
 fn main() -> Result<()> {
     let arg_matches = App::new("Zenith page server")
@@ -37,13 +162,6 @@ fn main() -> Result<()> {
                 .takes_value(true)
                 .help("listen for incoming page requests on ip:port (default: 127.0.0.1:5430)"),
         )
-        .arg(
-            Arg::with_name("interactive")
-                .short("i")
-                .long("interactive")
-                .takes_value(false)
-                .help("Interactive mode"),
-        )
         .arg(
             Arg::with_name("daemonize")
                 .short("d")
@@ -76,115 +194,91 @@ fn main() -> Result<()> {
                 .takes_value(true)
                 .help("Working directory for the pageserver"),
         )
+        .arg(
+            Arg::with_name("postgres-distrib")
+                .long("postgres-distrib")
+                .takes_value(true)
+                .help("Postgres distribution directory"),
+        )
+        .arg(
+            Arg::with_name("create-tenant")
+                .long("create-tenant")
+                .takes_value(true)
+                .help("Create tenant during init")
+                .requires("init"),
+        )
+        .arg(
+            Arg::with_name("auth-validation-public-key-path")
+                .long("auth-validation-public-key-path")
+                .takes_value(true)
+                .help("Path to public key used to validate jwt signature"),
+        )
+        .arg(
+            Arg::with_name("auth-type")
+                .long("auth-type")
+                .takes_value(true)
+                .help("Authentication scheme type. One of: Trust, MD5, ZenithJWT"),
+        )
         .get_matches();
 
-    let workdir = if let Some(workdir_arg) = arg_matches.value_of("workdir") {
-        PathBuf::from(workdir_arg)
-    } else if let Some(workdir_arg) = std::env::var_os("ZENITH_REPO_DIR") {
-        PathBuf::from(workdir_arg.to_str().unwrap())
+    let workdir = Path::new(arg_matches.value_of("workdir").unwrap_or(".zenith"));
+    let cfg_file_path = workdir.canonicalize()?.join("pageserver.toml");
+
+    let args_params = CfgFileParams::from_args(&arg_matches);
+
+    let init = arg_matches.is_present("init");
+    let create_tenant = arg_matches.value_of("create-tenant");
+
+    let params = if init {
+        // We're initializing the repo, so there's no config file yet
+        args_params
     } else {
-        PathBuf::from(".zenith")
+        // Supplement the CLI arguments with the config file
+        let cfg_file_contents = std::fs::read_to_string(&cfg_file_path)?;
+        let file_params: CfgFileParams = toml::from_str(&cfg_file_contents)?;
+        args_params.or(file_params)
     };
 
-    let pg_distrib_dir: PathBuf = {
-        if let Some(postgres_bin) = env::var_os("POSTGRES_DISTRIB_DIR") {
-            postgres_bin.into()
-        } else {
-            let cwd = env::current_dir()?;
-            cwd.join("tmp_install")
-        }
-    };
+    // Set CWD to workdir for non-daemon modes
+    env::set_current_dir(&workdir)?;
 
-    if !pg_distrib_dir.join("bin/postgres").exists() {
-        anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
-    }
+    // Ensure the config is valid, even if just init-ing
+    let mut conf = params.try_into_config()?;
 
-    let mut conf = PageServerConf {
-        daemonize: false,
-        interactive: false,
-        gc_horizon: DEFAULT_GC_HORIZON,
-        gc_period: Duration::from_secs(DEFAULT_GC_PERIOD_SEC),
-        listen_addr: "127.0.0.1:64000".parse().unwrap(),
-        // we will change the current working directory to the repository below,
-        // so always set 'workdir' to '.'
-        workdir: PathBuf::from("."),
-        pg_distrib_dir,
-    };
+    conf.daemonize = arg_matches.is_present("daemonize");
 
-    if arg_matches.is_present("daemonize") {
-        conf.daemonize = true;
-    }
-
-    if arg_matches.is_present("interactive") {
-        conf.interactive = true;
-    }
-
-    if conf.daemonize && conf.interactive {
-        eprintln!("--daemonize is not allowed with --interactive: choose one");
+    if init && conf.daemonize {
+        eprintln!("--daemonize cannot be used with --init");
         exit(1);
     }
 
-    if let Some(addr) = arg_matches.value_of("listen") {
-        conf.listen_addr = addr.parse()?;
-    }
-
-    if let Some(horizon) = arg_matches.value_of("gc_horizon") {
-        conf.gc_horizon = horizon.parse()?;
-    }
-
-    if let Some(period) = arg_matches.value_of("gc_period") {
-        conf.gc_period = parse(period)?;
-    }
-
     // The configuration is all set up now. Turn it into a 'static
     // that can be freely stored in structs and passed across threads
     // as a ref.
     let conf: &'static PageServerConf = Box::leak(Box::new(conf));
 
     // Create repo and exit if init was requested
-    if arg_matches.is_present("init") {
-        branches::init_repo(conf, &workdir)?;
+    if init {
+        branches::init_pageserver(conf, create_tenant)?;
+        // write the config file
+        let cfg_file_contents = toml::to_string_pretty(&params)?;
+        // TODO support enable-auth flag
+        std::fs::write(&cfg_file_path, cfg_file_contents)?;
+
         return Ok(());
     }
 
-    // Set CWD to workdir for non-daemon modes
-    env::set_current_dir(&workdir)?;
-
     start_pageserver(conf)
 }
 
 fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
-    let log_filename = "pageserver.log";
-    // Don't open the same file for output multiple times;
-    // the different fds could overwrite each other's output.
-    let log_file = OpenOptions::new()
-        .create(true)
-        .append(true)
-        .open(&log_filename)
-        .with_context(|| format!("failed to open {:?}", &log_filename))?;
-
     // Initialize logger
-    let logger_file = log_file.try_clone().unwrap();
-    let _scope_guard = init_logging(&conf, logger_file)?;
+    let (_scope_guard, log_file) = logger::init_logging(&conf, "pageserver.log")?;
     let _log_guard = slog_stdlog::init()?;
 
     // Note: this `info!(...)` macro comes from `log` crate
     info!("standard logging redirected to slog");
 
-    let tui_thread = if conf.interactive {
-        // Initialize the UI
-        Some(
-            thread::Builder::new()
-                .name("UI thread".into())
-                .spawn(|| {
-                    let _ = tui::ui_main();
-                })
-                .unwrap(),
-        )
-    } else {
-        None
-    };
-
     // TODO: Check that it looks like a valid repository before going further
 
     if conf.daemonize {
@@ -207,79 +301,45 @@ fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
         }
     }
 
-    // Check that we can bind to address before further initialization
-    info!("Starting pageserver on {}", conf.listen_addr);
-    let pageserver_listener = TcpListener::bind(conf.listen_addr)?;
+    // initialize authentication for incoming connections
+    let auth = match &conf.auth_type {
+        AuthType::Trust | AuthType::MD5 => None,
+        AuthType::ZenithJWT => {
+            // unwrap is ok because check is performed when creating config, so path is set and file exists
+            let key_path = conf.auth_validation_public_key_path.as_ref().unwrap();
+            Some(JwtAuth::from_key_path(key_path)?.into())
+        }
+    };
+    info!("Using auth: {:#?}", conf.auth_type);
 
-    // Initialize page cache, this will spawn walredo_thread
-    page_cache::init(conf);
+    // Spawn a new thread for the http endpoint
+    let cloned = auth.clone();
+    thread::Builder::new()
+        .name("http_endpoint_thread".into())
+        .spawn(move || {
+            let router = http::make_router(conf, cloned);
+            endpoint::serve_thread_main(router, conf.http_endpoint_addr.clone())
+        })?;
+
+    // Check that we can bind to address before starting threads to simplify shutdown
+    // sequence if port is occupied.
+    info!("Starting pageserver on {}", conf.listen_addr);
+    let pageserver_listener = TcpListener::bind(conf.listen_addr.clone())?;
+
+    // Initialize tenant manager.
+    tenant_mgr::init(conf);
 
     // Spawn a thread to listen for connections. It will spawn further threads
     // for each connection.
     let page_service_thread = thread::Builder::new()
         .name("Page Service thread".into())
-        .spawn(move || page_service::thread_main(conf, pageserver_listener))?;
+        .spawn(move || {
+            page_service::thread_main(conf, auth, pageserver_listener, conf.auth_type)
+        })?;
 
-    if let Some(tui_thread) = tui_thread {
-        // The TUI thread exits when the user asks to Quit.
-        tui_thread.join().unwrap();
-    } else {
-        page_service_thread
-            .join()
-            .expect("Page service thread has panicked")?
-    }
+    page_service_thread
+        .join()
+        .expect("Page service thread has panicked")?;
 
     Ok(())
 }
-
-fn init_logging(
-    conf: &PageServerConf,
-    log_file: File,
-) -> Result<slog_scope::GlobalLoggerGuard, io::Error> {
-    if conf.interactive {
-        Ok(tui::init_logging())
-    } else if conf.daemonize {
-        let decorator = slog_term::PlainSyncDecorator::new(log_file);
-        let drain = slog_term::FullFormat::new(decorator).build();
-        let drain = slog::Filter::new(drain, |record: &slog::Record| {
-            if record.level().is_at_least(slog::Level::Info) {
-                return true;
-            }
-            false
-        });
-        let drain = std::sync::Mutex::new(drain).fuse();
-        let logger = slog::Logger::root(
-            drain,
-            slog::o!(
-                "location" =>
-                FnValue(move |record| {
-                    format!("{}, {}:{}",
-                            record.module(),
-                            record.file(),
-                            record.line()
-                            )
-                    }
-                )
-            ),
-        );
-        Ok(slog_scope::set_global_logger(logger))
-    } else {
-        let decorator = slog_term::TermDecorator::new().build();
-        let drain = slog_term::FullFormat::new(decorator).build().fuse();
-        let drain = slog_async::Async::new(drain).chan_size(1000).build().fuse();
-        let drain = slog::Filter::new(drain, |record: &slog::Record| {
-            if record.level().is_at_least(slog::Level::Info) {
-                return true;
-            }
-            if record.level().is_at_least(slog::Level::Debug)
-                && record.module().starts_with("pageserver")
-            {
-                return true;
-            }
-            false
-        })
-        .fuse();
-        let logger = slog::Logger::root(drain, slog::o!());
-        Ok(slog_scope::set_global_logger(logger))
-    }
-}

pageserver/src/branches.rs

@@ -1,33 +1,34 @@
-//
-// Branch management code
-//
+//!
+//! Branch management code
+//!
 // TODO: move all paths construction to conf impl
 //
 
-use anyhow::{anyhow, bail, Context, Result};
-use bytes::Bytes;
-use fs::File;
-use postgres_ffi::{pg_constants, xlog_utils};
-use rand::Rng;
+use anyhow::{bail, ensure, Context, Result};
+use postgres_ffi::ControlFileData;
 use serde::{Deserialize, Serialize};
-use std::env;
-use std::io::{Read, Write};
 use std::{
-    collections::HashMap,
-    fs, io,
-    path::{Path, PathBuf},
+    fs,
+    path::Path,
     process::{Command, Stdio},
     str::FromStr,
+    sync::Arc,
 };
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
+
+use log::*;
 use zenith_utils::lsn::Lsn;
 
-use crate::page_cache;
+use crate::logger;
 use crate::restore_local_repo;
-use crate::{repository::Repository, PageServerConf, ZTimelineId};
+use crate::tenant_mgr;
+use crate::walredo::WalRedoManager;
+use crate::{repository::Repository, PageServerConf};
 
 #[derive(Serialize, Deserialize, Clone)]
 pub struct BranchInfo {
     pub name: String,
+    #[serde(with = "hex")]
     pub timeline_id: ZTimelineId,
     pub latest_valid_lsn: Option<Lsn>,
     pub ancestor_id: Option<String>,
@@ -40,32 +41,93 @@ pub struct PointInTime {
     pub lsn: Lsn,
 }
 
-pub fn init_repo(conf: &'static PageServerConf, repo_dir: &Path) -> Result<()> {
+pub fn init_pageserver(conf: &'static PageServerConf, create_tenant: Option<&str>) -> Result<()> {
+    // Initialize logger
+    let (_scope_guard, _log_file) = logger::init_logging(&conf, "pageserver.log")?;
+    let _log_guard = slog_stdlog::init()?;
+
+    // We don't use the real WAL redo manager, because we don't want to spawn the WAL redo
+    // process during repository initialization.
+    //
+    // FIXME: That caused trouble, because the WAL redo manager spawned a thread that launched
+    // initdb in the background, and it kept running even after the "zenith init" had exited.
+    // In tests, we started the  page server immediately after that, so that initdb was still
+    // running in the background, and we failed to run initdb again in the same directory. This
+    // has been solved for the rapid init+start case now, but the general race condition remains
+    // if you restart the server quickly. The WAL redo manager doesn't use a separate thread
+    // anymore, but I think that could still happen.
+    let dummy_redo_mgr = Arc::new(crate::walredo::DummyRedoManager {});
+
+    if let Some(tenantid) = create_tenant {
+        let tenantid = ZTenantId::from_str(tenantid)?;
+        println!("initializing tenantid {}", tenantid);
+        create_repo(conf, tenantid, dummy_redo_mgr).with_context(|| "failed to create repo")?;
+    }
+    fs::create_dir_all(conf.tenants_path())?;
+
+    println!("pageserver init succeeded");
+    Ok(())
+}
+
+pub fn create_repo(
+    conf: &'static PageServerConf,
+    tenantid: ZTenantId,
+    wal_redo_manager: Arc<dyn WalRedoManager + Send + Sync>,
+) -> Result<Arc<dyn Repository>> {
+    let repo_dir = conf.tenant_path(&tenantid);
+    if repo_dir.exists() {
+        bail!("repo for {} already exists", tenantid)
+    }
+
     // top-level dir may exist if we are creating it through CLI
-    fs::create_dir_all(repo_dir)
+    fs::create_dir_all(&repo_dir)
         .with_context(|| format!("could not create directory {}", repo_dir.display()))?;
 
-    env::set_current_dir(repo_dir)?;
+    // Note: this `info!(...)` macro comes from `log` crate
+    info!("standard logging redirected to slog");
 
-    fs::create_dir(std::path::Path::new("timelines"))?;
-    fs::create_dir(std::path::Path::new("refs"))?;
-    fs::create_dir(std::path::Path::new("refs").join("branches"))?;
-    fs::create_dir(std::path::Path::new("refs").join("tags"))?;
+    fs::create_dir(conf.timelines_path(&tenantid))?;
+    fs::create_dir_all(conf.branches_path(&tenantid))?;
+    fs::create_dir_all(conf.tags_path(&tenantid))?;
 
-    println!("created directory structure in {}", repo_dir.display());
+    info!("created directory structure in {}", repo_dir.display());
 
-    // Run initdb
-    //
-    // We create the cluster temporarily in a "tmp" directory inside the repository,
-    // and move it to the right location from there.
-    let tmppath = std::path::Path::new("tmp");
+    let tli = create_timeline(conf, None, &tenantid)?;
 
-    print!("running initdb... ");
-    io::stdout().flush()?;
+    let repo = Arc::new(crate::layered_repository::LayeredRepository::new(
+        conf,
+        wal_redo_manager,
+        tenantid,
+    ));
+
+    // Load data into pageserver
+    // TODO To implement zenith import we need to
+    //      move data loading out of create_repo()
+    bootstrap_timeline(conf, tenantid, tli, &*repo)?;
+
+    Ok(repo)
+}
+
+// Returns checkpoint LSN from controlfile
+fn get_lsn_from_controlfile(path: &Path) -> Result<Lsn> {
+    // Read control file to extract the LSN
+    let controlfile_path = path.join("global").join("pg_control");
+    let controlfile = ControlFileData::decode(&fs::read(controlfile_path)?)?;
+    let lsn = controlfile.checkPoint;
+
+    Ok(Lsn(lsn))
+}
+
+// Create the cluster temporarily in a initdbpath directory inside the repository
+// to get bootstrap data for timeline initialization.
+//
+fn run_initdb(conf: &'static PageServerConf, initdbpath: &Path) -> Result<()> {
+    info!("running initdb... ");
 
     let initdb_path = conf.pg_bin_dir().join("initdb");
-    let initdb_otput = Command::new(initdb_path)
-        .args(&["-D", tmppath.to_str().unwrap()])
+    let initdb_output = Command::new(initdb_path)
+        .args(&["-D", initdbpath.to_str().unwrap()])
+        .args(&["-U", &conf.superuser])
         .arg("--no-instructions")
         .env_clear()
         .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
@@ -73,82 +135,77 @@ pub fn init_repo(conf: &'static PageServerConf, repo_dir: &Path) -> Result<()> {
         .stdout(Stdio::null())
         .output()
         .with_context(|| "failed to execute initdb")?;
-    if !initdb_otput.status.success() {
-        anyhow::bail!("initdb failed");
+    if !initdb_output.status.success() {
+        anyhow::bail!(
+            "initdb failed: '{}'",
+            String::from_utf8_lossy(&initdb_output.stderr)
+        );
     }
-    println!("initdb succeeded");
-
-    // Read control file to extract the LSN and system id
-    let controlfile_path = tmppath.join("global").join("pg_control");
-    let controlfile = postgres_ffi::decode_pg_control(Bytes::from(fs::read(controlfile_path)?))?;
-    // let systemid = controlfile.system_identifier;
-    let lsn = controlfile.checkPoint;
-    let lsnstr = format!("{:016X}", lsn);
-
-    // Bootstrap the repository by loading the newly-initdb'd cluster into 'main' branch.
-    let tli = create_timeline(conf, None)?;
-    let timelinedir = conf.timeline_path(tli);
-
-    // We don't use page_cache here, because we don't want to spawn the WAL redo thread during
-    // repository initialization.
-    //
-    // FIXME: That caused trouble, because the WAL redo thread launched initdb in the background,
-    // and it kept running even after the "zenith init" had exited. In tests, we started the
-    // page server immediately after that, so that initdb was still running in the background,
-    // and we failed to run initdb again in the same directory. This has been solved for the
-    // rapid init+start case now, but the general race condition remains if you restart the
-    // server quickly.
-    let storage = crate::rocksdb_storage::RocksObjectStore::create(conf)?;
-
-    let repo = crate::object_repository::ObjectRepository::new(
-        conf,
-        std::sync::Arc::new(storage),
-        std::sync::Arc::new(crate::walredo::DummyRedoManager {}),
-    );
-    let timeline = repo.create_empty_timeline(tli, Lsn(lsn))?;
-
-    restore_local_repo::import_timeline_from_postgres_datadir(&tmppath, &*timeline, Lsn(lsn))?;
-
-    // Move the initial WAL file
-    fs::rename(
-        tmppath.join("pg_wal").join("000000010000000000000001"),
-        timelinedir
-            .join("wal")
-            .join("000000010000000000000001.partial"),
-    )?;
-    println!("created initial timeline {}", tli);
-
-    let data = tli.to_string();
-    fs::write(conf.branch_path("main"), data)?;
-    println!("created main branch");
-
-    // Remove pg_wal
-    fs::remove_dir_all(tmppath.join("pg_wal"))?;
-
-    force_crash_recovery(&tmppath)?;
-    println!("updated pg_control");
-
-    // Move the data directory as an initial base backup.
-    // FIXME: It would be enough to only copy the non-relation files here, the relation
-    // data was already loaded into the repository.
-    let target = timelinedir.join("snapshots").join(&lsnstr);
-    fs::rename(tmppath, &target)?;
-
-    println!(
-        "new zenith repository was created in {}",
-        repo_dir.display()
-    );
+    info!("initdb succeeded");
 
     Ok(())
 }
 
-pub(crate) fn get_branches(
-    conf: &PageServerConf,
-    repository: &dyn Repository,
-) -> Result<Vec<BranchInfo>> {
+//
+// - run initdb to init temporary instance and get bootstrap data
+// - after initialization complete, remove the temp dir.
+//
+fn bootstrap_timeline(
+    conf: &'static PageServerConf,
+    tenantid: ZTenantId,
+    tli: ZTimelineId,
+    repo: &dyn Repository,
+) -> Result<()> {
+    let initdb_path = conf.tenant_path(&tenantid).join("tmp");
+
+    // Init temporarily repo to get bootstrap data
+    run_initdb(conf, &initdb_path)?;
+    let pgdata_path = initdb_path;
+
+    let lsn = get_lsn_from_controlfile(&pgdata_path)?.align();
+
+    info!("bootstrap_timeline {:?} at lsn {}", pgdata_path, lsn);
+
+    let timeline = repo.create_empty_timeline(tli, lsn)?;
+    restore_local_repo::import_timeline_from_postgres_datadir(&pgdata_path, &*timeline, lsn)?;
+
+    let wal_dir = pgdata_path.join("pg_wal");
+    restore_local_repo::import_timeline_wal(&wal_dir, &*timeline, timeline.get_last_record_lsn())?;
+
+    println!(
+        "created initial timeline {} timeline.lsn {}",
+        tli,
+        timeline.get_last_record_lsn()
+    );
+
+    let data = tli.to_string();
+    fs::write(conf.branch_path("main", &tenantid), data)?;
+    println!("created main branch");
+
+    // Remove temp dir. We don't need it anymore
+    fs::remove_dir_all(pgdata_path)?;
+
+    Ok(())
+}
+
+pub(crate) fn get_tenants(conf: &PageServerConf) -> Result<Vec<String>> {
+    let tenants_dir = conf.tenants_path();
+
+    std::fs::read_dir(&tenants_dir)?
+        .map(|dir_entry_res| {
+            let dir_entry = dir_entry_res?;
+            ensure!(dir_entry.file_type()?.is_dir());
+            Ok(dir_entry.file_name().to_str().unwrap().to_owned())
+        })
+        .collect()
+}
+
+pub(crate) fn get_branches(conf: &PageServerConf, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
+    let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
+
     // Each branch has a corresponding record (text file) in the refs/branches
     // with timeline_id.
-    let branches_dir = std::path::Path::new("refs").join("branches");
+    let branches_dir = conf.branches_path(tenantid);
 
     std::fs::read_dir(&branches_dir)?
         .map(|dir_entry_res| {
@@ -156,12 +213,12 @@ pub(crate) fn get_branches(
             let name = dir_entry.file_name().to_str().unwrap().to_string();
             let timeline_id = std::fs::read_to_string(dir_entry.path())?.parse::<ZTimelineId>()?;
 
-            let latest_valid_lsn = repository
+            let latest_valid_lsn = repo
                 .get_timeline(timeline_id)
-                .map(|timeline| timeline.get_last_valid_lsn())
+                .map(|timeline| timeline.get_last_record_lsn())
                 .ok();
 
-            let ancestor_path = conf.ancestor_path(timeline_id);
+            let ancestor_path = conf.ancestor_path(&timeline_id, tenantid);
             let mut ancestor_id: Option<String> = None;
             let mut ancestor_lsn: Option<String> = None;
 
@@ -194,74 +251,40 @@ pub(crate) fn get_branches(
         .collect()
 }
 
-pub(crate) fn get_system_id(conf: &PageServerConf) -> Result<u64> {
-    // let branches = get_branches();
-
-    let branches_dir = std::path::Path::new("refs").join("branches");
-    let branches = std::fs::read_dir(&branches_dir)?
-        .map(|dir_entry_res| {
-            let dir_entry = dir_entry_res?;
-            let name = dir_entry.file_name().to_str().unwrap().to_string();
-            let timeline_id = std::fs::read_to_string(dir_entry.path())?.parse::<ZTimelineId>()?;
-            Ok((name, timeline_id))
-        })
-        .collect::<Result<HashMap<String, ZTimelineId>>>()?;
-
-    let main_tli = branches
-        .get("main")
-        .ok_or_else(|| anyhow!("Branch main not found"))?;
-
-    let (_, main_snap_dir) = find_latest_snapshot(conf, *main_tli)?;
-    let controlfile_path = main_snap_dir.join("global").join("pg_control");
-    let controlfile = postgres_ffi::decode_pg_control(Bytes::from(fs::read(controlfile_path)?))?;
-    Ok(controlfile.system_identifier)
-}
-
 pub(crate) fn create_branch(
     conf: &PageServerConf,
     branchname: &str,
     startpoint_str: &str,
+    tenantid: &ZTenantId,
 ) -> Result<BranchInfo> {
-    if conf.branch_path(&branchname).exists() {
+    let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
+
+    if conf.branch_path(branchname, tenantid).exists() {
         anyhow::bail!("branch {} already exists", branchname);
     }
 
-    let mut startpoint = parse_point_in_time(conf, startpoint_str)?;
+    let mut startpoint = parse_point_in_time(conf, startpoint_str, tenantid)?;
 
     if startpoint.lsn == Lsn(0) {
         // Find end of WAL on the old timeline
-        let end_of_wal = find_end_of_wal(conf, startpoint.timelineid)?;
-        println!("branching at end of WAL: {}", end_of_wal);
+        let end_of_wal = repo
+            .get_timeline(startpoint.timelineid)?
+            .get_last_record_lsn();
+        info!("branching at end of WAL: {}", end_of_wal);
         startpoint.lsn = end_of_wal;
     }
 
     // create a new timeline directory for it
-    let newtli = create_timeline(conf, Some(startpoint))?;
-    let newtimelinedir = conf.timeline_path(newtli);
+    let newtli = create_timeline(conf, Some(startpoint), tenantid)?;
 
     // Let the Repository backend do its initialization
-    let repo = page_cache::get_repository();
     repo.branch_timeline(startpoint.timelineid, newtli, startpoint.lsn)?;
 
-    // Copy the latest snapshot (TODO: before the startpoint) and all WAL
-    // TODO: be smarter and avoid the copying...
-    let (_maxsnapshot, oldsnapshotdir) = find_latest_snapshot(conf, startpoint.timelineid)?;
-    let copy_opts = fs_extra::dir::CopyOptions::new();
-    fs_extra::dir::copy(oldsnapshotdir, newtimelinedir.join("snapshots"), &copy_opts)?;
-
-    let oldtimelinedir = conf.timeline_path(startpoint.timelineid);
-    copy_wal(
-        &oldtimelinedir.join("wal"),
-        &newtimelinedir.join("wal"),
-        startpoint.lsn,
-        pg_constants::WAL_SEGMENT_SIZE,
-    )?;
-
     // Remember the human-readable branch name for the new timeline.
     // FIXME: there's a race condition, if you create a branch with the same
     // name concurrently.
     let data = newtli.to_string();
-    fs::write(conf.branch_path(&branchname), data)?;
+    fs::write(conf.branch_path(&branchname, tenantid), data)?;
 
     Ok(BranchInfo {
         name: branchname.to_string(),
@@ -291,7 +314,11 @@ pub(crate) fn create_branch(
 //    mytag
 //
 //
-fn parse_point_in_time(conf: &PageServerConf, s: &str) -> Result<PointInTime> {
+fn parse_point_in_time(
+    conf: &PageServerConf,
+    s: &str,
+    tenantid: &ZTenantId,
+) -> Result<PointInTime> {
     let mut strings = s.split('@');
     let name = strings.next().unwrap();
 
@@ -306,21 +333,21 @@ fn parse_point_in_time(conf: &PageServerConf, s: &str) -> Result<PointInTime> {
 
     // Check if it's a tag
     if lsn.is_none() {
-        let tagpath = conf.tag_path(name);
+        let tagpath = conf.tag_path(name, &tenantid);
         if tagpath.exists() {
             let pointstr = fs::read_to_string(tagpath)?;
 
-            return parse_point_in_time(conf, &pointstr);
+            return parse_point_in_time(conf, &pointstr, &tenantid);
         }
     }
 
     // Check if it's a branch
     // Check if it's branch @ LSN
-    let branchpath = conf.branch_path(name);
+    let branchpath = conf.branch_path(name, &tenantid);
     if branchpath.exists() {
         let pointstr = fs::read_to_string(branchpath)?;
 
-        let mut result = parse_point_in_time(conf, &pointstr)?;
+        let mut result = parse_point_in_time(conf, &pointstr, &tenantid)?;
 
         result.lsn = lsn.unwrap_or(Lsn(0));
         return Ok(result);
@@ -329,7 +356,7 @@ fn parse_point_in_time(conf: &PageServerConf, s: &str) -> Result<PointInTime> {
     // Check if it's a timelineid
     // Check if it's timelineid @ LSN
     if let Ok(timelineid) = ZTimelineId::from_str(name) {
-        let tlipath = conf.timeline_path(timelineid);
+        let tlipath = conf.timeline_path(&timelineid, &tenantid);
         if tlipath.exists() {
             return Ok(PointInTime {
                 timelineid,
@@ -341,41 +368,18 @@ fn parse_point_in_time(conf: &PageServerConf, s: &str) -> Result<PointInTime> {
     bail!("could not parse point-in-time {}", s);
 }
 
-// If control file says the cluster was shut down cleanly, modify it, to mark
-// it as crashed. That forces crash recovery when you start the cluster.
-//
-// FIXME:
-// We currently do this to the initial snapshot in "zenith init". It would
-// be more natural to do this when the snapshot is restored instead, but we
-// currently don't have any code to create new snapshots, so it doesn't matter
-// Or better yet, use a less hacky way of putting the cluster into recovery.
-// Perhaps create a backup label file in the data directory when it's restored.
-fn force_crash_recovery(datadir: &Path) -> Result<()> {
-    // Read in the control file
-    let controlfilepath = datadir.to_path_buf().join("global").join("pg_control");
-    let mut controlfile =
-        postgres_ffi::decode_pg_control(Bytes::from(fs::read(controlfilepath.as_path())?))?;
-
-    controlfile.state = postgres_ffi::DBState_DB_IN_PRODUCTION;
-
-    fs::write(
-        controlfilepath.as_path(),
-        postgres_ffi::encode_pg_control(controlfile),
-    )?;
-
-    Ok(())
-}
-
-fn create_timeline(conf: &PageServerConf, ancestor: Option<PointInTime>) -> Result<ZTimelineId> {
+fn create_timeline(
+    conf: &PageServerConf,
+    ancestor: Option<PointInTime>,
+    tenantid: &ZTenantId,
+) -> Result<ZTimelineId> {
     // Create initial timeline
-    let mut tli_buf = [0u8; 16];
-    rand::thread_rng().fill(&mut tli_buf);
-    let timelineid = ZTimelineId::from(tli_buf);
 
-    let timelinedir = conf.timeline_path(timelineid);
+    let timelineid = ZTimelineId::generate();
+
+    let timelinedir = conf.timeline_path(&timelineid, tenantid);
 
     fs::create_dir(&timelinedir)?;
-    fs::create_dir(&timelinedir.join("snapshots"))?;
     fs::create_dir(&timelinedir.join("wal"))?;
 
     if let Some(ancestor) = ancestor {
@@ -385,79 +389,3 @@ fn create_timeline(conf: &PageServerConf, ancestor: Option<PointInTime>) -> Resu
 
     Ok(timelineid)
 }
-
-///
-/// Copy all WAL segments from one directory to another, up to given LSN.
-///
-/// If the given LSN is in the middle of a segment, the last segment containing it
-/// is written out as .partial, and padded with zeros.
-///
-fn copy_wal(src_dir: &Path, dst_dir: &Path, upto: Lsn, wal_seg_size: usize) -> Result<()> {
-    let last_segno = upto.segment_number(wal_seg_size);
-    let last_segoff = upto.segment_offset(wal_seg_size);
-
-    for entry in fs::read_dir(src_dir).unwrap().flatten() {
-        let entry_name = entry.file_name();
-        let fname = entry_name.to_str().unwrap();
-
-        // Check if the filename looks like an xlog file, or a .partial file.
-        if !xlog_utils::IsXLogFileName(fname) && !xlog_utils::IsPartialXLogFileName(fname) {
-            continue;
-        }
-        let (segno, _tli) = xlog_utils::XLogFromFileName(fname, wal_seg_size as usize);
-
-        let copylen;
-        let mut dst_fname = PathBuf::from(fname);
-        if segno > last_segno {
-            // future segment, skip
-            continue;
-        } else if segno < last_segno {
-            copylen = wal_seg_size;
-            dst_fname.set_extension("");
-        } else {
-            copylen = last_segoff;
-            dst_fname.set_extension("partial");
-        }
-
-        let src_file = File::open(entry.path())?;
-        let mut dst_file = File::create(dst_dir.join(&dst_fname))?;
-        std::io::copy(&mut src_file.take(copylen as u64), &mut dst_file)?;
-
-        if copylen < wal_seg_size {
-            std::io::copy(
-                &mut std::io::repeat(0).take((wal_seg_size - copylen) as u64),
-                &mut dst_file,
-            )?;
-        }
-    }
-    Ok(())
-}
-
-// Find the end of valid WAL in a wal directory
-pub fn find_end_of_wal(conf: &PageServerConf, timeline: ZTimelineId) -> Result<Lsn> {
-    let waldir = conf.timeline_path(timeline).join("wal");
-    let (lsn, _tli) = xlog_utils::find_end_of_wal(&waldir, pg_constants::WAL_SEGMENT_SIZE, true);
-    Ok(Lsn(lsn))
-}
-
-// Find the latest snapshot for a timeline
-fn find_latest_snapshot(conf: &PageServerConf, timeline: ZTimelineId) -> Result<(Lsn, PathBuf)> {
-    let snapshotsdir = conf.snapshots_path(timeline);
-    let paths = fs::read_dir(&snapshotsdir)?;
-    let mut maxsnapshot = Lsn(0);
-    let mut snapshotdir: Option<PathBuf> = None;
-    for path in paths {
-        let path = path?;
-        let filename = path.file_name().to_str().unwrap().to_owned();
-        if let Ok(lsn) = Lsn::from_hex(&filename) {
-            maxsnapshot = std::cmp::max(lsn, maxsnapshot);
-            snapshotdir = Some(path.path());
-        }
-    }
-    if maxsnapshot == Lsn(0) {
-        // TODO: check ancestor timeline
-        anyhow::bail!("no snapshot found in {}", snapshotsdir.display());
-    }
-
-    Ok((maxsnapshot, snapshotdir.unwrap()))
-}

pageserver/src/http/mod.rs

@@ -0,0 +1,3 @@
+pub mod models;
+pub mod routes;
+pub use routes::make_router;

pageserver/src/http/models.rs

@@ -0,0 +1,17 @@
+use serde::{Deserialize, Serialize};
+
+use crate::ZTenantId;
+
+#[derive(Serialize, Deserialize)]
+pub struct BranchCreateRequest {
+    #[serde(with = "hex")]
+    pub tenant_id: ZTenantId,
+    pub name: String,
+    pub start_point: String,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct TenantCreateRequest {
+    #[serde(with = "hex")]
+    pub tenant_id: ZTenantId,
+}

pageserver/src/http/openapi_spec.yml

@@ -0,0 +1,239 @@
+openapi: "3.0.2"
+info:
+  title: Page Server API
+  version: "1.0"
+servers:
+  - url: ""
+paths:
+  /v1/status:
+    description: Healthcheck endpoint
+    get:
+      description: Healthcheck
+      security: []
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+  /v1/branch/{tenant_id}:
+    parameters:
+      - name: tenant_id
+        in: path
+        required: true
+        schema:
+          type: string
+          format: hex
+    get:
+      description: Get branches for tenant
+      responses:
+        "200":
+          description: BranchInfo
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/BranchInfo"
+        "400":
+          description: Error when no tenant id found in path
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+  /v1/branch/:
+    post:
+      description: Create branch
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - "tenant_id"
+                - "name"
+                - "start_point"
+              properties:
+                tenant_id:
+                  type: string
+                  format: hex
+                name:
+                  type: string
+                start_point:
+                  type: string
+      responses:
+        "201":
+          description: BranchInfo
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/BranchInfo"
+        "400":
+          description: Malformed branch create request
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+  /v1/tenant/:
+    get:
+      description: Get tenants list
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+    post:
+      description: Create tenant
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - "tenant_id"
+              properties:
+                tenant_id:
+                  type: string
+                  format: hex
+      responses:
+        "201":
+          description: CREATED
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+        "400":
+          description: Malformed tenant create request
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+
+components:
+  securitySchemes:
+    JWT:
+      type: http
+      scheme: bearer
+      bearerFormat: JWT
+  schemas:
+    BranchInfo:
+      type: object
+      required:
+        - name
+        - timeline_id
+      properties:
+        name:
+          type: string
+        timeline_id:
+          type: string
+          format: hex
+        ancestor_id:
+          type: string
+        ancestor_lsn:
+          type: string
+    Error:
+      type: object
+      required:
+        - msg
+      properties:
+        msg:
+          type: string
+    UnauthorizedError:
+      type: object
+      required:
+        - msg
+      properties:
+        msg:
+          type: string
+    ForbiddenError:
+      type: object
+      required:
+        - msg
+      properties:
+        msg:
+          type: string
+
+security:
+  - JWT: []

pageserver/src/http/routes.rs

@@ -0,0 +1,166 @@
+use std::sync::Arc;
+
+use anyhow::Result;
+use hyper::header;
+use hyper::StatusCode;
+use hyper::{Body, Request, Response, Uri};
+use routerify::{ext::RequestExt, RouterBuilder};
+use zenith_utils::auth::JwtAuth;
+use zenith_utils::http::endpoint::attach_openapi_ui;
+use zenith_utils::http::endpoint::auth_middleware;
+use zenith_utils::http::endpoint::check_permission;
+use zenith_utils::http::error::ApiError;
+use zenith_utils::http::{
+    endpoint,
+    error::HttpErrorBody,
+    json::{json_request, json_response},
+};
+
+use super::models::BranchCreateRequest;
+use super::models::TenantCreateRequest;
+use crate::{
+    branches::{self},
+    tenant_mgr, PageServerConf, ZTenantId,
+};
+
+#[derive(Debug)]
+struct State {
+    conf: &'static PageServerConf,
+    auth: Option<Arc<JwtAuth>>,
+    allowlist_routes: Vec<Uri>,
+}
+
+impl State {
+    fn new(conf: &'static PageServerConf, auth: Option<Arc<JwtAuth>>) -> Self {
+        let allowlist_routes = ["/v1/status", "/v1/doc", "/swagger.yml"]
+            .iter()
+            .map(|v| v.parse().unwrap())
+            .collect::<Vec<_>>();
+        Self {
+            conf,
+            auth,
+            allowlist_routes,
+        }
+    }
+}
+
+#[inline(always)]
+fn get_state(request: &Request<Body>) -> &State {
+    request
+        .data::<Arc<State>>()
+        .expect("unknown state type")
+        .as_ref()
+}
+
+#[inline(always)]
+fn get_config(request: &Request<Body>) -> &'static PageServerConf {
+    get_state(request).conf
+}
+
+// healthcheck handler
+async fn status_handler(_: Request<Body>) -> Result<Response<Body>, ApiError> {
+    Ok(Response::builder()
+        .status(StatusCode::OK)
+        .header(header::CONTENT_TYPE, "application/json")
+        .body(Body::from("{}"))
+        .map_err(ApiError::from_err)?)
+}
+
+async fn branch_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let request_data: BranchCreateRequest = json_request(&mut request).await?;
+
+    check_permission(&request, Some(request_data.tenant_id))?;
+
+    let response_data = tokio::task::spawn_blocking(move || {
+        branches::create_branch(
+            get_config(&request),
+            &request_data.name,
+            &request_data.start_point,
+            &request_data.tenant_id,
+        )
+    })
+    .await
+    .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::CREATED, response_data)?)
+}
+
+async fn branch_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let tenantid: ZTenantId = match request.param("tenant_id") {
+        Some(arg) => arg
+            .parse()
+            .map_err(|_| ApiError::BadRequest("failed to parse tenant id".to_string()))?,
+        None => {
+            return Err(ApiError::BadRequest(
+                "no tenant id specified in path param".to_string(),
+            ))
+        }
+    };
+
+    check_permission(&request, Some(tenantid))?;
+
+    let response_data = tokio::task::spawn_blocking(move || {
+        crate::branches::get_branches(get_config(&request), &tenantid)
+    })
+    .await
+    .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::OK, response_data)?)
+}
+
+async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    // check for management permission
+    check_permission(&request, None)?;
+
+    let response_data =
+        tokio::task::spawn_blocking(move || crate::branches::get_tenants(get_config(&request)))
+            .await
+            .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::OK, response_data)?)
+}
+
+async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    // check for management permission
+    check_permission(&request, None)?;
+
+    let request_data: TenantCreateRequest = json_request(&mut request).await?;
+
+    let response_data = tokio::task::spawn_blocking(move || {
+        tenant_mgr::create_repository_for_tenant(get_config(&request), request_data.tenant_id)
+    })
+    .await
+    .map_err(ApiError::from_err)??;
+    Ok(json_response(StatusCode::CREATED, response_data)?)
+}
+
+async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
+    json_response(
+        StatusCode::NOT_FOUND,
+        HttpErrorBody::from_msg("page not found".to_owned()),
+    )
+}
+
+pub fn make_router(
+    conf: &'static PageServerConf,
+    auth: Option<Arc<JwtAuth>>,
+) -> RouterBuilder<hyper::Body, ApiError> {
+    let spec = include_bytes!("openapi_spec.yml");
+    let mut router = attach_openapi_ui(endpoint::make_router(), spec, "/swagger.yml", "/v1/doc");
+    if auth.is_some() {
+        router = router.middleware(auth_middleware(|request| {
+            let state = get_state(request);
+            if state.allowlist_routes.contains(request.uri()) {
+                None
+            } else {
+                state.auth.as_deref()
+            }
+        }))
+    }
+
+    router
+        .data(Arc::new(State::new(conf, auth)))
+        .get("/v1/status", status_handler)
+        .get("/v1/branch/:tenant_id", branch_list_handler)
+        .post("/v1/branch", branch_create_handler)
+        .get("/v1/tenant", tenant_list_handler)
+        .post("/v1/tenant", tenant_create_handler)
+        .any(handler_404)
+}

pageserver/src/layered_repository/README.md

@@ -0,0 +1,366 @@
+# Overview
+
+The on-disk format is based on immutable files. The page server
+receives a stream of incoming WAL, parses the WAL records to determine
+which pages they apply to, and accumulates the incoming changes in
+memory. Every now and then, the accumulated changes are written out to
+new files.
+
+The files are called "layer files". Each layer file corresponds
+to one RELISH_SEG_SIZE slice of a PostgreSQL relation fork or
+non-rel file in a range of LSNs. The layer files
+for each timeline are stored in the timeline's subdirectory under
+.zenith/tenants/<tenantid>/timelines.
+
+There are two kind of layer file: base images, and deltas. A base
+image file contains a layer of a segment as it was at one LSN,
+whereas a delta file contains modifications to a segment - mostly in
+the form of WAL records - in a range of LSN
+
+base image file:
+
+    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>
+
+delta file:
+
+    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>_<end LSN>
+
+For example:
+
+    rel_1663_13990_2609_0_10_000000000169C348
+    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000
+
+In addition to the relations, with "rel_*" prefix, we use the same
+format for storing various smaller files from the PostgreSQL data
+directory. They will use different suffixes and the naming scheme up
+to the LSNs vary. The Zenith source code uses the term "relish" to
+mean "a relation, or other file that's treated like a relation in the
+storage" For example, a base image of a CLOG segment would be named
+like this:
+
+    pg_xact_0000_0_00000000198B06B0
+
+There is no difference in how the relation and non-relation files are
+managed, except that the first part of file names is different.
+Internally, the relations and non-relation files that are managed in
+the versioned store are together called "relishes".
+
+If a file has been dropped, the last layer file for it is created
+with the _DROPPED suffix, e.g.
+
+    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000_DROPPED
+
+
+## Notation used in this document
+
+The full path of a delta file looks like this:
+
+    .zenith/tenants/941ddc8604413b88b3d208bddf90396c/timelines/4af489b06af8eed9e27a841775616962/rel_1663_13990_2609_0_10_000000000169C348_0000000001702000
+
+For simplicity, the examples below use a simplified notation for the
+paths.  The tenant ID is left out, the timeline ID is replaced with
+the human-readable branch name, and spcnode+dbnode+relnode+forkum+segno
+with a human-readable table name. The LSNs are also shorter. For
+example, a base image file at LSN 100 and a delta file between 100-200
+for 'orders' table on 'main' branch is represented like this:
+
+    main/orders_100
+    main/orders_100_200
+
+
+# Creating layer files
+
+Let's start with a simple example with a system that contains one
+branch called 'main' and two tables, 'orders' and 'customers'. The end
+of WAL is currently at LSN 250. In this starting situation, you would
+have these files on disk:
+
+	main/orders_100
+	main/orders_100_200
+	main/orders_200
+	main/customers_100
+	main/customers_100_200
+	main/customers_200
+
+In addition to those files, the recent changes between LSN 200 and the
+end of WAL at 250 are kept in memory. If the page server crashes, the
+latest records between 200-250 need to be re-read from the WAL.
+
+Whenever enough WAL has been accumulated in memory, the page server
+writes out the changes in memory into new layer files. This process
+is called "checkpointing" (not to be confused with the PostgreSQL
+checkpoints, that's a different thing). The page server only creates
+layer files for relations that have been modified since the last
+checkpoint. For example, if the current end of WAL is at LSN 450, and
+the last checkpoint happened at LSN 400 but there hasn't been any
+recent changes to 'customers' table, you would have these files on
+disk:
+
+	main/orders_100
+	main/orders_100_200
+	main/orders_200
+	main/orders_200_300
+	main/orders_300
+	main/orders_300_400
+	main/orders_400
+	main/customers_100
+	main/customers_100_200
+	main/customers_200
+
+If the customers table is modified later, a new file is created for it
+at the next checkpoint. The new file will cover the "gap" from the
+last layer file, so the LSN ranges are always contiguous:
+
+	main/orders_100
+	main/orders_100_200
+	main/orders_200
+	main/orders_200_300
+	main/orders_300
+	main/orders_300_400
+	main/orders_400
+	main/customers_100
+	main/customers_100_200
+	main/customers_200
+	main/customers_200_500
+	main/customers_500
+
+## Reading page versions
+
+Whenever a GetPage@LSN request comes in from the compute node, the
+page server needs to reconstruct the requested page, as it was at the
+requested LSN. To do that, the page server first checks the recent
+in-memory layer; if the requested page version is found there, it can
+be returned immediatedly without looking at the files on
+disk. Otherwise the page server needs to locate the layer file that
+contains the requested page version.
+
+For example, if a request comes in for table 'orders' at LSN 250, the
+page server would load the 'main/orders_200_300' file into memory, and
+reconstruct and return the requested page from it, as it was at
+LSN 250. Because the layer file consists of a full image of the
+relation at the start LSN and the WAL, reconstructing the page
+involves replaying any WAL records applicable to the page between LSNs
+200-250, starting from the base image at LSN 200.
+
+# Multiple branches
+
+Imagine that a child branch is created at LSN 250:
+
+            @250
+    ----main--+-------------------------->
+               \
+                +---child-------------->
+
+
+Then, the 'orders' table is updated differently on the 'main' and
+'child' branches. You now have this situation on disk:
+
+    main/orders_100
+    main/orders_100_200
+    main/orders_200
+    main/orders_200_300
+    main/orders_300
+    main/orders_300_400
+    main/orders_400
+    main/customers_100
+    main/customers_100_200
+    main/customers_200
+    child/orders_250_300
+    child/orders_300
+    child/orders_300_400
+    child/orders_400
+
+Because the 'customers' table hasn't been modified on the child
+branch, there is no file for it there. If you request a page for it on
+the 'child' branch, the page server will not find any layer file
+for it in the 'child' directory, so it will recurse to look into the
+parent 'main' branch instead.
+
+From the 'child' branch's point of view, the history for each relation
+is linear, and the request's LSN identifies unambiguously which file
+you need to look at. For example, the history for the 'orders' table
+on the 'main' branch consists of these files:
+
+    main/orders_100
+    main/orders_100_200
+    main/orders_200
+    main/orders_200_300
+    main/orders_300
+    main/orders_300_400
+    main/orders_400
+
+And from the 'child' branch's point of view, it consists of these
+files:
+
+    main/orders_100
+    main/orders_100_200
+    main/orders_200
+    main/orders_200_300
+    child/orders_250_300
+    child/orders_300
+    child/orders_300_400
+    child/orders_400
+
+The branch metadata includes the point where the child branch was
+created, LSN 250. If a page request comes with LSN 275, we read the
+page version from the 'child/orders_250_300' file. We might also
+need to reconstruct the page version as it was at LSN 250, in order
+to replay the WAL up to LSN 275, using 'main/orders_200_300' and
+'main/orders_200'. The page versions between 250-300 in the
+'main/orders_200_300' file are ignored when operating on the child
+branch.
+
+Note: It doesn't make any difference if the child branch is created
+when the end of the main branch was at LSN 250, or later when the tip of
+the main branch had already moved on. The latter case, creating a
+branch at a historic LSN, is how we support PITR in Zenith.
+
+
+# Garbage collection
+
+In this scheme, we keep creating new layer files over time. We also
+need a mechanism to remove old files that are no longer needed,
+because disk space isn't infinite.
+
+What files are still needed? Currently, the page server supports PITR
+and branching from any branch at any LSN that is "recent enough" from
+the tip of the branch.  "Recent enough" is defined as an LSN horizon,
+which by default is 64 MB.  (See DEFAULT_GC_HORIZON). For this
+example, let's assume that the LSN horizon is 150 units.
+
+Let's look at the single branch scenario again. Imagine that the end
+of the branch is LSN 525, so that the GC horizon is currently at
+525-150 = 375
+
+	main/orders_100
+	main/orders_100_200
+	main/orders_200
+	main/orders_200_300
+	main/orders_300
+	main/orders_300_400
+	main/orders_400
+	main/orders_400_500
+	main/orders_500
+	main/customers_100
+	main/customers_100_200
+	main/customers_200
+
+We can remove the following files because the end LSNs of those files are
+older than GC horizon 375, and there are more recent layer files for the
+table:
+
+	main/orders_100       DELETE
+	main/orders_100_200   DELETE
+	main/orders_200       DELETE
+	main/orders_200_300   DELETE
+	main/orders_300       STILL NEEDED BY orders_300_400
+	main/orders_300_400   KEEP, NEWER THAN GC HORIZON
+	main/orders_400       .. 
+	main/orders_400_500   .. 
+	main/orders_500       .. 
+	main/customers_100      DELETE
+	main/customers_100_200  DELETE
+	main/customers_200      KEEP, NO NEWER VERSION
+
+'main/customers_100_200' is old enough, but it cannot be
+removed because there is no newer layer file for the table.
+
+Things get slightly more complicated with multiple branches. All of
+the above still holds, but in addition to recent files we must also
+retain older shapshot files that are still needed by child branches.
+For example, if child branch is created at LSN 150, and the 'customers'
+table is updated on the branch, you would have these files:
+
+	main/orders_100        KEEP, NEEDED BY child BRANCH
+	main/orders_100_200    KEEP, NEEDED BY child BRANCH
+	main/orders_200        DELETE
+	main/orders_200_300    DELETE
+	main/orders_300        KEEP, NEWER THAN GC HORIZON
+	main/orders_300_400    KEEP, NEWER THAN GC HORIZON
+	main/orders_400        KEEP, NEWER THAN GC HORIZON
+	main/orders_400_500    KEEP, NEWER THAN GC HORIZON
+	main/orders_500        KEEP, NEWER THAN GC HORIZON
+	main/customers_100       DELETE
+	main/customers_100_200   DELETE
+	main/customers_200       KEEP, NO NEWER VERSION
+	child/customers_150_300  DELETE
+	child/customers_300      KEEP, NO NEWER VERSION
+
+In this situation, 'main/orders_100' and 'main/orders_100_200' cannot
+be removed, even though they are older than the GC horizon, because
+they are still needed by the child branch. 'main/orders_200'
+and 'main/orders_200_300' can still be removed.
+
+If 'orders' is modified later on the 'child' branch, we will create a
+new base image and delta file for it on the child:
+
+	main/orders_100
+	main/orders_100_200
+
+	main/orders_300
+	main/orders_300_400
+	main/orders_400
+	main/orders_400_500
+	main/orders_500
+	main/customers_200
+	child/customers_300
+	child/orders_150_400
+	child/orders_400
+
+After this, the 'main/orders_100' and 'main/orders_100_200' file could
+be removed. It is no longer needed by the child branch, because there
+is a newer layer file there. TODO: This optimization hasn't been
+implemented! The GC algorithm will currently keep the file on the
+'main' branch anyway, for as long as the child branch exists.
+
+
+# TODO: On LSN ranges
+
+In principle, each relation can be checkpointed separately, i.e. the
+LSN ranges of the files don't need to line up. So this would be legal:
+
+	main/orders_100
+	main/orders_100_200
+	main/orders_200
+	main/orders_200_300
+	main/orders_300
+	main/orders_300_400
+	main/orders_400
+	main/customers_150
+	main/customers_150_250
+	main/customers_250
+	main/customers_250_500
+	main/customers_500
+
+However, the code currently always checkpoints all relations together.
+So that situation doesn't arise in practice.
+
+It would also be OK to have overlapping LSN ranges for the same relation:
+
+	main/orders_100
+	main/orders_100_200
+	main/orders_200
+	main/orders_200_300
+	main/orders_300
+	main/orders_250_350
+	main/orders_350
+	main/orders_300_400
+	main/orders_400
+
+The code that reads the layer files should cope with this, but this
+situation doesn't arise either, because the checkpointing code never
+does that.  It could be useful, however, as a transient state when
+garbage collecting around branch points, or explicit recovery
+points. For example, if we start with this:
+
+	main/orders_100
+	main/orders_100_200
+	main/orders_200
+	main/orders_200_300
+	main/orders_300
+
+And there is a branch or explicit recovery point at LSN 150, we could
+replace 'main/orders_100_200' with 'main/orders_150' to keep a
+layer only at that exact point that's still needed, removing the
+other page versions around it. But such compaction has not been
+implemented yet.

pageserver/src/layered_repository/blob.rs

@@ -0,0 +1,45 @@
+use std::{fs::File, io::Write};
+
+use anyhow::Result;
+use bookfile::{BookWriter, BoundedReader, ChapterId, ChapterWriter};
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+pub struct BlobRange {
+    offset: u64,
+    size: usize,
+}
+
+pub fn read_blob(reader: &BoundedReader<&'_ File>, range: &BlobRange) -> Result<Vec<u8>> {
+    let mut buf = vec![0u8; range.size];
+    reader.read_exact_at(&mut buf, range.offset)?;
+    Ok(buf)
+}
+
+pub struct BlobWriter {
+    writer: ChapterWriter<File>,
+    offset: u64,
+}
+
+impl BlobWriter {
+    // This function takes a BookWriter and creates a new chapter to ensure offset is 0.
+    pub fn new(book_writer: BookWriter<File>, chapter_id: impl Into<ChapterId>) -> Self {
+        let writer = book_writer.new_chapter(chapter_id);
+        Self { writer, offset: 0 }
+    }
+
+    pub fn write_blob(&mut self, blob: &[u8]) -> Result<BlobRange> {
+        self.writer.write_all(blob)?;
+
+        let range = BlobRange {
+            offset: self.offset,
+            size: blob.len(),
+        };
+        self.offset += blob.len() as u64;
+        Ok(range)
+    }
+
+    pub fn close(self) -> bookfile::Result<BookWriter<File>> {
+        self.writer.close()
+    }
+}

pageserver/src/layered_repository/delta_layer.rs

@@ -0,0 +1,549 @@
+//!
+//! A DeltaLayer represents a collection of WAL records or page images in a range of
+//! LSNs, for one segment. It is stored on a file on disk.
+//!
+//! Usually a delta layer only contains differences - in the form of WAL records against
+//! a base LSN. However, if a segment is newly created, by creating a new relation or
+//! extending an old one, there might be no base image. In that case, all the entries in
+//! the delta layer must be page images or WAL records with the 'will_init' flag set, so
+//! that they can be replayed without referring to an older page version. Also in some
+//! circumstances, the predecessor layer might actually be another delta layer. That
+//! can happen when you create a new branch in the middle of a delta layer, and the WAL
+//! records on the new branch are put in a new delta layer.
+//!
+//! When a delta file needs to be accessed, we slurp the metadata and relsize chapters
+//! into memory, into the DeltaLayerInner struct. See load() and unload() functions.
+//! To access a page/WAL record, we search `page_version_metas` for the block # and LSN.
+//! The byte ranges in the metadata can be used to find the page/WAL record in
+//! PAGE_VERSIONS_CHAPTER.
+//!
+//! On disk, the delta files are stored in timelines/<timelineid> directory.
+//! Currently, there are no subdirectories, and each delta file is named like this:
+//!
+//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>_<end LSN>
+//!
+//! For example:
+//!
+//!    1663_13990_2609_0_5_000000000169C348_000000000169C349
+//!
+//! If a relation is dropped, we add a '_DROPPED' to the end of the filename to indicate that.
+//! So the above example would become:
+//!
+//!    1663_13990_2609_0_5_000000000169C348_000000000169C349_DROPPED
+//!
+//! The end LSN indicates when it was dropped in that case, we don't store it in the
+//! file contents in any way.
+//!
+//! A detlta file is constructed using the 'bookfile' crate. Each file consists of two
+//! parts: the page versions and the relation sizes. They are stored as separate chapters.
+//!
+use crate::layered_repository::blob::BlobWriter;
+use crate::layered_repository::filename::{DeltaFileName, PathOrConf};
+use crate::layered_repository::storage_layer::{
+    Layer, PageReconstructData, PageReconstructResult, PageVersion, SegmentTag,
+};
+use crate::repository::WALRecord;
+use crate::waldecoder;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
+use anyhow::{bail, Result};
+use bytes::Bytes;
+use log::*;
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
+// avoid binding to Write (conflicts with std::io::Write)
+// while being able to use std::fmt::Write's methods
+use std::fmt::Write as _;
+use std::fs;
+use std::fs::File;
+use std::io::Write;
+use std::ops::Bound::Included;
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, Mutex, MutexGuard};
+
+use bookfile::{Book, BookWriter};
+
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
+
+use super::blob::{read_blob, BlobRange};
+
+// Magic constant to identify a Zenith delta file
+static DELTA_FILE_MAGIC: u32 = 0x5A616E01;
+
+/// Mapping from (block #, lsn) -> page/WAL record
+/// byte ranges in PAGE_VERSIONS_CHAPTER
+static PAGE_VERSION_METAS_CHAPTER: u64 = 1;
+/// Page/WAL bytes - cannot be interpreted
+/// without PAGE_VERSION_METAS_CHAPTER
+static PAGE_VERSIONS_CHAPTER: u64 = 2;
+static REL_SIZES_CHAPTER: u64 = 3;
+
+#[derive(Serialize, Deserialize)]
+struct PageVersionMeta {
+    page_image_range: Option<BlobRange>,
+    record_range: Option<BlobRange>,
+}
+
+///
+/// DeltaLayer is the in-memory data structure associated with an
+/// on-disk delta file.  We keep a DeltaLayer in memory for each
+/// file, in the LayerMap. If a layer is in "loaded" state, we have a
+/// copy of the file in memory, in 'inner'. Otherwise the struct is
+/// just a placeholder for a file that exists on disk, and it needs to
+/// be loaded before using it in queries.
+///
+pub struct DeltaLayer {
+    path_or_conf: PathOrConf,
+
+    pub tenantid: ZTenantId,
+    pub timelineid: ZTimelineId,
+    pub seg: SegmentTag,
+
+    //
+    // This entry contains all the changes from 'start_lsn' to 'end_lsn'. The
+    // start is inclusive, and end is exclusive.
+    //
+    pub start_lsn: Lsn,
+    pub end_lsn: Lsn,
+
+    dropped: bool,
+
+    /// Predecessor layer
+    predecessor: Option<Arc<dyn Layer>>,
+
+    inner: Mutex<DeltaLayerInner>,
+}
+
+pub struct DeltaLayerInner {
+    /// If false, the 'page_version_metas' and 'relsizes' have not been
+    /// loaded into memory yet.
+    loaded: bool,
+
+    /// All versions of all pages in the file are are kept here.
+    /// Indexed by block number and LSN.
+    page_version_metas: BTreeMap<(u32, Lsn), PageVersionMeta>,
+
+    /// `relsizes` tracks the size of the relation at different points in time.
+    relsizes: BTreeMap<Lsn, u32>,
+}
+
+impl Layer for DeltaLayer {
+    fn get_timeline_id(&self) -> ZTimelineId {
+        return self.timelineid;
+    }
+
+    fn get_seg_tag(&self) -> SegmentTag {
+        return self.seg;
+    }
+
+    fn is_dropped(&self) -> bool {
+        return self.dropped;
+    }
+
+    fn get_start_lsn(&self) -> Lsn {
+        return self.start_lsn;
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
+        return self.end_lsn;
+    }
+
+    fn filename(&self) -> PathBuf {
+        PathBuf::from(
+            DeltaFileName {
+                seg: self.seg,
+                start_lsn: self.start_lsn,
+                end_lsn: self.end_lsn,
+                dropped: self.dropped,
+            }
+            .to_string(),
+        )
+    }
+
+    /// Look up given page in the cache.
+    fn get_page_reconstruct_data(
+        &self,
+        blknum: u32,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<PageReconstructResult> {
+        let mut cont_lsn: Option<Lsn> = Some(lsn);
+
+        assert!(self.seg.blknum_in_seg(blknum));
+
+        {
+            // Open the file and lock the metadata in memory
+            // TODO: avoid opening the file for each read
+            let (_path, book) = self.open_book()?;
+            let page_version_reader = book.chapter_reader(PAGE_VERSIONS_CHAPTER)?;
+            let inner = self.load()?;
+
+            // Scan the metadata BTreeMap backwards, starting from the given entry.
+            let minkey = (blknum, Lsn(0));
+            let maxkey = (blknum, lsn);
+            let mut iter = inner
+                .page_version_metas
+                .range((Included(&minkey), Included(&maxkey)));
+            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
+                if let Some(img_range) = &entry.page_image_range {
+                    // Found a page image, return it
+                    let img = Bytes::from(read_blob(&page_version_reader, img_range)?);
+                    reconstruct_data.page_img = Some(img);
+                    cont_lsn = None;
+                    break;
+                } else if let Some(rec_range) = &entry.record_range {
+                    let rec = WALRecord::des(&read_blob(&page_version_reader, rec_range)?)?;
+                    let will_init = rec.will_init;
+                    reconstruct_data.records.push(rec);
+                    if will_init {
+                        // This WAL record initializes the page, so no need to go further back
+                        cont_lsn = None;
+                        break;
+                    } else {
+                        // This WAL record needs to be applied against an older page image
+                        cont_lsn = Some(*entry_lsn);
+                    }
+                } else {
+                    // No base image, and no WAL record. Huh?
+                    bail!("no page image or WAL record for requested page");
+                }
+            }
+
+            // release metadata lock and close the file
+        }
+
+        // If an older page image is needed to reconstruct the page, let the
+        // caller know about the predecessor layer.
+        if let Some(cont_lsn) = cont_lsn {
+            if let Some(cont_layer) = &self.predecessor {
+                Ok(PageReconstructResult::Continue(
+                    cont_lsn,
+                    Arc::clone(cont_layer),
+                ))
+            } else {
+                Ok(PageReconstructResult::Missing(cont_lsn))
+            }
+        } else {
+            Ok(PageReconstructResult::Complete)
+        }
+    }
+
+    /// Get size of the relation at given LSN
+    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
+        assert!(lsn >= self.start_lsn);
+
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let inner = self.load()?;
+        let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn)));
+
+        let result;
+        if let Some((_entry_lsn, entry)) = iter.next_back() {
+            result = *entry;
+        // Use the base image if needed
+        } else if let Some(predecessor) = &self.predecessor {
+            result = predecessor.get_seg_size(lsn)?;
+        } else {
+            result = 0;
+        }
+        Ok(result)
+    }
+
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
+        assert!(lsn >= self.start_lsn);
+
+        // Is the requested LSN after the rel was dropped?
+        if self.dropped && lsn >= self.end_lsn {
+            return Ok(false);
+        }
+
+        // Otherwise, it exists.
+        Ok(true)
+    }
+
+    ///
+    /// Release most of the memory used by this layer. If it's accessed again later,
+    /// it will need to be loaded back.
+    ///
+    fn unload(&self) -> Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.page_version_metas = BTreeMap::new();
+        inner.relsizes = BTreeMap::new();
+        inner.loaded = false;
+        Ok(())
+    }
+
+    fn delete(&self) -> Result<()> {
+        // delete underlying file
+        fs::remove_file(self.path())?;
+        Ok(())
+    }
+
+    fn is_incremental(&self) -> bool {
+        true
+    }
+
+    /// debugging function to print out the contents of the layer
+    fn dump(&self) -> Result<()> {
+        println!(
+            "----- delta layer for {} {}-{} ----",
+            self.seg, self.start_lsn, self.end_lsn
+        );
+
+        println!("--- relsizes ---");
+        let inner = self.load()?;
+        for (k, v) in inner.relsizes.iter() {
+            println!("  {}: {}", k, v);
+        }
+        println!("--- page versions ---");
+        let (_path, book) = self.open_book()?;
+        let chapter = book.chapter_reader(PAGE_VERSIONS_CHAPTER)?;
+        for (k, v) in inner.page_version_metas.iter() {
+            let mut desc = String::new();
+
+            if let Some(page_image_range) = v.page_image_range.as_ref() {
+                let image = read_blob(&chapter, &page_image_range)?;
+                write!(&mut desc, " img {} bytes", image.len())?;
+            }
+            if let Some(record_range) = v.record_range.as_ref() {
+                let record_bytes = read_blob(&chapter, record_range)?;
+                let rec = WALRecord::des(&record_bytes)?;
+                let wal_desc = waldecoder::describe_wal_record(&rec.rec);
+                write!(
+                    &mut desc,
+                    " rec {} bytes will_init: {} {}",
+                    rec.rec.len(),
+                    rec.will_init,
+                    wal_desc
+                )?;
+            }
+            println!("  blk {} at {}: {}", k.0, k.1, desc);
+        }
+
+        Ok(())
+    }
+}
+
+impl DeltaLayer {
+    fn path(&self) -> PathBuf {
+        Self::path_for(
+            &self.path_or_conf,
+            self.timelineid,
+            self.tenantid,
+            &DeltaFileName {
+                seg: self.seg,
+                start_lsn: self.start_lsn,
+                end_lsn: self.end_lsn,
+                dropped: self.dropped,
+            },
+        )
+    }
+
+    fn path_for(
+        path_or_conf: &PathOrConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        fname: &DeltaFileName,
+    ) -> PathBuf {
+        match path_or_conf {
+            PathOrConf::Path(path) => path.clone(),
+            PathOrConf::Conf(conf) => conf
+                .timeline_path(&timelineid, &tenantid)
+                .join(fname.to_string()),
+        }
+    }
+
+    /// Create a new delta file, using the given btreemaps containing the page versions and
+    /// relsizes.
+    ///
+    /// This is used to write the in-memory layer to disk. The in-memory layer uses the same
+    /// data structure with two btreemaps as we do, so passing the btreemaps is currently
+    /// expedient.
+    pub fn create(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        seg: SegmentTag,
+        start_lsn: Lsn,
+        end_lsn: Lsn,
+        dropped: bool,
+        predecessor: Option<Arc<dyn Layer>>,
+        page_versions: BTreeMap<(u32, Lsn), PageVersion>,
+        relsizes: BTreeMap<Lsn, u32>,
+    ) -> Result<DeltaLayer> {
+        let delta_layer = DeltaLayer {
+            path_or_conf: PathOrConf::Conf(conf),
+            timelineid: timelineid,
+            tenantid: tenantid,
+            seg: seg,
+            start_lsn: start_lsn,
+            end_lsn,
+            dropped,
+            inner: Mutex::new(DeltaLayerInner {
+                loaded: true,
+                page_version_metas: BTreeMap::new(),
+                relsizes: relsizes,
+            }),
+            predecessor,
+        };
+        let mut inner = delta_layer.inner.lock().unwrap();
+
+        // Write the in-memory btreemaps into a file
+        let path = delta_layer.path();
+
+        // Note: This overwrites any existing file. There shouldn't be any.
+        // FIXME: throw an error instead?
+        let file = File::create(&path)?;
+        let book = BookWriter::new(file, DELTA_FILE_MAGIC)?;
+
+        let mut page_version_writer = BlobWriter::new(book, PAGE_VERSIONS_CHAPTER);
+
+        for (key, page_version) in page_versions {
+            let page_image_range = page_version
+                .page_image
+                .map(|page_image| page_version_writer.write_blob(page_image.as_ref()))
+                .transpose()?;
+
+            let record_range = page_version
+                .record
+                .map(|record| {
+                    let buf = WALRecord::ser(&record)?;
+                    page_version_writer.write_blob(&buf)
+                })
+                .transpose()?;
+
+            let old = inner.page_version_metas.insert(
+                key,
+                PageVersionMeta {
+                    page_image_range,
+                    record_range,
+                },
+            );
+
+            assert!(old.is_none());
+        }
+
+        let book = page_version_writer.close()?;
+
+        // Write out page versions
+        let mut chapter = book.new_chapter(PAGE_VERSION_METAS_CHAPTER);
+        let buf = BTreeMap::ser(&inner.page_version_metas)?;
+        chapter.write_all(&buf)?;
+        let book = chapter.close()?;
+
+        // and relsizes to separate chapter
+        let mut chapter = book.new_chapter(REL_SIZES_CHAPTER);
+        let buf = BTreeMap::ser(&inner.relsizes)?;
+        chapter.write_all(&buf)?;
+        let book = chapter.close()?;
+
+        book.close()?;
+
+        trace!("saved {}", &path.display());
+
+        drop(inner);
+
+        Ok(delta_layer)
+    }
+
+    fn open_book(&self) -> Result<(PathBuf, Book<File>)> {
+        let path = Self::path_for(
+            &self.path_or_conf,
+            self.timelineid,
+            self.tenantid,
+            &DeltaFileName {
+                seg: self.seg,
+                start_lsn: self.start_lsn,
+                end_lsn: self.end_lsn,
+                dropped: self.dropped,
+            },
+        );
+
+        let file = File::open(&path)?;
+        let book = Book::new(file)?;
+
+        Ok((path, book))
+    }
+
+    ///
+    /// Load the contents of the file into memory
+    ///
+    fn load(&self) -> Result<MutexGuard<DeltaLayerInner>> {
+        // quick exit if already loaded
+        let mut inner = self.inner.lock().unwrap();
+
+        if inner.loaded {
+            return Ok(inner);
+        }
+
+        let (path, book) = self.open_book()?;
+
+        let chapter = book.read_chapter(PAGE_VERSION_METAS_CHAPTER)?;
+        let page_version_metas = BTreeMap::des(&chapter)?;
+
+        let chapter = book.read_chapter(REL_SIZES_CHAPTER)?;
+        let relsizes = BTreeMap::des(&chapter)?;
+
+        debug!("loaded from {}", &path.display());
+
+        *inner = DeltaLayerInner {
+            loaded: true,
+            page_version_metas,
+            relsizes,
+        };
+
+        Ok(inner)
+    }
+
+    /// Create a DeltaLayer struct representing an existing file on disk.
+    pub fn new(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        filename: &DeltaFileName,
+        predecessor: Option<Arc<dyn Layer>>,
+    ) -> DeltaLayer {
+        DeltaLayer {
+            path_or_conf: PathOrConf::Conf(conf),
+            timelineid,
+            tenantid,
+            seg: filename.seg,
+            start_lsn: filename.start_lsn,
+            end_lsn: filename.end_lsn,
+            dropped: filename.dropped,
+            inner: Mutex::new(DeltaLayerInner {
+                loaded: false,
+                page_version_metas: BTreeMap::new(),
+                relsizes: BTreeMap::new(),
+            }),
+            predecessor,
+        }
+    }
+
+    /// Create a DeltaLayer struct representing an existing file on disk.
+    ///
+    /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary.
+    pub fn new_for_path(
+        path: &Path,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        filename: &DeltaFileName,
+    ) -> DeltaLayer {
+        DeltaLayer {
+            path_or_conf: PathOrConf::Path(path.to_path_buf()),
+            timelineid,
+            tenantid,
+            seg: filename.seg,
+            start_lsn: filename.start_lsn,
+            end_lsn: filename.end_lsn,
+            dropped: filename.dropped,
+            inner: Mutex::new(DeltaLayerInner {
+                loaded: false,
+                page_version_metas: BTreeMap::new(),
+                relsizes: BTreeMap::new(),
+            }),
+            predecessor: None,
+        }
+    }
+}

pageserver/src/layered_repository/filename.rs

@@ -0,0 +1,319 @@
+//!
+//! Helper functions for dealing with filenames of the image and delta layer files.
+//!
+use crate::layered_repository::storage_layer::SegmentTag;
+use crate::relish::*;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
+use std::fmt;
+use std::fs;
+use std::path::PathBuf;
+
+use anyhow::Result;
+use log::*;
+use zenith_utils::lsn::Lsn;
+
+// Note: LayeredTimeline::load_layer_map() relies on this sort order
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+pub struct DeltaFileName {
+    pub seg: SegmentTag,
+    pub start_lsn: Lsn,
+    pub end_lsn: Lsn,
+    pub dropped: bool,
+}
+
+/// Represents the filename of a DeltaLayer
+///
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>
+///
+/// or if it was dropped:
+///
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>_DROPPED
+///
+impl DeltaFileName {
+    ///
+    /// Parse a string as a delta file name. Returns None if the filename does not
+    /// match the expected pattern.
+    ///
+    pub fn from_str(fname: &str) -> Option<Self> {
+        let rel;
+        let mut parts;
+        if let Some(rest) = fname.strip_prefix("rel_") {
+            parts = rest.split('_');
+            rel = RelishTag::Relation(RelTag {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+                relnode: parts.next()?.parse::<u32>().ok()?,
+                forknum: parts.next()?.parse::<u8>().ok()?,
+            });
+        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
+            parts = rest.split('_');
+            rel = RelishTag::FileNodeMap {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
+            parts = rest.split('_');
+            rel = RelishTag::TwoPhase {
+                xid: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
+            parts = rest.split('_');
+            rel = RelishTag::Checkpoint;
+        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
+            parts = rest.split('_');
+            rel = RelishTag::ControlFile;
+        } else {
+            return None;
+        }
+
+        let segno = parts.next()?.parse::<u32>().ok()?;
+
+        let seg = SegmentTag { rel, segno };
+
+        let start_lsn = Lsn::from_hex(parts.next()?).ok()?;
+        let end_lsn = Lsn::from_hex(parts.next()?).ok()?;
+
+        let mut dropped = false;
+        if let Some(suffix) = parts.next() {
+            if suffix == "DROPPED" {
+                dropped = true;
+            } else {
+                return None;
+            }
+        }
+        if parts.next().is_some() {
+            return None;
+        }
+
+        Some(DeltaFileName {
+            seg,
+            start_lsn,
+            end_lsn,
+            dropped,
+        })
+    }
+
+    fn to_string(&self) -> String {
+        let basename = match self.seg.rel {
+            RelishTag::Relation(reltag) => format!(
+                "rel_{}_{}_{}_{}",
+                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
+            ),
+            RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno,
+            } => format!("pg_xact_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno,
+            } => format!("pg_multixact_members_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno,
+            } => format!("pg_multixact_offsets_{:04X}", segno),
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
+            RelishTag::Checkpoint => format!("pg_control_checkpoint"),
+            RelishTag::ControlFile => format!("pg_control"),
+        };
+
+        format!(
+            "{}_{}_{:016X}_{:016X}{}",
+            basename,
+            self.seg.segno,
+            u64::from(self.start_lsn),
+            u64::from(self.end_lsn),
+            if self.dropped { "_DROPPED" } else { "" }
+        )
+    }
+}
+
+impl fmt::Display for DeltaFileName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.to_string())
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+pub struct ImageFileName {
+    pub seg: SegmentTag,
+    pub lsn: Lsn,
+}
+
+///
+/// Represents the filename of an ImageLayer
+///
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<LSN>
+///
+impl ImageFileName {
+    ///
+    /// Parse a string as an image file name. Returns None if the filename does not
+    /// match the expected pattern.
+    ///
+    pub fn from_str(fname: &str) -> Option<Self> {
+        let rel;
+        let mut parts;
+        if let Some(rest) = fname.strip_prefix("rel_") {
+            parts = rest.split('_');
+            rel = RelishTag::Relation(RelTag {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+                relnode: parts.next()?.parse::<u32>().ok()?,
+                forknum: parts.next()?.parse::<u8>().ok()?,
+            });
+        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
+            parts = rest.split('_');
+            rel = RelishTag::FileNodeMap {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
+            parts = rest.split('_');
+            rel = RelishTag::TwoPhase {
+                xid: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
+            parts = rest.split('_');
+            rel = RelishTag::Checkpoint;
+        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
+            parts = rest.split('_');
+            rel = RelishTag::ControlFile;
+        } else {
+            return None;
+        }
+
+        let segno = parts.next()?.parse::<u32>().ok()?;
+
+        let seg = SegmentTag { rel, segno };
+
+        let lsn = Lsn::from_hex(parts.next()?).ok()?;
+
+        if parts.next().is_some() {
+            return None;
+        }
+
+        Some(ImageFileName { seg, lsn })
+    }
+
+    fn to_string(&self) -> String {
+        let basename = match self.seg.rel {
+            RelishTag::Relation(reltag) => format!(
+                "rel_{}_{}_{}_{}",
+                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
+            ),
+            RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno,
+            } => format!("pg_xact_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno,
+            } => format!("pg_multixact_members_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno,
+            } => format!("pg_multixact_offsets_{:04X}", segno),
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
+            RelishTag::Checkpoint => format!("pg_control_checkpoint"),
+            RelishTag::ControlFile => format!("pg_control"),
+        };
+
+        format!(
+            "{}_{}_{:016X}",
+            basename,
+            self.seg.segno,
+            u64::from(self.lsn),
+        )
+    }
+}
+
+impl fmt::Display for ImageFileName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.to_string())
+    }
+}
+
+/// Scan timeline directory and create ImageFileName and DeltaFilename
+/// structs representing all files on disk
+///
+/// TODO: returning an Iterator would be more idiomatic
+pub fn list_files(
+    conf: &'static PageServerConf,
+    timelineid: ZTimelineId,
+    tenantid: ZTenantId,
+) -> Result<(Vec<ImageFileName>, Vec<DeltaFileName>)> {
+    let path = conf.timeline_path(&timelineid, &tenantid);
+
+    let mut deltafiles: Vec<DeltaFileName> = Vec::new();
+    let mut imgfiles: Vec<ImageFileName> = Vec::new();
+    for direntry in fs::read_dir(path)? {
+        let fname = direntry?.file_name();
+        let fname = fname.to_str().unwrap();
+
+        if let Some(deltafilename) = DeltaFileName::from_str(fname) {
+            deltafiles.push(deltafilename);
+        } else if let Some(imgfilename) = ImageFileName::from_str(fname) {
+            imgfiles.push(imgfilename);
+        } else if fname == "wal" || fname == "metadata" || fname == "ancestor" {
+            // ignore these
+        } else {
+            warn!("unrecognized filename in timeline dir: {}", fname);
+        }
+    }
+    return Ok((imgfiles, deltafiles));
+}
+
+/// Helper enum to hold a PageServerConf, or a path
+///
+/// This is used by DeltaLayer and ImageLayer. Normally, this holds a reference to the
+/// global config, and paths to layer files are constructed using the tenant/timeline
+/// path from the config. But in the 'dump_layerfile' binary, we need to construct a Layer
+/// struct for a file on disk, without having a page server running, so that we have no
+/// config. In that case, we use the Path variant to hold the full path to the file on
+/// disk.
+pub enum PathOrConf {
+    Path(PathBuf),
+    Conf(&'static PageServerConf),
+}

pageserver/src/layered_repository/image_layer.rs

@@ -0,0 +1,436 @@
+//! An ImageLayer represents an image or a snapshot of a segment at one particular LSN.
+//! It is stored in a file on disk.
+//!
+//! On disk, the image files are stored in timelines/<timelineid> directory.
+//! Currently, there are no subdirectories, and each image layer file is named like this:
+//!
+//! Note that segno is
+//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<LSN>
+//!
+//! For example:
+//!
+//!    1663_13990_2609_0_5_000000000169C348
+//!
+//! An image file is constructed using the 'bookfile' crate.
+//!
+//! Only metadata is loaded into memory by the load function.
+//! When images are needed, they are read directly from disk.
+//!
+//! For blocky relishes, the images are stored in BLOCKY_IMAGES_CHAPTER.
+//! All the images are required to be BLOCK_SIZE, which allows for random access.
+//!
+//! For non-blocky relishes, the image can be found in NONBLOCKY_IMAGE_CHAPTER.
+//!
+use crate::layered_repository::filename::{ImageFileName, PathOrConf};
+use crate::layered_repository::storage_layer::{
+    Layer, PageReconstructData, PageReconstructResult, SegmentTag,
+};
+use crate::layered_repository::LayeredTimeline;
+use crate::layered_repository::RELISH_SEG_SIZE;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
+use anyhow::{anyhow, ensure, Result};
+use bytes::Bytes;
+use log::*;
+use std::convert::TryInto;
+use std::fs;
+use std::fs::File;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::sync::{Mutex, MutexGuard};
+
+use bookfile::{Book, BookWriter};
+
+use zenith_utils::lsn::Lsn;
+
+// Magic constant to identify a Zenith segment image file
+const IMAGE_FILE_MAGIC: u32 = 0x5A616E01 + 1;
+
+/// Contains each block in block # order
+const BLOCKY_IMAGES_CHAPTER: u64 = 1;
+const NONBLOCKY_IMAGE_CHAPTER: u64 = 2;
+
+const BLOCK_SIZE: usize = 8192;
+
+///
+/// ImageLayer is the in-memory data structure associated with an on-disk image
+/// file.  We keep an ImageLayer in memory for each file, in the LayerMap. If a
+/// layer is in "loaded" state, we have a copy of the file in memory, in 'inner'.
+/// Otherwise the struct is just a placeholder for a file that exists on disk,
+/// and it needs to be loaded before using it in queries.
+///
+pub struct ImageLayer {
+    path_or_conf: PathOrConf,
+    pub tenantid: ZTenantId,
+    pub timelineid: ZTimelineId,
+    pub seg: SegmentTag,
+
+    // This entry contains an image of all pages as of this LSN
+    pub lsn: Lsn,
+
+    inner: Mutex<ImageLayerInner>,
+}
+
+#[derive(Clone)]
+enum ImageType {
+    Blocky { num_blocks: u32 },
+    NonBlocky,
+}
+
+pub struct ImageLayerInner {
+    /// If false, the 'image_type' has not been
+    /// loaded into memory yet.
+    loaded: bool,
+
+    /// Derived from filename and bookfile chapter metadata
+    image_type: ImageType,
+}
+
+impl Layer for ImageLayer {
+    fn filename(&self) -> PathBuf {
+        PathBuf::from(
+            ImageFileName {
+                seg: self.seg,
+                lsn: self.lsn,
+            }
+            .to_string(),
+        )
+    }
+
+    fn get_timeline_id(&self) -> ZTimelineId {
+        return self.timelineid;
+    }
+
+    fn get_seg_tag(&self) -> SegmentTag {
+        return self.seg;
+    }
+
+    fn is_dropped(&self) -> bool {
+        return false;
+    }
+
+    fn get_start_lsn(&self) -> Lsn {
+        return self.lsn;
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
+        return self.lsn;
+    }
+
+    /// Look up given page in the file
+    fn get_page_reconstruct_data(
+        &self,
+        blknum: u32,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<PageReconstructResult> {
+        assert!(lsn >= self.lsn);
+
+        let inner = self.load()?;
+
+        let base_blknum = blknum % RELISH_SEG_SIZE;
+
+        let (_path, book) = self.open_book()?;
+
+        let buf = match &inner.image_type {
+            ImageType::Blocky { num_blocks } => {
+                if base_blknum >= *num_blocks {
+                    return Ok(PageReconstructResult::Missing(lsn));
+                }
+
+                let mut buf = vec![0u8; BLOCK_SIZE];
+                let offset = BLOCK_SIZE as u64 * base_blknum as u64;
+
+                let chapter = book.chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
+                chapter.read_exact_at(&mut buf, offset)?;
+
+                buf
+            }
+            ImageType::NonBlocky => {
+                ensure!(base_blknum == 0);
+                book.read_chapter(NONBLOCKY_IMAGE_CHAPTER)?.into_vec()
+            }
+        };
+
+        reconstruct_data.page_img = Some(Bytes::from(buf));
+        Ok(PageReconstructResult::Complete)
+    }
+
+    /// Get size of the segment
+    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
+        assert!(lsn >= self.lsn);
+
+        let inner = self.load()?;
+        match inner.image_type {
+            ImageType::Blocky { num_blocks } => Ok(num_blocks),
+            ImageType::NonBlocky => Err(anyhow!("get_seg_size called for non-blocky segment")),
+        }
+    }
+
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
+        assert!(lsn >= self.lsn);
+        Ok(true)
+    }
+
+    ///
+    /// Release most of the memory used by this layer. If it's accessed again later,
+    /// it will need to be loaded back.
+    ///
+    fn unload(&self) -> Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.image_type = ImageType::Blocky { num_blocks: 0 };
+        inner.loaded = false;
+        Ok(())
+    }
+
+    fn delete(&self) -> Result<()> {
+        // delete underlying file
+        fs::remove_file(self.path())?;
+        Ok(())
+    }
+
+    fn is_incremental(&self) -> bool {
+        false
+    }
+
+    /// debugging function to print out the contents of the layer
+    fn dump(&self) -> Result<()> {
+        println!("----- image layer for {} at {} ----", self.seg, self.lsn);
+
+        let inner = self.load()?;
+
+        match inner.image_type {
+            ImageType::Blocky { num_blocks } => println!("({}) blocks ", num_blocks),
+            ImageType::NonBlocky => {
+                let (_path, book) = self.open_book()?;
+                let chapter = book.read_chapter(NONBLOCKY_IMAGE_CHAPTER)?;
+                println!("non-blocky ({} bytes)", chapter.len());
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl ImageLayer {
+    fn path(&self) -> PathBuf {
+        Self::path_for(
+            &self.path_or_conf,
+            self.timelineid,
+            self.tenantid,
+            &ImageFileName {
+                seg: self.seg,
+                lsn: self.lsn,
+            },
+        )
+    }
+
+    fn path_for(
+        path_or_conf: &PathOrConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        fname: &ImageFileName,
+    ) -> PathBuf {
+        match path_or_conf {
+            PathOrConf::Path(path) => path.to_path_buf(),
+            PathOrConf::Conf(conf) => conf
+                .timeline_path(&timelineid, &tenantid)
+                .join(fname.to_string()),
+        }
+    }
+
+    /// Create a new image file, using the given array of pages.
+    fn create(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        seg: SegmentTag,
+        lsn: Lsn,
+        base_images: Vec<Bytes>,
+    ) -> Result<ImageLayer> {
+        let image_type = if seg.rel.is_blocky() {
+            let num_blocks: u32 = base_images.len().try_into()?;
+            ImageType::Blocky { num_blocks }
+        } else {
+            assert_eq!(base_images.len(), 1);
+            ImageType::NonBlocky
+        };
+
+        let layer = ImageLayer {
+            path_or_conf: PathOrConf::Conf(conf),
+            timelineid: timelineid,
+            tenantid: tenantid,
+            seg: seg,
+            lsn: lsn,
+            inner: Mutex::new(ImageLayerInner {
+                loaded: true,
+                image_type: image_type.clone(),
+            }),
+        };
+        let inner = layer.inner.lock().unwrap();
+
+        // Write the images into a file
+        let path = layer.path();
+
+        // Note: This overwrites any existing file. There shouldn't be any.
+        // FIXME: throw an error instead?
+        let file = File::create(&path)?;
+        let book = BookWriter::new(file, IMAGE_FILE_MAGIC)?;
+
+        let book = match &image_type {
+            ImageType::Blocky { .. } => {
+                let mut chapter = book.new_chapter(BLOCKY_IMAGES_CHAPTER);
+                for block_bytes in base_images {
+                    assert_eq!(block_bytes.len(), BLOCK_SIZE);
+                    chapter.write_all(&block_bytes)?;
+                }
+                chapter.close()?
+            }
+            ImageType::NonBlocky => {
+                let mut chapter = book.new_chapter(NONBLOCKY_IMAGE_CHAPTER);
+                chapter.write_all(&base_images[0])?;
+                chapter.close()?
+            }
+        };
+
+        book.close()?;
+
+        trace!("saved {}", &path.display());
+
+        drop(inner);
+
+        Ok(layer)
+    }
+
+    // Create a new image file by materializing every page in a source layer
+    // at given LSN.
+    pub fn create_from_src(
+        conf: &'static PageServerConf,
+        timeline: &LayeredTimeline,
+        src: &dyn Layer,
+        lsn: Lsn,
+    ) -> Result<ImageLayer> {
+        let seg = src.get_seg_tag();
+        let timelineid = timeline.timelineid;
+
+        let startblk;
+        let size;
+        if seg.rel.is_blocky() {
+            size = src.get_seg_size(lsn)?;
+            startblk = seg.segno * RELISH_SEG_SIZE;
+        } else {
+            size = 1;
+            startblk = 0;
+        }
+
+        trace!(
+            "creating new ImageLayer for {} on timeline {} at {}",
+            seg,
+            timelineid,
+            lsn,
+        );
+
+        let mut base_images: Vec<Bytes> = Vec::new();
+        for blknum in startblk..(startblk + size) {
+            let img = timeline.materialize_page(seg, blknum, lsn, &*src)?;
+
+            base_images.push(img);
+        }
+
+        Self::create(conf, timelineid, timeline.tenantid, seg, lsn, base_images)
+    }
+
+    ///
+    /// Load the contents of the file into memory
+    ///
+    fn load(&self) -> Result<MutexGuard<ImageLayerInner>> {
+        // quick exit if already loaded
+        let mut inner = self.inner.lock().unwrap();
+
+        if inner.loaded {
+            return Ok(inner);
+        }
+
+        let (path, book) = self.open_book()?;
+
+        let image_type = if self.seg.rel.is_blocky() {
+            let chapter = book.chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
+            let images_len = chapter.len();
+            ensure!(images_len % BLOCK_SIZE as u64 == 0);
+            let num_blocks: u32 = (images_len / BLOCK_SIZE as u64).try_into()?;
+            ImageType::Blocky { num_blocks }
+        } else {
+            let _chapter = book.chapter_reader(NONBLOCKY_IMAGE_CHAPTER)?;
+            ImageType::NonBlocky
+        };
+
+        debug!("loaded from {}", &path.display());
+
+        *inner = ImageLayerInner {
+            loaded: true,
+            image_type,
+        };
+
+        Ok(inner)
+    }
+
+    fn open_book(&self) -> Result<(PathBuf, Book<File>)> {
+        let path = Self::path_for(
+            &self.path_or_conf,
+            self.timelineid,
+            self.tenantid,
+            &ImageFileName {
+                seg: self.seg,
+                lsn: self.lsn,
+            },
+        );
+
+        let file = File::open(&path)?;
+        let book = Book::new(file)?;
+
+        Ok((path, book))
+    }
+
+    /// Create an ImageLayer struct representing an existing file on disk
+    pub fn new(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        filename: &ImageFileName,
+    ) -> ImageLayer {
+        ImageLayer {
+            path_or_conf: PathOrConf::Conf(conf),
+            timelineid,
+            tenantid,
+            seg: filename.seg,
+            lsn: filename.lsn,
+            inner: Mutex::new(ImageLayerInner {
+                loaded: false,
+                image_type: ImageType::Blocky { num_blocks: 0 },
+            }),
+        }
+    }
+
+    /// Create an ImageLayer struct representing an existing file on disk.
+    ///
+    /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary.
+    pub fn new_for_path(
+        path: &Path,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        filename: &ImageFileName,
+    ) -> ImageLayer {
+        ImageLayer {
+            path_or_conf: PathOrConf::Path(path.to_path_buf()),
+            timelineid,
+            tenantid,
+            seg: filename.seg,
+            lsn: filename.lsn,
+            inner: Mutex::new(ImageLayerInner {
+                loaded: false,
+                image_type: ImageType::Blocky { num_blocks: 0 },
+            }),
+        }
+    }
+}

pageserver/src/layered_repository/inmemory_layer.rs

@@ -0,0 +1,642 @@
+//!
+//! An in-memory layer stores recently received page versions in memory. The page versions
+//! are held in a BTreeMap, and there's another BTreeMap to track the size of the relation.
+//!
+use crate::layered_repository::filename::DeltaFileName;
+use crate::layered_repository::storage_layer::{
+    Layer, PageReconstructData, PageReconstructResult, PageVersion, SegmentTag, RELISH_SEG_SIZE,
+};
+use crate::layered_repository::LayeredTimeline;
+use crate::layered_repository::{DeltaLayer, ImageLayer};
+use crate::repository::WALRecord;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
+use anyhow::{bail, Result};
+use bytes::Bytes;
+use log::*;
+use std::collections::BTreeMap;
+use std::ops::Bound::Included;
+use std::path::PathBuf;
+use std::sync::{Arc, Mutex};
+
+use zenith_utils::lsn::Lsn;
+
+pub struct InMemoryLayer {
+    conf: &'static PageServerConf,
+    tenantid: ZTenantId,
+    timelineid: ZTimelineId,
+    seg: SegmentTag,
+
+    ///
+    /// This layer contains all the changes from 'start_lsn'. The
+    /// start is inclusive. There is no end LSN; we only use in-memory
+    /// layer at the end of a timeline.
+    ///
+    start_lsn: Lsn,
+
+    /// LSN of the oldest page version stored in this layer
+    oldest_pending_lsn: Lsn,
+
+    /// The above fields never change. The parts that do change are in 'inner',
+    /// and protected by mutex.
+    inner: Mutex<InMemoryLayerInner>,
+
+    /// Predecessor layer
+    predecessor: Option<Arc<dyn Layer>>,
+}
+
+pub struct InMemoryLayerInner {
+    /// If this relation was dropped, remember when that happened.
+    drop_lsn: Option<Lsn>,
+
+    ///
+    /// All versions of all pages in the layer are are kept here.
+    /// Indexed by block number and LSN.
+    ///
+    page_versions: BTreeMap<(u32, Lsn), PageVersion>,
+
+    ///
+    /// `segsizes` tracks the size of the segment at different points in time.
+    ///
+    segsizes: BTreeMap<Lsn, u32>,
+}
+
+impl InMemoryLayerInner {
+    fn get_seg_size(&self, lsn: Lsn) -> u32 {
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let mut iter = self.segsizes.range((Included(&Lsn(0)), Included(&lsn)));
+
+        if let Some((_entry_lsn, entry)) = iter.next_back() {
+            *entry
+        } else {
+            0
+        }
+    }
+}
+
+impl Layer for InMemoryLayer {
+    // An in-memory layer doesn't really have a filename as it's not stored on disk,
+    // but we construct a filename as if it was a delta layer
+    fn filename(&self) -> PathBuf {
+        let inner = self.inner.lock().unwrap();
+
+        let end_lsn;
+        let dropped;
+        if let Some(drop_lsn) = inner.drop_lsn {
+            end_lsn = drop_lsn;
+            dropped = true;
+        } else {
+            end_lsn = Lsn(u64::MAX);
+            dropped = false;
+        }
+
+        let delta_filename = DeltaFileName {
+            seg: self.seg,
+            start_lsn: self.start_lsn,
+            end_lsn: end_lsn,
+            dropped: dropped,
+        }
+        .to_string();
+
+        PathBuf::from(format!("inmem-{}", delta_filename))
+    }
+
+    fn get_timeline_id(&self) -> ZTimelineId {
+        return self.timelineid;
+    }
+
+    fn get_seg_tag(&self) -> SegmentTag {
+        return self.seg;
+    }
+
+    fn get_start_lsn(&self) -> Lsn {
+        return self.start_lsn;
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
+        let inner = self.inner.lock().unwrap();
+
+        if let Some(drop_lsn) = inner.drop_lsn {
+            drop_lsn
+        } else {
+            Lsn(u64::MAX)
+        }
+    }
+
+    fn is_dropped(&self) -> bool {
+        let inner = self.inner.lock().unwrap();
+        inner.drop_lsn.is_some()
+    }
+
+    /// Look up given page in the cache.
+    fn get_page_reconstruct_data(
+        &self,
+        blknum: u32,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<PageReconstructResult> {
+        let mut cont_lsn: Option<Lsn> = Some(lsn);
+
+        assert!(self.seg.blknum_in_seg(blknum));
+
+        {
+            let inner = self.inner.lock().unwrap();
+
+            // Scan the BTreeMap backwards, starting from reconstruct_data.lsn.
+            let minkey = (blknum, Lsn(0));
+            let maxkey = (blknum, lsn);
+            let mut iter = inner
+                .page_versions
+                .range((Included(&minkey), Included(&maxkey)));
+            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
+                if let Some(img) = &entry.page_image {
+                    reconstruct_data.page_img = Some(img.clone());
+                    cont_lsn = None;
+                    break;
+                } else if let Some(rec) = &entry.record {
+                    reconstruct_data.records.push(rec.clone());
+                    if rec.will_init {
+                        // This WAL record initializes the page, so no need to go further back
+                        cont_lsn = None;
+                        break;
+                    } else {
+                        // This WAL record needs to be applied against an older page image
+                        cont_lsn = Some(*entry_lsn);
+                    }
+                } else {
+                    // No base image, and no WAL record. Huh?
+                    bail!("no page image or WAL record for requested page");
+                }
+            }
+
+            // release lock on 'inner'
+        }
+
+        // If an older page image is needed to reconstruct the page, let the
+        // caller know about the predecessor layer.
+        if let Some(cont_lsn) = cont_lsn {
+            if let Some(cont_layer) = &self.predecessor {
+                Ok(PageReconstructResult::Continue(
+                    cont_lsn,
+                    Arc::clone(cont_layer),
+                ))
+            } else {
+                Ok(PageReconstructResult::Missing(cont_lsn))
+            }
+        } else {
+            Ok(PageReconstructResult::Complete)
+        }
+    }
+
+    /// Get size of the relation at given LSN
+    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
+        assert!(lsn >= self.start_lsn);
+
+        let inner = self.inner.lock().unwrap();
+        Ok(inner.get_seg_size(lsn))
+    }
+
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
+        assert!(lsn >= self.start_lsn);
+
+        let inner = self.inner.lock().unwrap();
+
+        // Is the requested LSN after the segment was dropped?
+        if let Some(drop_lsn) = inner.drop_lsn {
+            if lsn >= drop_lsn {
+                return Ok(false);
+            }
+        }
+
+        // Otherwise, it exists
+        Ok(true)
+    }
+
+    /// Cannot unload anything in an in-memory layer, since there's no backing
+    /// store. To release memory used by an in-memory layer, use 'freeze' to turn
+    /// it into an on-disk layer.
+    fn unload(&self) -> Result<()> {
+        Ok(())
+    }
+
+    /// Nothing to do here. When you drop the last reference to the layer, it will
+    /// be deallocated.
+    fn delete(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn is_incremental(&self) -> bool {
+        self.predecessor.is_some()
+    }
+
+    /// debugging function to print out the contents of the layer
+    fn dump(&self) -> Result<()> {
+        let inner = self.inner.lock().unwrap();
+
+        let end_str = inner
+            .drop_lsn
+            .as_ref()
+            .map(|drop_lsn| drop_lsn.to_string())
+            .unwrap_or_default();
+
+        println!(
+            "----- in-memory layer for tli {} seg {} {}-{} ----",
+            self.timelineid, self.seg, self.start_lsn, end_str
+        );
+
+        for (k, v) in inner.segsizes.iter() {
+            println!("{}: {}", k, v);
+        }
+        //for (k, v) in inner.page_versions.iter() {
+        //    println!("blk {} at {}: {}/{}", k.0, k.1, v.page_image.is_some(), v.record.is_some());
+        //}
+
+        Ok(())
+    }
+}
+
+impl InMemoryLayer {
+    /// Return the oldest page version that's stored in this layer
+    pub fn get_oldest_pending_lsn(&self) -> Lsn {
+        self.oldest_pending_lsn
+    }
+
+    ///
+    /// Create a new, empty, in-memory layer
+    ///
+    pub fn create(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        seg: SegmentTag,
+        start_lsn: Lsn,
+        oldest_pending_lsn: Lsn,
+    ) -> Result<InMemoryLayer> {
+        println!(
+            "initializing new empty InMemoryLayer for writing {} on timeline {} at {}",
+            seg,
+            timelineid,
+            start_lsn
+        );
+
+        Ok(InMemoryLayer {
+            conf,
+            timelineid,
+            tenantid,
+            seg,
+            start_lsn,
+            oldest_pending_lsn,
+            inner: Mutex::new(InMemoryLayerInner {
+                drop_lsn: None,
+                page_versions: BTreeMap::new(),
+                segsizes: BTreeMap::new(),
+            }),
+            predecessor: None,
+        })
+    }
+
+    // Write operations
+
+    /// Remember new page version, as a WAL record over previous version
+    pub fn put_wal_record(&self, blknum: u32, rec: WALRecord) -> Result<()> {
+        self.put_page_version(
+            blknum,
+            rec.lsn,
+            PageVersion {
+                page_image: None,
+                record: Some(rec),
+            },
+        )
+    }
+
+    /// Remember new page version, as a full page image
+    pub fn put_page_image(&self, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()> {
+        self.put_page_version(
+            blknum,
+            lsn,
+            PageVersion {
+                page_image: Some(img),
+                record: None,
+            },
+        )
+    }
+
+    /// Common subroutine of the public put_wal_record() and put_page_image() functions.
+    /// Adds the page version to the in-memory tree
+    pub fn put_page_version(&self, blknum: u32, lsn: Lsn, pv: PageVersion) -> Result<()> {
+        assert!(self.seg.blknum_in_seg(blknum));
+
+        println!(
+            "put_page_version blk {} of {} at {}/{}",
+            blknum,
+            self.seg.rel,
+            self.timelineid,
+            lsn
+        );
+        let mut inner = self.inner.lock().unwrap();
+
+        let old = inner.page_versions.insert((blknum, lsn), pv);
+
+        if old.is_some() {
+            // We already had an entry for this LSN. That's odd..
+            warn!(
+                "Page version of rel {} blk {} at {} already exists",
+                self.seg.rel, blknum, lsn
+            );
+        }
+
+        // Also update the relation size, if this extended the relation.
+        if self.seg.rel.is_blocky() {
+            let newsize = blknum - self.seg.segno * RELISH_SEG_SIZE + 1;
+
+            // use inner get_seg_size, since calling self.get_seg_size will try to acquire self.inner.lock
+            // which we've just acquired above
+            let oldsize = inner.get_seg_size(lsn);
+            if newsize > oldsize {
+                println!(
+                    "enlarging segment {} from {} to {} blocks at {}",
+                    self.seg,
+                    oldsize,
+                    newsize,
+                    lsn
+                );
+                inner.segsizes.insert(lsn, newsize);
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Remember that the relation was truncated at given LSN
+    pub fn put_truncation(&self, lsn: Lsn, segsize: u32) -> anyhow::Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+        let old = inner.segsizes.insert(lsn, segsize);
+
+        if old.is_some() {
+            // We already had an entry for this LSN. That's odd..
+            warn!("Inserting truncation, but had an entry for the LSN already");
+        }
+
+        Ok(())
+    }
+
+    /// Remember that the segment was dropped at given LSN
+    pub fn drop_segment(&self, lsn: Lsn) -> anyhow::Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+
+        assert!(inner.drop_lsn.is_none());
+        inner.drop_lsn = Some(lsn);
+
+        println!("dropped segment {} at {}", self.seg, lsn);
+
+        let end_str = inner
+        .drop_lsn
+        .as_ref()
+        .map(|drop_lsn| drop_lsn.to_string())
+        .unwrap_or_default();
+
+        {
+            let mut result = format!(
+                "----- inmemory layer tli {} for {} {}-{} ----\n",
+                self.timelineid, self.seg, self.start_lsn, end_str
+            );
+
+            for (k, v) in inner.segsizes.iter() {
+                result += &format!("{}: {}\n", k, v);
+            }
+            for (k, v) in inner.page_versions.iter() {
+                result += &format!(
+                    "blk {} at {}: {}/{}\n",
+                    k.0,
+                    k.1,
+                    v.page_image.is_some(),
+                    v.record.is_some()
+                );
+            }
+
+            println!("{}", result);
+        }
+
+        Ok(())
+    }
+
+    ///
+    /// Initialize a new InMemoryLayer for, by copying the state at the given
+    /// point in time from given existing layer.
+    ///
+    pub fn create_successor_layer(
+        conf: &'static PageServerConf,
+        src: Arc<dyn Layer>,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        start_lsn: Lsn,
+        oldest_pending_lsn: Lsn,
+    ) -> Result<InMemoryLayer> {
+        let seg = src.get_seg_tag();
+
+        println!(
+            "initializing new InMemoryLayer for writing {} on timeline {} at {}",
+            seg,
+            timelineid,
+            start_lsn,
+        );
+
+        // For convenience, copy the segment size from the predecessor layer
+        let mut segsizes = BTreeMap::new();
+        if seg.rel.is_blocky() {
+            let size = src.get_seg_size(start_lsn)?;
+            segsizes.insert(start_lsn, size);
+        }
+
+        Ok(InMemoryLayer {
+            conf,
+            timelineid,
+            tenantid,
+            seg,
+            start_lsn,
+            oldest_pending_lsn,
+            inner: Mutex::new(InMemoryLayerInner {
+                drop_lsn: None,
+                page_versions: BTreeMap::new(),
+                segsizes: segsizes,
+            }),
+            predecessor: Some(src),
+        })
+    }
+
+    ///
+    /// Write the this in-memory layer to disk.
+    ///
+    /// The cutoff point for the layer that's written to disk is 'end_lsn'.
+    ///
+    /// Returns new layers that replace this one. Always returns a new image
+    /// layer containing the page versions at the cutoff LSN, that were written
+    /// to disk, and usually also a DeltaLayer that includes all the WAL records
+    /// between start LSN and the cutoff. (The delta layer is not needed when
+    /// a new relish is created with a single LSN, so that the start and end LSN
+    /// are the same.) If there were page versions newer than 'end_lsn', also
+    /// returns a new in-memory layer containing those page versions. The caller
+    /// replaces this layer with the returned layers in the layer map.
+    ///
+    pub fn freeze(
+        &self,
+        cutoff_lsn: Lsn,
+        // This is needed just to call materialize_page()
+        timeline: &LayeredTimeline,
+    ) -> Result<(Vec<Arc<dyn Layer>>, Option<Arc<InMemoryLayer>>)> {
+        println!(
+            "freezing in memory layer for {} on timeline {} at {}",
+            self.seg, self.timelineid, cutoff_lsn
+        );
+
+        let inner = self.inner.lock().unwrap();
+
+        // Normally, use the cutoff LSN as the end of the frozen layer.
+        // But if the relation was dropped, we know that there are no
+        // more changes coming in for it, and in particular we know that
+        // there are no changes "in flight" for the LSN anymore, so we use
+        // the drop LSN instead. The drop-LSN could be ahead of the
+        // caller-specified LSN!
+        let dropped = inner.drop_lsn.is_some();
+        let end_lsn = if dropped {
+            inner.drop_lsn.unwrap()
+        } else {
+            cutoff_lsn
+        };
+
+        // Divide all the page versions into old and new at the 'end_lsn' cutoff point.
+        let mut before_page_versions;
+        let mut before_segsizes;
+        let mut after_page_versions;
+        let mut after_segsizes;
+        if !dropped {
+            before_segsizes = BTreeMap::new();
+            after_segsizes = BTreeMap::new();
+            for (lsn, size) in inner.segsizes.iter() {
+                if *lsn > end_lsn {
+                    after_segsizes.insert(*lsn, *size);
+                } else {
+                    before_segsizes.insert(*lsn, *size);
+                }
+            }
+
+            before_page_versions = BTreeMap::new();
+            after_page_versions = BTreeMap::new();
+            for ((blknum, lsn), pv) in inner.page_versions.iter() {
+                if *lsn == end_lsn {
+                    // Page versions at the cutoff LSN will be stored in the
+                    // materialized image layer.
+                } else if *lsn > end_lsn {
+                    after_page_versions.insert((*blknum, *lsn), pv.clone());
+                } else {
+                    before_page_versions.insert((*blknum, *lsn), pv.clone());
+                }
+            }
+        } else {
+            before_page_versions = inner.page_versions.clone();
+            before_segsizes = inner.segsizes.clone();
+            after_segsizes = BTreeMap::new();
+            after_page_versions = BTreeMap::new();
+        }
+
+        // we can release the lock now.
+        drop(inner);
+
+        let mut frozen_layers: Vec<Arc<dyn Layer>> = Vec::new();
+
+        if self.start_lsn != end_lsn {
+            // Write the page versions before the cutoff to disk.
+            let delta_layer = DeltaLayer::create(
+                self.conf,
+                self.timelineid,
+                self.tenantid,
+                self.seg,
+                self.start_lsn,
+                end_lsn,
+                dropped,
+                self.predecessor.clone(),
+                before_page_versions,
+                before_segsizes,
+            )?;
+            let delta_layer_rc: Arc<dyn Layer> = Arc::new(delta_layer);
+            frozen_layers.push(delta_layer_rc);
+            println!(
+                "freeze: created delta layer {} {}-{}",
+                self.seg,
+                self.start_lsn,
+                end_lsn
+            );
+        } else {
+            assert!(before_page_versions.is_empty());
+        }
+
+        let mut new_open_rc = None;
+        if !dropped {
+            // Write a new base image layer at the cutoff point
+            let imgfile = ImageLayer::create_from_src(self.conf, timeline, self, end_lsn)?;
+            let imgfile_rc: Arc<dyn Layer> = Arc::new(imgfile);
+            frozen_layers.push(Arc::clone(&imgfile_rc));
+            println!("freeze: created image layer {} at {}", self.seg, end_lsn);
+
+            // If there were any page versions newer than the cutoff, initialize a new in-memory
+            // layer to hold them
+            if !after_segsizes.is_empty() || !after_page_versions.is_empty() {
+                let new_open = Self::create_successor_layer(
+                    self.conf,
+                    imgfile_rc,
+                    self.timelineid,
+                    self.tenantid,
+                    end_lsn,
+                    end_lsn,
+                )?;
+                let mut new_inner = new_open.inner.lock().unwrap();
+                new_inner.page_versions.append(&mut after_page_versions);
+                new_inner.segsizes.append(&mut after_segsizes);
+                drop(new_inner);
+                println!("freeze: created new in-mem layer {} {}-", self.seg, end_lsn);
+
+                new_open_rc = Some(Arc::new(new_open))
+            }
+        }
+
+        Ok((frozen_layers, new_open_rc))
+    }
+
+    /// debugging function to print out the contents of the layer
+    #[allow(unused)]
+    pub fn dump(&self) -> String {
+
+
+        let inner = self.inner.lock().unwrap();
+
+        let end_str = inner
+        .drop_lsn
+        .as_ref()
+        .map(|drop_lsn| drop_lsn.to_string())
+        .unwrap_or_default();
+
+        let mut result =  format!(
+            "----- dump inmemory layer for tli {} seg {} {}-{} ----",
+            self.timelineid, self.seg, self.start_lsn, end_str
+        );
+
+
+        for (k, v) in inner.segsizes.iter() {
+            result += &format!("{}: {}\n", k, v);
+        }
+        for (k, v) in inner.page_versions.iter() {
+            result += &format!(
+                "blk {} at {}: {}/{}\n",
+                k.0,
+                k.1,
+                v.page_image.is_some(),
+                v.record.is_some()
+            );
+        }
+
+        println!("inner println {}", result);
+        result
+
+    }
+}

pageserver/src/layered_repository/layer_map.rs

@@ -0,0 +1,387 @@
+//!
+//! The layer map tracks what layers exist for all the relishes in a timeline.
+//!
+//! When the timeline is first accessed, the server lists of all layer files
+//! in the timelines/<timelineid> directory, and populates this map with
+//! ImageLayer and DeltaLayer structs corresponding to each file. When new WAL
+//! is received, we create InMemoryLayers to hold the incoming records. Now and
+//! then, in the checkpoint() function, the in-memory layers are frozen, forming
+//! new image and delta layers and corresponding files are written to disk.
+//!
+
+use crate::layered_repository::storage_layer::{Layer, SegmentTag};
+use crate::layered_repository::InMemoryLayer;
+use crate::relish::*;
+use anyhow::Result;
+use lazy_static::lazy_static;
+use log::*;
+use std::cmp::Ordering;
+use std::collections::HashSet;
+use std::collections::{BTreeMap, BinaryHeap, HashMap};
+use std::ops::Bound::Included;
+use std::sync::Arc;
+use zenith_metrics::{register_int_gauge, IntGauge};
+use zenith_utils::lsn::Lsn;
+
+lazy_static! {
+    static ref NUM_INMEMORY_LAYERS: IntGauge =
+        register_int_gauge!("pageserver_inmemory_layers", "Number of layers in memory")
+            .expect("failed to define a metric");
+    static ref NUM_ONDISK_LAYERS: IntGauge =
+        register_int_gauge!("pageserver_ondisk_layers", "Number of layers on-disk")
+            .expect("failed to define a metric");
+}
+
+///
+/// LayerMap tracks what layers exist on a timeline.
+///
+pub struct LayerMap {
+    /// All the layers keyed by segment tag
+    segs: HashMap<SegmentTag, SegEntry>,
+
+    /// All in-memory layers, ordered by 'oldest_pending_lsn' of each layer.
+    /// This allows easy access to the in-memory layer that contains the
+    /// oldest WAL record.
+    open_segs: BinaryHeap<OpenSegEntry>,
+
+    /// Generation number, used to distinguish newly inserted entries in the
+    /// binary heap from older entries during checkpoint.
+    current_generation: u64,
+}
+
+///
+/// Per-segment entry in the LayerMap.segs hash map
+///
+/// The last layer that is open for writes is always an InMemoryLayer,
+/// and is kept in a separate field, because there can be only one for
+/// each segment. The older layers, stored on disk, are kept in a
+/// BTreeMap keyed by the layer's start LSN.
+struct SegEntry {
+    pub open: Option<Arc<InMemoryLayer>>,
+    pub historic: BTreeMap<Lsn, Arc<dyn Layer>>,
+}
+
+/// Entry held LayerMap.open_segs, with boilerplate comparison
+/// routines to implement a min-heap ordered by 'oldest_pending_lsn'
+///
+/// Each entry also carries a generation number. It can be used to distinguish
+/// entries with the same 'oldest_pending_lsn'.
+struct OpenSegEntry {
+    pub oldest_pending_lsn: Lsn,
+    pub layer: Arc<InMemoryLayer>,
+    pub generation: u64,
+}
+impl Ord for OpenSegEntry {
+    fn cmp(&self, other: &Self) -> Ordering {
+        // BinaryHeap is a max-heap, and we want a min-heap. Reverse the ordering here
+        // to get that.
+        other.oldest_pending_lsn.cmp(&self.oldest_pending_lsn)
+    }
+}
+impl PartialOrd for OpenSegEntry {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        // BinaryHeap is a max-heap, and we want a min-heap. Reverse the ordering here
+        // to get that. Entries with identical oldest_pending_lsn are ordered by generation
+        Some(
+            other
+                .oldest_pending_lsn
+                .cmp(&self.oldest_pending_lsn)
+                .then_with(|| other.generation.cmp(&self.generation)),
+        )
+    }
+}
+impl PartialEq for OpenSegEntry {
+    fn eq(&self, other: &Self) -> bool {
+        self.oldest_pending_lsn.eq(&other.oldest_pending_lsn)
+    }
+}
+impl Eq for OpenSegEntry {}
+
+impl LayerMap {
+    ///
+    /// Look up a layer using the given segment tag and LSN. This differs from a
+    /// plain key-value lookup in that if there is any layer that covers the
+    /// given LSN, or precedes the given LSN, it is returned. In other words,
+    /// you don't need to know the exact start LSN of the layer.
+    ///
+    pub fn get(&self, tag: &SegmentTag, lsn: Lsn) -> Option<Arc<dyn Layer>> {
+        let segentry = self.segs.get(tag)?;
+
+        if let Some(open) = &segentry.open {
+            if open.get_start_lsn() <= lsn {
+                let x: Arc<dyn Layer> = Arc::clone(&open) as _;
+                return Some(x);
+            }
+        }
+
+        if let Some((_k, v)) = segentry
+            .historic
+            .range((Included(Lsn(0)), Included(lsn)))
+            .next_back()
+        {
+            let x: Arc<dyn Layer> = Arc::clone(&v) as _;
+            Some(x)
+        } else {
+            None
+        }
+    }
+
+    ///
+    /// Get the open layer for given segment for writing. Or None if no open
+    /// layer exists.
+    ///
+    pub fn get_open(&self, tag: &SegmentTag) -> Option<Arc<InMemoryLayer>> {
+        let segentry = self.segs.get(tag)?;
+
+        if let Some(open) = &segentry.open {
+            Some(Arc::clone(open))
+        } else {
+            None
+        }
+    }
+
+    ///
+    /// Insert an open in-memory layer
+    ///
+    pub fn insert_open(&mut self, layer: Arc<InMemoryLayer>) {
+        let tag = layer.get_seg_tag();
+
+        if let Some(segentry) = self.segs.get_mut(&tag) {
+            if let Some(_old) = &segentry.open {
+                // FIXME: shouldn't exist, but check
+                println!("insert_open() // FIXME: shouldn't exist, but check");
+            }
+            segentry.open = Some(Arc::clone(&layer));
+        } else {
+            let segentry = SegEntry {
+                open: Some(Arc::clone(&layer)),
+                historic: BTreeMap::new(),
+            };
+            self.segs.insert(tag, segentry);
+        }
+
+        let opensegentry = OpenSegEntry {
+            oldest_pending_lsn: layer.get_oldest_pending_lsn(),
+            layer: layer,
+            generation: self.current_generation,
+        };
+        self.open_segs.push(opensegentry);
+
+        NUM_INMEMORY_LAYERS.inc();
+    }
+
+    /// Remove the oldest in-memory layer
+    pub fn pop_oldest_open(&mut self) {
+        let opensegentry = self.open_segs.pop().unwrap();
+        let segtag = opensegentry.layer.get_seg_tag();
+
+        let mut segentry = self.segs.get_mut(&segtag).unwrap();
+        segentry.open = None;
+        NUM_INMEMORY_LAYERS.dec();
+    }
+
+    ///
+    /// Insert an on-disk layer
+    ///
+    pub fn insert_historic(&mut self, layer: Arc<dyn Layer>) {
+        let tag = layer.get_seg_tag();
+        let start_lsn = layer.get_start_lsn();
+
+        if let Some(segentry) = self.segs.get_mut(&tag) {
+            segentry.historic.insert(start_lsn, layer);
+        } else {
+            let mut historic = BTreeMap::new();
+            historic.insert(start_lsn, layer);
+
+            let segentry = SegEntry {
+                open: None,
+                historic,
+            };
+            self.segs.insert(tag, segentry);
+        }
+        NUM_ONDISK_LAYERS.inc();
+    }
+
+    ///
+    /// Remove an on-disk layer from the map.
+    ///
+    /// This should be called when the corresponding file on disk has been deleted.
+    ///
+    pub fn remove_historic(&mut self, layer: &dyn Layer) {
+        let tag = layer.get_seg_tag();
+        let start_lsn = layer.get_start_lsn();
+
+        if let Some(segentry) = self.segs.get_mut(&tag) {
+            segentry.historic.remove(&start_lsn);
+        }
+        NUM_ONDISK_LAYERS.dec();
+    }
+
+    // List relations that exist at the lsn
+    pub fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashSet<RelTag>> {
+        let mut rels: HashSet<RelTag> = HashSet::new();
+
+        println!("layer list_rels()");
+
+        for (seg, segentry) in self.segs.iter() {
+            if let RelishTag::Relation(reltag) = seg.rel {
+                if (spcnode == 0 || reltag.spcnode == spcnode)
+                    && (dbnode == 0 || reltag.dbnode == dbnode)
+                {
+                    // Add only if it exists at the requested LSN.
+                    if let Some(open) = &segentry.open {
+                        println!("explore segentry.open {:?}", open.filename());
+
+                        if open.get_end_lsn() > lsn && open.get_start_lsn() <= lsn {
+                            rels.insert(reltag);
+                            println!("found rel {} at lsn {} in open layer start_lsn {} end lsn {}",
+                             reltag, lsn, open.get_start_lsn(), open.get_end_lsn());
+                            println!("{}", open.dump());
+
+                        }
+                        else
+                        {
+                            println!("found DROPPED rel {} at lsn {} in open layer start_lsn {} end lsn {}",
+                            reltag, lsn, open.get_start_lsn(), open.get_end_lsn());
+                            println!("{}", open.dump());  
+                        }
+                    } else if let Some((l_start_lsn, layer)) = segentry
+                        .historic
+                        .range((Included(Lsn(0)), Included(lsn)))
+                        .next_back()
+                    {
+                        println!("explore historic segment {:?}", layer.filename());
+
+                        if !layer.is_dropped() && l_start_lsn <= &lsn {
+                            rels.insert(reltag);
+                            println!("found rel {} in historic layer", reltag);
+                            layer.dump()?;
+                        }
+                    }
+                }
+            }
+        }
+        Ok(rels)
+    }
+
+    // List non-relation relishes that exist at the lsn
+    pub fn list_nonrels(&self, lsn: Lsn) -> Result<HashSet<RelishTag>> {
+        let mut rels: HashSet<RelishTag> = HashSet::new();
+
+        // Scan the timeline directory to get all rels in this timeline.
+        for (seg, segentry) in self.segs.iter() {
+            if let RelishTag::Relation(_) = seg.rel {
+            } else {
+                // Add only if it exists at the requested LSN.
+                if let Some(open) = &segentry.open {
+                    if open.get_end_lsn() > lsn && open.get_start_lsn() <= lsn {
+                        rels.insert(seg.rel);
+                    }
+                } else if let Some((l_start_lsn, layer)) = segentry
+                    .historic
+                    .range((Included(Lsn(0)), Included(lsn)))
+                    .next_back()
+                {
+                    if !layer.is_dropped() && l_start_lsn <= &lsn {
+                        rels.insert(seg.rel);
+                    }
+                }
+            }
+        }
+        Ok(rels)
+    }
+
+    /// Is there a newer image layer for given segment?
+    ///
+    /// This is used for garbage collection, to determine if an old layer can
+    /// be deleted.
+    pub fn newer_image_layer_exists(&self, seg: SegmentTag, lsn: Lsn) -> bool {
+        if let Some(segentry) = self.segs.get(&seg) {
+            // We only check on-disk layers, because
+            // in-memory layers are not durable
+            for (newer_lsn, layer) in segentry
+                .historic
+                .range((Included(lsn), Included(Lsn(u64::MAX))))
+            {
+                // Ignore layers that depend on an older layer.
+                if layer.is_incremental() {
+                    continue;
+                }
+                if layer.get_end_lsn() > lsn {
+                    trace!(
+                        "found later layer for {}, {} {}-{}",
+                        seg,
+                        lsn,
+                        newer_lsn,
+                        layer.get_end_lsn()
+                    );
+                    return true;
+                } else {
+                    trace!("found singleton layer for {}, {} {}", seg, lsn, newer_lsn);
+                    continue;
+                }
+            }
+        }
+        trace!("no later layer found for {}, {}", seg, lsn);
+        false
+    }
+
+    /// Return the oldest in-memory layer, along with its generation number.
+    pub fn peek_oldest_open(&self) -> Option<(Arc<InMemoryLayer>, u64)> {
+        if let Some(opensegentry) = self.open_segs.peek() {
+            Some((Arc::clone(&opensegentry.layer), opensegentry.generation))
+        } else {
+            None
+        }
+    }
+
+    /// Increment the generation number used to stamp open in-memory layers. Layers
+    /// added with `insert_open` after this call will be associated with the new
+    /// generation. Returns the new generation number.
+    pub fn increment_generation(&mut self) -> u64 {
+        self.current_generation += 1;
+        self.current_generation
+    }
+
+    pub fn iter_historic_layers(&self) -> HistoricLayerIter {
+        HistoricLayerIter {
+            segiter: self.segs.iter(),
+            iter: None,
+        }
+    }
+}
+
+impl Default for LayerMap {
+    fn default() -> Self {
+        LayerMap {
+            segs: HashMap::new(),
+            open_segs: BinaryHeap::new(),
+            current_generation: 0,
+        }
+    }
+}
+
+pub struct HistoricLayerIter<'a> {
+    segiter: std::collections::hash_map::Iter<'a, SegmentTag, SegEntry>,
+    iter: Option<std::collections::btree_map::Iter<'a, Lsn, Arc<dyn Layer>>>,
+}
+
+impl<'a> Iterator for HistoricLayerIter<'a> {
+    type Item = Arc<dyn Layer>;
+
+    fn next(&mut self) -> std::option::Option<<Self as std::iter::Iterator>::Item> {
+        loop {
+            if let Some(x) = &mut self.iter {
+                if let Some(x) = x.next() {
+                    return Some(Arc::clone(&*x.1));
+                }
+            }
+            if let Some(seg) = self.segiter.next() {
+                self.iter = Some(seg.1.historic.iter());
+                continue;
+            } else {
+                return None;
+            }
+        }
+    }
+}

pageserver/src/layered_repository/storage_layer.rs

@@ -0,0 +1,176 @@
+//!
+//! Common traits and structs for layers
+//!
+
+use crate::relish::RelishTag;
+use crate::repository::WALRecord;
+use crate::ZTimelineId;
+use anyhow::Result;
+use bytes::Bytes;
+use serde::{Deserialize, Serialize};
+use std::fmt;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use zenith_utils::lsn::Lsn;
+
+// Size of one segment in pages (10 MB)
+pub const RELISH_SEG_SIZE: u32 = 10 * 1024 * 1024 / 8192;
+
+///
+/// Each relish stored in the repository is divided into fixed-sized "segments",
+/// with 10 MB of key-space, or 1280 8k pages each.
+///
+#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy)]
+pub struct SegmentTag {
+    pub rel: RelishTag,
+    pub segno: u32,
+}
+
+impl fmt::Display for SegmentTag {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}.{}", self.rel, self.segno)
+    }
+}
+
+impl SegmentTag {
+    pub const fn from_blknum(rel: RelishTag, blknum: u32) -> SegmentTag {
+        SegmentTag {
+            rel,
+            segno: blknum / RELISH_SEG_SIZE,
+        }
+    }
+
+    pub fn blknum_in_seg(&self, blknum: u32) -> bool {
+        blknum / RELISH_SEG_SIZE == self.segno
+    }
+}
+
+///
+/// Represents a version of a page at a specific LSN. The LSN is the key of the
+/// entry in the 'page_versions' hash, it is not duplicated here.
+///
+/// A page version can be stored as a full page image, or as WAL record that needs
+/// to be applied over the previous page version to reconstruct this version.
+///
+/// It's also possible to have both a WAL record and a page image in the same
+/// PageVersion. That happens if page version is originally stored as a WAL record
+/// but it is later reconstructed by a GetPage@LSN request by performing WAL
+/// redo. The get_page_at_lsn() code will store the reconstructed pag image next to
+/// the WAL record in that case. TODO: That's pretty accidental, not the result
+/// of any grand design. If we want to keep reconstructed page versions around, we
+/// probably should have a separate buffer cache so that we could control the
+/// replacement policy globally. Or if we keep a reconstructed page image, we
+/// could throw away the WAL record.
+///
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PageVersion {
+    /// an 8kb page image
+    pub page_image: Option<Bytes>,
+    /// WAL record to get from previous page version to this one.
+    pub record: Option<WALRecord>,
+}
+
+///
+/// Data needed to reconstruct a page version
+///
+/// 'page_img' is the old base image of the page to start the WAL replay with.
+/// It can be None, if the first WAL record initializes the page (will_init)
+/// 'records' contains the records to apply over the base image.
+///
+pub struct PageReconstructData {
+    pub records: Vec<WALRecord>,
+    pub page_img: Option<Bytes>,
+}
+
+/// Return value from Layer::get_page_reconstruct_data
+pub enum PageReconstructResult {
+    /// Got all the data needed to reconstruct the requested page
+    Complete,
+    /// This layer didn't contain all the required data, the caller should collect
+    /// more data from the returned predecessor layer at the returned LSN.
+    Continue(Lsn, Arc<dyn Layer>),
+    /// This layer didn't contain data needed to reconstruct the page version at
+    /// the returned LSN. This is usually considered an error, but might be OK
+    /// in some circumstances.
+    Missing(Lsn),
+}
+
+///
+/// A Layer corresponds to one RELISH_SEG_SIZE slice of a relish in a range of LSNs.
+/// There are two kinds of layers, in-memory and on-disk layers. In-memory
+/// layers are used to ingest incoming WAL, and provide fast access
+/// to the recent page versions. On-disk layers are stored as files on disk, and
+/// are immutable. This trait presents the common functionality of
+/// in-memory and on-disk layers.
+///
+pub trait Layer: Send + Sync {
+    /// Identify the timeline this relish belongs to
+    fn get_timeline_id(&self) -> ZTimelineId;
+
+    /// Identify the relish segment
+    fn get_seg_tag(&self) -> SegmentTag;
+
+    /// Inclusive start bound of the LSN range that this layer hold
+    fn get_start_lsn(&self) -> Lsn;
+
+    /// 'end_lsn' meaning depends on the layer kind:
+    /// - in-memory layer is either unbounded (end_lsn = MAX_LSN) or dropped (end_lsn = drop_lsn)
+    /// - image layer represents snapshot at one LSN, so end_lsn = lsn
+    /// - delta layer has end_lsn
+    ///
+    /// TODO Is end_lsn always exclusive for all layer kinds?
+    fn get_end_lsn(&self) -> Lsn;
+
+    /// Is the segment represented by this layer dropped by PostgreSQL?
+    fn is_dropped(&self) -> bool;
+
+    /// Filename used to store this layer on disk. (Even in-memory layers
+    /// implement this, to print a handy unique identifier for the layer for
+    /// log messages, even though they're never not on disk.)
+    fn filename(&self) -> PathBuf;
+
+    ///
+    /// Return data needed to reconstruct given page at LSN.
+    ///
+    /// It is up to the caller to collect more data from previous layer and
+    /// perform WAL redo, if necessary.
+    ///
+    /// Note that the 'blknum' is the offset of the page from the beginning
+    /// of the *relish*, not the beginning of the segment. The requested
+    /// 'blknum' must be covered by this segment.
+    ///
+    /// See PageReconstructResult for possible return values. The collected data
+    /// is appended to reconstruct_data; the caller should pass an empty struct
+    /// on first call. If this returns PageReconstructResult::Continue, call
+    /// again on the returned predecessor layer with the same 'reconstruct_data'
+    /// to collect more data.
+    fn get_page_reconstruct_data(
+        &self,
+        blknum: u32,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<PageReconstructResult>;
+
+    /// Return size of the segment at given LSN. (Only for blocky relations.)
+    fn get_seg_size(&self, lsn: Lsn) -> Result<u32>;
+
+    /// Does the segment exist at given LSN? Or was it dropped before it.
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool>;
+
+    /// Does this layer only contain some data for the segment (incremental),
+    /// or does it contain a version of every page? This is important to know
+    /// for garbage collecting old layers: an incremental layer depends on
+    /// the previous non-incremental layer.
+    fn is_incremental(&self) -> bool;
+
+    /// Release memory used by this layer. There is no corresponding 'load'
+    /// function, that's done implicitly when you call one of the get-functions.
+    fn unload(&self) -> Result<()>;
+
+    /// Permanently remove this layer from disk.
+    fn delete(&self) -> Result<()>;
+
+    /// Dump summary of the contents of the layer to stdout
+    fn dump(&self) -> Result<()>;
+}

pageserver/src/lib.rs

@@ -1,34 +1,43 @@
-use serde::{Deserialize, Serialize};
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
-use std::fmt;
-use std::net::SocketAddr;
 use std::path::PathBuf;
-use std::str::FromStr;
 use std::time::Duration;
 
+use lazy_static::lazy_static;
+use zenith_metrics::{register_int_gauge_vec, IntGaugeVec};
+
 pub mod basebackup;
 pub mod branches;
-pub mod object_repository;
-pub mod object_store;
-pub mod page_cache;
+pub mod http;
+pub mod layered_repository;
+pub mod logger;
 pub mod page_service;
+pub mod relish;
 pub mod repository;
 pub mod restore_local_repo;
-pub mod rocksdb_storage;
-pub mod tui;
-pub mod tui_event;
-mod tui_logger;
+pub mod tenant_mgr;
 pub mod waldecoder;
 pub mod walreceiver;
 pub mod walredo;
 
+lazy_static! {
+    static ref LIVE_CONNECTIONS_COUNT: IntGaugeVec = register_int_gauge_vec!(
+        "pageserver_live_connections_count",
+        "Number of live network connections",
+        &["pageserver_connection_kind"]
+    )
+    .expect("failed to define a metric");
+}
+
 #[derive(Debug, Clone)]
 pub struct PageServerConf {
     pub daemonize: bool,
-    pub interactive: bool,
-    pub listen_addr: SocketAddr,
+    pub listen_addr: String,
+    pub http_endpoint_addr: String,
     pub gc_horizon: u64,
     pub gc_period: Duration,
+    pub superuser: String,
 
     // Repository directory, relative to current working directory.
     // Normally, the page server changes the current working directory
@@ -39,6 +48,10 @@ pub struct PageServerConf {
     pub workdir: PathBuf,
 
     pub pg_distrib_dir: PathBuf,
+
+    pub auth_type: AuthType,
+
+    pub auth_validation_public_key_path: Option<PathBuf>,
 }
 
 impl PageServerConf {
@@ -46,24 +59,44 @@ impl PageServerConf {
     // Repository paths, relative to workdir.
     //
 
-    fn tag_path(&self, name: &str) -> PathBuf {
-        self.workdir.join("refs").join("tags").join(name)
+    fn tenants_path(&self) -> PathBuf {
+        self.workdir.join("tenants")
     }
 
-    fn branch_path(&self, name: &str) -> PathBuf {
-        self.workdir.join("refs").join("branches").join(name)
+    fn tenant_path(&self, tenantid: &ZTenantId) -> PathBuf {
+        self.tenants_path().join(tenantid.to_string())
     }
 
-    fn timeline_path(&self, timelineid: ZTimelineId) -> PathBuf {
-        self.workdir.join("timelines").join(timelineid.to_string())
+    fn tags_path(&self, tenantid: &ZTenantId) -> PathBuf {
+        self.tenant_path(tenantid).join("refs").join("tags")
     }
 
-    fn snapshots_path(&self, timelineid: ZTimelineId) -> PathBuf {
-        self.timeline_path(timelineid).join("snapshots")
+    fn tag_path(&self, tag_name: &str, tenantid: &ZTenantId) -> PathBuf {
+        self.tags_path(tenantid).join(tag_name)
     }
 
-    fn ancestor_path(&self, timelineid: ZTimelineId) -> PathBuf {
-        self.timeline_path(timelineid).join("ancestor")
+    fn branches_path(&self, tenantid: &ZTenantId) -> PathBuf {
+        self.tenant_path(tenantid).join("refs").join("branches")
+    }
+
+    fn branch_path(&self, branch_name: &str, tenantid: &ZTenantId) -> PathBuf {
+        self.branches_path(tenantid).join(branch_name)
+    }
+
+    fn timelines_path(&self, tenantid: &ZTenantId) -> PathBuf {
+        self.tenant_path(tenantid).join("timelines")
+    }
+
+    fn timeline_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
+        self.timelines_path(tenantid).join(timelineid.to_string())
+    }
+
+    fn ancestor_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
+        self.timeline_path(timelineid, tenantid).join("ancestor")
+    }
+
+    fn wal_dir_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
+        self.timeline_path(timelineid, tenantid).join("wal")
     }
 
     //
@@ -78,64 +111,3 @@ impl PageServerConf {
         self.pg_distrib_dir.join("lib")
     }
 }
-
-/// Zenith Timeline ID is a 128-bit random ID.
-///
-/// Zenith timeline IDs are different from PostgreSQL timeline
-/// IDs. They serve a similar purpose though: they differentiate
-/// between different "histories" of the same cluster.  However,
-/// PostgreSQL timeline IDs are a bit cumbersome, because they are only
-/// 32-bits wide, and they must be in ascending order in any given
-/// timeline history.  Those limitations mean that we cannot generate a
-/// new PostgreSQL timeline ID by just generating a random number. And
-/// that in turn is problematic for the "pull/push" workflow, where you
-/// have a local copy of a zenith repository, and you periodically sync
-/// the local changes with a remote server. When you work "detached"
-/// from the remote server, you cannot create a PostgreSQL timeline ID
-/// that's guaranteed to be different from all existing timelines in
-/// the remote server. For example, if two people are having a clone of
-/// the repository on their laptops, and they both create a new branch
-/// with different name. What timeline ID would they assign to their
-/// branches? If they pick the same one, and later try to push the
-/// branches to the same remote server, they will get mixed up.
-///
-/// To avoid those issues, Zenith has its own concept of timelines that
-/// is separate from PostgreSQL timelines, and doesn't have those
-/// limitations. A zenith timeline is identified by a 128-bit ID, which
-/// is usually printed out as a hex string.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
-pub struct ZTimelineId([u8; 16]);
-
-impl FromStr for ZTimelineId {
-    type Err = hex::FromHexError;
-
-    fn from_str(s: &str) -> Result<ZTimelineId, Self::Err> {
-        let timelineid = hex::decode(s)?;
-
-        let mut buf: [u8; 16] = [0u8; 16];
-        buf.copy_from_slice(timelineid.as_slice());
-        Ok(ZTimelineId(buf))
-    }
-}
-
-impl ZTimelineId {
-    pub fn from(b: [u8; 16]) -> ZTimelineId {
-        ZTimelineId(b)
-    }
-
-    pub fn get_from_buf(buf: &mut dyn bytes::Buf) -> ZTimelineId {
-        let mut arr = [0u8; 16];
-        buf.copy_to_slice(&mut arr);
-        ZTimelineId::from(arr)
-    }
-
-    pub fn as_arr(&self) -> [u8; 16] {
-        self.0
-    }
-}
-
-impl fmt::Display for ZTimelineId {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_str(&hex::encode(self.0))
-    }
-}

pageserver/src/logger.rs

@@ -0,0 +1,45 @@
+use crate::PageServerConf;
+
+use anyhow::{Context, Result};
+use slog::{Drain, FnValue};
+use std::fs::{File, OpenOptions};
+
+pub fn init_logging(
+    _conf: &PageServerConf,
+    log_filename: &str,
+) -> Result<(slog_scope::GlobalLoggerGuard, File)> {
+    // Don't open the same file for output multiple times;
+    // the different fds could overwrite each other's output.
+    let log_file = OpenOptions::new()
+        .create(true)
+        .append(true)
+        .open(&log_filename)
+        .with_context(|| format!("failed to open {:?}", &log_filename))?;
+
+    let logger_file = log_file.try_clone().unwrap();
+
+    let decorator = slog_term::PlainSyncDecorator::new(logger_file);
+    let drain = slog_term::FullFormat::new(decorator).build();
+    let drain = slog::Filter::new(drain, |record: &slog::Record| {
+        if record.level().is_at_least(slog::Level::Info) {
+            return true;
+        }
+        true
+    });
+    let drain = std::sync::Mutex::new(drain).fuse();
+    let logger = slog::Logger::root(
+        drain,
+        slog::o!(
+            "location" =>
+                FnValue(move |record| {
+                    format!("{}, {}:{}",
+                            record.module(),
+                            record.file(),
+                            record.line()
+                    )
+                }
+                )
+        ),
+    );
+    Ok((slog_scope::set_global_logger(logger), log_file))
+}

pageserver/src/object_repository.rs

@@ -1,956 +0,0 @@
-//!
-//! Implementation of the Repository/Timeline traits, using a key-value store
-//! (ObjectStore) to for the actual storage.
-//!
-//! This maps the relation-oriented operations in the Timeline interface into
-//! objects stored in an ObjectStore. Relation size is stored as a separate object
-//! in the key-value store. If a page is written beyond the current end-of-file,
-//! we also insert the new size as a new "page version" in the key-value store.
-//!
-//! Also, this implements Copy-on-Write forking of timelines. For each timeline,
-//! we store the parent timeline in the object store, in a little metadata blob.
-//! When we need to find a version of a page, we walk the timeline history backwards
-//! until we find the page we're looking for, making a separate lookup into the
-//! key-value store for each timeline.
-
-use crate::object_store::{ObjectKey, ObjectStore};
-use crate::repository::*;
-use crate::restore_local_repo::import_timeline_wal;
-use crate::walredo::WalRedoManager;
-use crate::{PageServerConf, ZTimelineId};
-use anyhow::{bail, Context, Result};
-use bytes::Bytes;
-use log::*;
-use postgres_ffi::pg_constants;
-use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
-use std::collections::HashSet;
-use std::convert::TryInto;
-use std::sync::{Arc, Mutex};
-use std::thread;
-use std::time::Duration;
-use zenith_utils::bin_ser::BeSer;
-use zenith_utils::lsn::{AtomicLsn, Lsn};
-use zenith_utils::seqwait::SeqWait;
-
-///
-/// A repository corresponds to one .zenith directory. One repository holds multiple
-/// timelines, forked off from the same initial call to 'initdb'.
-///
-pub struct ObjectRepository {
-    obj_store: Arc<dyn ObjectStore>,
-    conf: &'static PageServerConf,
-    timelines: Mutex<HashMap<ZTimelineId, Arc<ObjectTimeline>>>,
-    walredo_mgr: Arc<dyn WalRedoManager>,
-}
-
-// Timeout when waiting or WAL receiver to catch up to an LSN given in a GetPage@LSN call.
-static TIMEOUT: Duration = Duration::from_secs(600);
-
-impl ObjectRepository {
-    pub fn new(
-        conf: &'static PageServerConf,
-        obj_store: Arc<dyn ObjectStore>,
-        walredo_mgr: Arc<dyn WalRedoManager>,
-    ) -> ObjectRepository {
-        ObjectRepository {
-            conf,
-            obj_store,
-            timelines: Mutex::new(HashMap::new()),
-            walredo_mgr,
-        }
-    }
-}
-
-impl Repository for ObjectRepository {
-    /// Get Timeline handle for given zenith timeline ID.
-    fn get_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>> {
-        let mut timelines = self.timelines.lock().unwrap();
-
-        match timelines.get(&timelineid) {
-            Some(timeline) => Ok(timeline.clone()),
-            None => {
-                let timeline = ObjectTimeline::open(
-                    Arc::clone(&self.obj_store),
-                    timelineid,
-                    self.walredo_mgr.clone(),
-                )?;
-
-                // Load any new WAL after the last checkpoint into the repository.
-                info!(
-                    "Loading WAL for timeline {} starting at {}",
-                    timelineid,
-                    timeline.get_last_record_lsn()
-                );
-                let wal_dir = self.conf.timeline_path(timelineid).join("wal");
-                import_timeline_wal(&wal_dir, &timeline, timeline.get_last_record_lsn())?;
-
-                let timeline_rc = Arc::new(timeline);
-
-                if self.conf.gc_horizon != 0 {
-                    ObjectTimeline::launch_gc_thread(self.conf, timeline_rc.clone());
-                }
-
-                timelines.insert(timelineid, timeline_rc.clone());
-
-                Ok(timeline_rc)
-            }
-        }
-    }
-
-    /// Create a new, empty timeline. The caller is responsible for loading data into it
-    fn create_empty_timeline(
-        &self,
-        timelineid: ZTimelineId,
-        start_lsn: Lsn,
-    ) -> Result<Arc<dyn Timeline>> {
-        let mut timelines = self.timelines.lock().unwrap();
-
-        // Write metadata key
-        let metadata = MetadataEntry {
-            last_valid_lsn: start_lsn,
-            last_record_lsn: start_lsn,
-            ancestor_timeline: None,
-            ancestor_lsn: start_lsn,
-        };
-        self.obj_store.put(
-            &timeline_metadata_key(timelineid),
-            Lsn(0),
-            &MetadataEntry::ser(&metadata)?,
-        )?;
-
-        info!("Created empty timeline {}", timelineid);
-
-        let timeline = ObjectTimeline::open(
-            Arc::clone(&self.obj_store),
-            timelineid,
-            self.walredo_mgr.clone(),
-        )?;
-
-        let timeline_rc = Arc::new(timeline);
-        let r = timelines.insert(timelineid, timeline_rc.clone());
-        assert!(r.is_none());
-
-        // don't start the garbage collector for unit tests, either.
-
-        Ok(timeline_rc)
-    }
-
-    /// Branch a timeline
-    fn branch_timeline(&self, src: ZTimelineId, dst: ZTimelineId, at_lsn: Lsn) -> Result<()> {
-        // just to check the source timeline exists
-        let _ = self.get_timeline(src)?;
-
-        // Write a metadata key, noting the ancestor of th new timeline. There is initially
-        // no data in it, but all the read-calls know to look into the ancestor.
-        let metadata = MetadataEntry {
-            last_valid_lsn: at_lsn,
-            last_record_lsn: at_lsn,
-            ancestor_timeline: Some(src),
-            ancestor_lsn: at_lsn,
-        };
-        self.obj_store.put(
-            &timeline_metadata_key(dst),
-            Lsn(0),
-            &MetadataEntry::ser(&metadata)?,
-        )?;
-
-        Ok(())
-    }
-}
-
-///
-/// A handle to a specific timeline in the repository. This is the API
-/// that's exposed to the rest of the system.
-///
-pub struct ObjectTimeline {
-    timelineid: ZTimelineId,
-
-    // Backing key-value store
-    obj_store: Arc<dyn ObjectStore>,
-
-    // WAL redo manager, for reconstructing page versions from WAL records.
-    walredo_mgr: Arc<dyn WalRedoManager>,
-
-    // What page versions do we hold in the cache? If we get a request > last_valid_lsn,
-    // we need to wait until we receive all the WAL up to the request. The SeqWait
-    // provides functions for that. TODO: If we get a request for an old LSN, such that
-    // the versions have already been garbage collected away, we should throw an error,
-    // but we don't track that currently.
-    //
-    // last_record_lsn points to the end of last processed WAL record.
-    // It can lag behind last_valid_lsn, if the WAL receiver has received some WAL
-    // after the end of last record, but not the whole next record yet. In the
-    // page cache, we care about last_valid_lsn, but if the WAL receiver needs to
-    // restart the streaming, it needs to restart at the end of last record, so
-    // we track them separately. last_record_lsn should perhaps be in
-    // walreceiver.rs instead of here, but it seems convenient to keep all three
-    // values together.
-    //
-    last_valid_lsn: SeqWait<Lsn>,
-    last_record_lsn: AtomicLsn,
-
-    ancestor_timeline: Option<ZTimelineId>,
-    ancestor_lsn: Lsn,
-}
-
-impl ObjectTimeline {
-    /// Open a Timeline handle.
-    ///
-    /// Loads the metadata for the timeline into memory.
-    fn open(
-        obj_store: Arc<dyn ObjectStore>,
-        timelineid: ZTimelineId,
-        walredo_mgr: Arc<dyn WalRedoManager>,
-    ) -> Result<ObjectTimeline> {
-        // Load metadata into memory
-        let v = obj_store
-            .get(&timeline_metadata_key(timelineid), Lsn(0))
-            .with_context(|| "timeline not found in repository")?;
-        let metadata = MetadataEntry::des(&v)?;
-
-        let timeline = ObjectTimeline {
-            timelineid,
-            obj_store,
-            walredo_mgr,
-            last_valid_lsn: SeqWait::new(metadata.last_valid_lsn),
-            last_record_lsn: AtomicLsn::new(metadata.last_record_lsn.0),
-            ancestor_timeline: metadata.ancestor_timeline,
-            ancestor_lsn: metadata.ancestor_lsn,
-        };
-        Ok(timeline)
-    }
-}
-
-impl Timeline for ObjectTimeline {
-    //------------------------------------------------------------------------------
-    // Public GET functions
-    //------------------------------------------------------------------------------
-
-    /// Look up given page in the cache.
-    fn get_page_at_lsn(&self, tag: BufferTag, req_lsn: Lsn) -> Result<Bytes> {
-        let lsn = self.wait_lsn(req_lsn)?;
-
-        self.get_page_at_lsn_nowait(tag, lsn)
-    }
-
-    /// Get size of relation
-    fn get_rel_size(&self, rel: RelTag, lsn: Lsn) -> Result<u32> {
-        let lsn = self.wait_lsn(lsn)?;
-
-        match self.relsize_get_nowait(rel, lsn)? {
-            Some(nblocks) => Ok(nblocks),
-            None => bail!("relation {} not found at {}", rel, lsn),
-        }
-    }
-
-    /// Does relation exist at given LSN?
-    fn get_rel_exists(&self, rel: RelTag, req_lsn: Lsn) -> Result<bool> {
-        let lsn = self.wait_lsn(req_lsn)?;
-        let key = relation_size_key(self.timelineid, rel);
-        let mut iter = self.object_versions(&*self.obj_store, &key, lsn)?;
-        if let Some((_key, _val)) = iter.next().transpose()? {
-            debug!("Relation {} exists at {}", rel, lsn);
-            return Ok(true);
-        }
-        debug!("Relation {} doesn't exist at {}", rel, lsn);
-        Ok(false)
-    }
-
-    /// Get a list of all distinct relations in given tablespace and database.
-    fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashSet<RelTag>> {
-        // List all relations in this timeline.
-        let mut all_rels = self
-            .obj_store
-            .list_rels(self.timelineid, spcnode, dbnode, lsn)?;
-
-        // Also list all relations in ancestor timelines. If a relation hasn't been modified
-        // after the fork, there will be no trace of it in the object store with the current
-        // timeline id.
-        let mut prev_timeline: Option<ZTimelineId> = self.ancestor_timeline;
-        let mut lsn = self.ancestor_lsn;
-        while let Some(timeline) = prev_timeline {
-            let this_rels = self.obj_store.list_rels(timeline, spcnode, dbnode, lsn)?;
-
-            for rel in this_rels {
-                all_rels.insert(rel);
-            }
-
-            // Load ancestor metadata.
-            let v = self
-                .obj_store
-                .get(&timeline_metadata_key(timeline), Lsn(0))
-                .with_context(|| "timeline not found in repository")?;
-            let metadata = MetadataEntry::des(&v)?;
-
-            prev_timeline = metadata.ancestor_timeline;
-            lsn = metadata.ancestor_lsn;
-        }
-
-        Ok(all_rels)
-    }
-
-    //------------------------------------------------------------------------------
-    // Public PUT functions, to update the repository with new page versions.
-    //
-    // These are called by the WAL receiver to digest WAL records.
-    //------------------------------------------------------------------------------
-
-    /// Put a new page version that can be constructed from a WAL record
-    ///
-    /// This will implicitly extend the relation, if the page is beyond the
-    /// current end-of-file.
-    fn put_wal_record(&self, tag: BufferTag, rec: WALRecord) -> Result<()> {
-        let lsn = rec.lsn;
-        let key = ObjectKey {
-            timeline: self.timelineid,
-            buf_tag: tag,
-        };
-        let val = PageEntry::WALRecord(rec);
-
-        self.obj_store.put(&key, lsn, &PageEntry::ser(&val)?)?;
-        debug!(
-            "put_wal_record rel {} blk {} at {}",
-            tag.rel, tag.blknum, lsn
-        );
-
-        // Also check if this created or extended the file
-        let old_nblocks = self.relsize_get_nowait(tag.rel, lsn)?.unwrap_or(0);
-
-        if tag.blknum >= old_nblocks {
-            let new_nblocks = tag.blknum + 1;
-            let key = relation_size_key(self.timelineid, tag.rel);
-            let val = RelationSizeEntry::Size(new_nblocks);
-
-            trace!(
-                "Extended relation {} from {} to {} blocks at {}",
-                tag.rel,
-                old_nblocks,
-                new_nblocks,
-                lsn
-            );
-
-            self.obj_store
-                .put(&key, lsn, &RelationSizeEntry::ser(&val)?)?;
-        }
-
-        Ok(())
-    }
-
-    ///
-    /// Memorize a full image of a page version
-    ///
-    fn put_page_image(&self, tag: BufferTag, lsn: Lsn, img: Bytes) -> Result<()> {
-        let key = ObjectKey {
-            timeline: self.timelineid,
-            buf_tag: tag,
-        };
-        let val = PageEntry::Page(img);
-
-        self.obj_store.put(&key, lsn, &PageEntry::ser(&val)?)?;
-
-        debug!(
-            "put_page_image rel {} blk {} at {}",
-            tag.rel, tag.blknum, lsn
-        );
-
-        // Also check if this created or extended the file
-        let old_nblocks = self.relsize_get_nowait(tag.rel, lsn)?.unwrap_or(0);
-
-        if tag.blknum >= old_nblocks {
-            let new_nblocks = tag.blknum + 1;
-            let key = relation_size_key(self.timelineid, tag.rel);
-            let val = RelationSizeEntry::Size(new_nblocks);
-
-            trace!(
-                "Extended relation {} from {} to {} blocks at {}",
-                tag.rel,
-                old_nblocks,
-                new_nblocks,
-                lsn
-            );
-
-            self.obj_store
-                .put(&key, lsn, &RelationSizeEntry::ser(&val)?)?;
-        }
-
-        Ok(())
-    }
-
-    ///
-    /// Adds a relation-wide WAL record (like truncate) to the repository,
-    /// associating it with all pages started with specified block number
-    ///
-    fn put_truncation(&self, rel: RelTag, lsn: Lsn, nblocks: u32) -> Result<()> {
-        let key = relation_size_key(self.timelineid, rel);
-        let val = RelationSizeEntry::Size(nblocks);
-
-        info!("Truncate relation {} to {} blocks at {}", rel, nblocks, lsn);
-
-        self.obj_store
-            .put(&key, lsn, &RelationSizeEntry::ser(&val)?)?;
-
-        Ok(())
-    }
-
-    /// Remember the all WAL before the given LSN has been processed.
-    ///
-    /// The WAL receiver calls this after the put_* functions, to indicate that
-    /// all WAL before this point has been digested. Before that, if you call
-    /// GET on an earlier LSN, it will block.
-    fn advance_last_valid_lsn(&self, lsn: Lsn) {
-        let old = self.last_valid_lsn.advance(lsn);
-
-        // Can't move backwards.
-        if lsn < old {
-            warn!(
-                "attempted to move last valid LSN backwards (was {}, new {})",
-                old, lsn
-            );
-        }
-    }
-
-    fn get_last_valid_lsn(&self) -> Lsn {
-        self.last_valid_lsn.load()
-    }
-
-    fn init_valid_lsn(&self, lsn: Lsn) {
-        let old = self.last_valid_lsn.advance(lsn);
-        assert!(old == Lsn(0));
-        let old = self.last_record_lsn.fetch_max(lsn);
-        assert!(old == Lsn(0));
-    }
-
-    /// Like `advance_last_valid_lsn`, but this always points to the end of
-    /// a WAL record, not in the middle of one.
-    ///
-    /// This must be <= last valid LSN. This is tracked separately from last
-    /// valid LSN, so that the WAL receiver knows where to restart streaming.
-    ///
-    /// NOTE: this updates last_valid_lsn as well.
-    fn advance_last_record_lsn(&self, lsn: Lsn) {
-        // Can't move backwards.
-        let old = self.last_record_lsn.fetch_max(lsn);
-        assert!(old <= lsn);
-
-        // Also advance last_valid_lsn
-        let old = self.last_valid_lsn.advance(lsn);
-        // Can't move backwards.
-        if lsn < old {
-            warn!(
-                "attempted to move last record LSN backwards (was {}, new {})",
-                old, lsn
-            );
-        }
-    }
-    fn get_last_record_lsn(&self) -> Lsn {
-        self.last_record_lsn.load()
-    }
-
-    ///
-    /// Flush to disk all data that was written with the put_* functions
-    ///
-    /// NOTE: This has nothing to do with checkpoint in PostgreSQL. We don't
-    /// know anything about them here in the repository.
-
-    // Flush all the changes written so far with PUT functions to disk.
-    // RocksDB writes out things as we go (?), so we don't need to do much here. We just
-    // write out the last valid and record LSNs.
-    fn checkpoint(&self) -> Result<()> {
-        let metadata = MetadataEntry {
-            last_valid_lsn: self.last_valid_lsn.load(),
-            last_record_lsn: self.last_record_lsn.load(),
-            ancestor_timeline: self.ancestor_timeline,
-            ancestor_lsn: self.ancestor_lsn,
-        };
-        self.obj_store.put(
-            &timeline_metadata_key(self.timelineid),
-            Lsn(0),
-            &MetadataEntry::ser(&metadata)?,
-        )?;
-
-        trace!("checkpoint at {}", metadata.last_valid_lsn);
-
-        Ok(())
-    }
-}
-
-impl ObjectTimeline {
-    fn get_page_at_lsn_nowait(&self, tag: BufferTag, lsn: Lsn) -> Result<Bytes> {
-        // Look up the page entry. If it's a page image, return that. If it's a WAL record,
-        // ask the WAL redo service to reconstruct the page image from the WAL records.
-        let searchkey = ObjectKey {
-            timeline: self.timelineid,
-            buf_tag: tag,
-        };
-        let mut iter = self.object_versions(&*self.obj_store, &searchkey, lsn)?;
-
-        if let Some((version_lsn, value)) = iter.next().transpose()? {
-            let page_img: Bytes;
-
-            match PageEntry::des(&value)? {
-                PageEntry::Page(img) => {
-                    page_img = img;
-                }
-                PageEntry::WALRecord(_rec) => {
-                    // Request the WAL redo manager to apply the WAL records for us.
-                    let (base_img, records) = self.collect_records_for_apply(tag, lsn)?;
-                    page_img = self.walredo_mgr.request_redo(tag, lsn, base_img, records)?;
-
-                    self.put_page_image(tag, lsn, page_img.clone())?;
-                }
-            }
-            // FIXME: assumes little-endian. Only used for the debugging log though
-            let page_lsn_hi = u32::from_le_bytes(page_img.get(0..4).unwrap().try_into().unwrap());
-            let page_lsn_lo = u32::from_le_bytes(page_img.get(4..8).unwrap().try_into().unwrap());
-            trace!(
-                "Returning page with LSN {:X}/{:X} for {} blk {} from {} (request {})",
-                page_lsn_hi,
-                page_lsn_lo,
-                tag.rel,
-                tag.blknum,
-                version_lsn,
-                lsn
-            );
-            return Ok(page_img);
-        }
-        static ZERO_PAGE: [u8; 8192] = [0u8; 8192];
-        trace!("page {} blk {} at {} not found", tag.rel, tag.blknum, lsn);
-        Ok(Bytes::from_static(&ZERO_PAGE))
-        /* return Err("could not find page image")?; */
-    }
-
-    ///
-    /// Internal function to get relation size at given LSN.
-    ///
-    /// The caller must ensure that WAL has been received up to 'lsn'.
-    ///
-    fn relsize_get_nowait(&self, rel: RelTag, lsn: Lsn) -> Result<Option<u32>> {
-        let key = relation_size_key(self.timelineid, rel);
-        let mut iter = self.object_versions(&*self.obj_store, &key, lsn)?;
-
-        if let Some((version_lsn, value)) = iter.next().transpose()? {
-            match RelationSizeEntry::des(&value)? {
-                RelationSizeEntry::Size(nblocks) => {
-                    trace!(
-                        "relation {} has size {} at {} (request {})",
-                        rel,
-                        nblocks,
-                        version_lsn,
-                        lsn
-                    );
-                    Ok(Some(nblocks))
-                }
-                RelationSizeEntry::Unlink => {
-                    trace!(
-                        "relation {} not found; it was dropped at lsn {}",
-                        rel,
-                        version_lsn
-                    );
-                    Ok(None)
-                }
-            }
-        } else {
-            info!("relation {} not found at {}", rel, lsn);
-            Ok(None)
-        }
-    }
-
-    ///
-    /// Collect all the WAL records that are needed to reconstruct a page
-    /// image for the given cache entry.
-    ///
-    /// Returns an old page image (if any), and a vector of WAL records to apply
-    /// over it.
-    ///
-    fn collect_records_for_apply(
-        &self,
-        tag: BufferTag,
-        lsn: Lsn,
-    ) -> Result<(Option<Bytes>, Vec<WALRecord>)> {
-        let mut base_img: Option<Bytes> = None;
-        let mut records: Vec<WALRecord> = Vec::new();
-
-        // Scan backwards, collecting the WAL records, until we hit an
-        // old page image.
-        let searchkey = ObjectKey {
-            timeline: self.timelineid,
-            buf_tag: tag,
-        };
-        let mut iter = self.object_versions(&*self.obj_store, &searchkey, lsn)?;
-        while let Some((_key, value)) = iter.next().transpose()? {
-            match PageEntry::des(&value)? {
-                PageEntry::Page(img) => {
-                    // We have a base image. No need to dig deeper into the list of
-                    // records
-                    base_img = Some(img);
-                    break;
-                }
-                PageEntry::WALRecord(rec) => {
-                    records.push(rec.clone());
-                    // If this WAL record initializes the page, no need to dig deeper.
-                    if rec.will_init {
-                        break;
-                    }
-                }
-            }
-        }
-        records.reverse();
-        Ok((base_img, records))
-    }
-
-    fn launch_gc_thread(conf: &'static PageServerConf, timeline_rc: Arc<ObjectTimeline>) {
-        let _gc_thread = thread::Builder::new()
-            .name("Garbage collection thread".into())
-            .spawn(move || {
-                // FIXME
-                timeline_rc.do_gc(conf).expect("GC thread died");
-            })
-            .unwrap();
-    }
-
-    fn do_gc(&self, conf: &'static PageServerConf) -> Result<()> {
-        loop {
-            thread::sleep(conf.gc_period);
-
-            // FIXME: broken
-            /*
-            let last_lsn = self.get_last_valid_lsn();
-
-            // checked_sub() returns None on overflow.
-            if let Some(horizon) = last_lsn.checked_sub(conf.gc_horizon) {
-                let mut maxkey = StorageKey {
-                    tag: BufferTag {
-                        rel: RelTag {
-                            spcnode: u32::MAX,
-                            dbnode: u32::MAX,
-                            relnode: u32::MAX,
-                            forknum: u8::MAX,
-                        },
-                        blknum: u32::MAX,
-                    },
-                    lsn: Lsn::MAX,
-                };
-                let now = Instant::now();
-                let mut reconstructed = 0u64;
-                let mut truncated = 0u64;
-                let mut inspected = 0u64;
-                let mut deleted = 0u64;
-                loop {
-                    let mut iter = self.db.raw_iterator();
-                    iter.seek_for_prev(maxkey.to_bytes());
-                    if iter.valid() {
-                        let key = StorageKey::des(iter.key().unwrap());
-                        let val = StorageValue::des(iter.value().unwrap());
-
-                        inspected += 1;
-
-                        // Construct boundaries for old records cleanup
-                        maxkey.tag = key.tag;
-                        let last_lsn = key.lsn;
-                        maxkey.lsn = min(horizon, last_lsn); // do not remove last version
-
-                        let mut minkey = maxkey.clone();
-                        minkey.lsn = Lsn(0); // first version
-
-                        // reconstruct most recent page version
-                        if let StorageValueContent::Image(_) = val.content {
-                            // force reconstruction of most recent page version
-                            let (base_img, records) =
-                                self.collect_records_for_apply(key.tag, key.lsn);
-
-                            trace!(
-                                "Reconstruct most recent page {} blk {} at {} from {} records",
-                                key.tag.rel,
-                                key.tag.blknum,
-                                key.lsn,
-                                records.len()
-                            );
-
-                            let new_img = self
-                                .walredo_mgr
-                                .request_redo(key.tag, key.lsn, base_img, records)?;
-                            self.put_page_image(key.tag, key.lsn, new_img.clone());
-
-                            reconstructed += 1;
-                        }
-
-                        iter.seek_for_prev(maxkey.to_bytes());
-                        if iter.valid() {
-                            // do not remove last version
-                            if last_lsn > horizon {
-                                // locate most recent record before horizon
-                                let key = StorageKey::des(iter.key().unwrap());
-                                if key.tag == maxkey.tag {
-                                    let val = StorageValue::des(iter.value().unwrap());
-                                    if let StorageValueContent::Image(_) = val.content {
-                                        let (base_img, records) =
-                                            self.collect_records_for_apply(key.tag, key.lsn);
-                                        trace!("Reconstruct horizon page {} blk {} at {} from {} records",
-                                              key.tag.rel, key.tag.blknum, key.lsn, records.len());
-                                        let new_img = self
-                                            .walredo_mgr
-                                            .request_redo(key.tag, key.lsn, base_img, records)?;
-                                        self.put_page_image(key.tag, key.lsn, new_img.clone());
-
-                                        truncated += 1;
-                                    } else {
-                                        trace!(
-                                            "Keeping horizon page {} blk {} at {}",
-                                            key.tag.rel,
-                                            key.tag.blknum,
-                                            key.lsn
-                                        );
-                                    }
-                                }
-                            } else {
-                                trace!(
-                                    "Last page {} blk {} at {}, horizon {}",
-                                    key.tag.rel,
-                                    key.tag.blknum,
-                                    key.lsn,
-                                    horizon
-                                );
-                            }
-                            // remove records prior to horizon
-                            loop {
-                                iter.prev();
-                                if !iter.valid() {
-                                    break;
-                                }
-                                let key = StorageKey::des(iter.key().unwrap());
-                                if key.tag != maxkey.tag {
-                                    break;
-                                }
-                                let mut val = StorageValue::des(iter.value().unwrap());
-                                if val.alive {
-                                    val.alive = false;
-                                    self.storage.put(key, val)?;
-                                    deleted += 1;
-                                    trace!(
-                                        "deleted: {} blk {} at {}",
-                                        key.tag.rel,
-                                        key.tag.blknum,
-                                        key.lsn
-                                    );
-                                } else {
-                                    break;
-                                }
-                            }
-                        }
-                        maxkey = minkey;
-                    } else {
-                        break;
-                    }
-                }
-                info!("Garbage collection completed in {:?}:\n{} version chains inspected, {} pages reconstructed, {} version histories truncated, {} versions deleted",
-                      now.elapsed(), inspected, reconstructed, truncated, deleted);
-            }
-             */
-        }
-    }
-
-    //
-    // Wait until WAL has been received up to the given LSN.
-    //
-    fn wait_lsn(&self, mut lsn: Lsn) -> Result<Lsn> {
-        // When invalid LSN is requested, it means "don't wait, return latest version of the page"
-        // This is necessary for bootstrap.
-        if lsn == Lsn(0) {
-            let last_valid_lsn = self.last_valid_lsn.load();
-            trace!(
-                "walreceiver doesn't work yet last_valid_lsn {}, requested {}",
-                last_valid_lsn,
-                lsn
-            );
-            lsn = last_valid_lsn;
-        }
-        trace!(
-            "Start waiting for LSN {}, valid LSN is {}",
-            lsn,
-            self.last_valid_lsn.load()
-        );
-        self.last_valid_lsn
-            .wait_for_timeout(lsn, TIMEOUT)
-            .with_context(|| {
-                format!(
-                    "Timed out while waiting for WAL record at LSN {} to arrive. valid LSN in {}",
-                    lsn,
-                    self.last_valid_lsn.load(),
-                )
-            })?;
-        //trace!("Stop waiting for LSN {}, valid LSN is {}", lsn,  self.last_valid_lsn.load());
-
-        Ok(lsn)
-    }
-
-    ///
-    /// Iterate through object versions with given key, in reverse LSN order.
-    ///
-    /// This implements following the timeline history over the plain
-    /// ObjectStore::object_versions function, which doesn't know
-    /// about the relationships between timeline.
-    ///
-    fn object_versions<'a>(
-        &self,
-        obj_store: &'a dyn ObjectStore,
-        key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<ObjectVersionIter<'a>> {
-        let current_iter = obj_store.object_versions(key, lsn)?;
-
-        Ok(ObjectVersionIter {
-            obj_store,
-            buf_tag: key.buf_tag,
-            current_iter,
-            ancestor_timeline: self.ancestor_timeline,
-            ancestor_lsn: self.ancestor_lsn,
-        })
-    }
-}
-
-///
-/// We store two kinds of page versions in the repository:
-///
-/// 1. Ready-made images of the block
-/// 2. WAL records, to be applied on top of the "previous" entry
-///
-/// Some WAL records will initialize the page from scratch. For such records,
-/// the 'will_init' flag is set. They don't need the previous page image before
-/// applying. The 'will_init' flag is set for records containing a full-page image,
-/// and for records with the BKPBLOCK_WILL_INIT flag. These differ from PageImages
-/// stored directly in the cache entry in that you still need to run the WAL redo
-/// routine to generate the page image.
-///
-#[derive(Debug, Clone, Serialize, Deserialize)]
-enum PageEntry {
-    Page(Bytes),
-    WALRecord(WALRecord),
-}
-
-///
-/// In addition to page versions, we store relation size as a separate, versioned,
-/// object.
-///
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum RelationSizeEntry {
-    Size(u32),
-
-    /// Tombstone for a dropped relation.
-    //
-    // TODO: Not used. Currently, we never drop relations. The parsing
-    // of relation drops in COMMIT/ABORT records has not been
-    // implemented. We should also have a mechanism to remove
-    // "orphaned" relfiles, if the compute node crashes before writing
-    // the COMMIT/ABORT record.
-    Unlink,
-}
-
-const fn relation_size_key(timelineid: ZTimelineId, rel: RelTag) -> ObjectKey {
-    ObjectKey {
-        timeline: timelineid,
-        buf_tag: BufferTag {
-            rel,
-            blknum: u32::MAX,
-        },
-    }
-}
-
-///
-/// In addition to those per-page and per-relation entries, we also
-/// store a little metadata blob for each timeline. It is stored using
-/// STORAGE_SPECIAL_FORKNUM.
-///
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct MetadataEntry {
-    last_valid_lsn: Lsn,
-    last_record_lsn: Lsn,
-    ancestor_timeline: Option<ZTimelineId>,
-    ancestor_lsn: Lsn,
-}
-
-const fn timeline_metadata_key(timelineid: ZTimelineId) -> ObjectKey {
-    ObjectKey {
-        timeline: timelineid,
-        buf_tag: BufferTag {
-            rel: RelTag {
-                forknum: pg_constants::ROCKSDB_SPECIAL_FORKNUM,
-                spcnode: 0,
-                dbnode: 0,
-                relnode: 0,
-            },
-            blknum: 0,
-        },
-    }
-}
-
-///
-/// Iterator for `object_versions`. Returns all page versions of a given block, in
-/// reverse LSN order. This implements the traversal of ancestor timelines. If
-/// a page isn't found in the most recent timeline, this iterates to the parent,
-/// until a page version is found.
-///
-struct ObjectVersionIter<'a> {
-    obj_store: &'a dyn ObjectStore,
-
-    buf_tag: BufferTag,
-
-    /// Iterator on the current timeline.
-    current_iter: Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>,
-
-    /// Ancestor of the current timeline being iterated.
-    ancestor_timeline: Option<ZTimelineId>,
-    ancestor_lsn: Lsn,
-}
-
-impl<'a> Iterator for ObjectVersionIter<'a> {
-    type Item = Result<(Lsn, Vec<u8>)>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.next_result().transpose()
-    }
-}
-
-impl<'a> ObjectVersionIter<'a> {
-    ///
-    /// "transposed" version of the standard Iterator::next function.
-    ///
-    /// The rust standard Iterator::next function returns an
-    /// Option of a Result, but it's more convenient to work with
-    /// Result of a Option so that you can use ? to check for errors.
-    ///
-    fn next_result(&mut self) -> Result<Option<(Lsn, Vec<u8>)>> {
-        loop {
-            // If there is another entry on the current timeline, return it.
-            if let Some(result) = self.current_iter.next() {
-                return Ok(Some(result));
-            }
-
-            // Out of entries on this timeline. Move to the ancestor, if any.
-            if let Some(ancestor_timeline) = self.ancestor_timeline {
-                let searchkey = ObjectKey {
-                    timeline: ancestor_timeline,
-                    buf_tag: self.buf_tag,
-                };
-                let ancestor_iter = self
-                    .obj_store
-                    .object_versions(&searchkey, self.ancestor_lsn)?;
-
-                // Load the parent timeline's metadata. (We don't
-                // actually need it yet, only if we need to follow to
-                // the grandparent timeline)
-                let v = self
-                    .obj_store
-                    .get(&timeline_metadata_key(ancestor_timeline), Lsn(0))
-                    .with_context(|| "timeline not found in repository")?;
-                let ancestor_metadata = MetadataEntry::des(&v)?;
-
-                self.ancestor_timeline = ancestor_metadata.ancestor_timeline;
-                self.ancestor_lsn = ancestor_metadata.ancestor_lsn;
-                self.current_iter = ancestor_iter;
-            } else {
-                return Ok(None);
-            }
-        }
-    }
-}

pageserver/src/object_store.rs

@@ -1,61 +0,0 @@
-use crate::repository::{BufferTag, RelTag};
-use crate::ZTimelineId;
-use anyhow::Result;
-use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
-use std::iter::Iterator;
-use zenith_utils::lsn::Lsn;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ObjectKey {
-    pub timeline: ZTimelineId,
-    pub buf_tag: BufferTag,
-}
-
-///
-/// Low-level storage abstraction.
-///
-/// All the data in the repository is stored in a key-value store. This trait
-/// abstracts the details of the key-value store.
-///
-/// A simple key-value store would support just GET and PUT operations with
-/// a key, but the upper layer needs slightly complicated read operations
-///
-/// The most frequently used function is 'object_versions'. It is used
-/// to look up a page version. It is LSN aware, in that the caller
-/// specifies an LSN, and the function returns all values for that
-/// block with the same or older LSN.
-///
-pub trait ObjectStore: Send + Sync {
-    ///
-    /// Store a value with given key.
-    ///
-    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()>;
-
-    /// Read entry with the exact given key.
-    ///
-    /// This is used for retrieving metadata with special key that doesn't
-    /// correspond to any real relation.
-    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>>;
-
-    /// Iterate through all page versions of one object.
-    ///
-    /// Returns all page versions in descending LSN order, along with the LSN
-    /// of each page version.
-    fn object_versions<'a>(
-        &'a self,
-        key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>>;
-
-    /// Iterate through all keys with given tablespace and database ID, and LSN <= 'lsn'.
-    ///
-    /// This is used to implement 'create database'
-    fn list_rels(
-        &self,
-        timelineid: ZTimelineId,
-        spcnode: u32,
-        dbnode: u32,
-        lsn: Lsn,
-    ) -> Result<HashSet<RelTag>>;
-}

pageserver/src/page_cache.rs

@@ -1,35 +0,0 @@
-//! This module acts as a switchboard to access different repositories managed by this
-//! page server. Currently, a Page Server can only manage one repository, so there
-//! isn't much here. If we implement multi-tenancy, this will probably be changed into
-//! a hash map, keyed by the tenant ID.
-
-use crate::object_repository::ObjectRepository;
-use crate::repository::Repository;
-use crate::rocksdb_storage::RocksObjectStore;
-use crate::walredo::PostgresRedoManager;
-use crate::PageServerConf;
-use lazy_static::lazy_static;
-use std::sync::{Arc, Mutex};
-
-lazy_static! {
-    pub static ref REPOSITORY: Mutex<Option<Arc<dyn Repository>>> = Mutex::new(None);
-}
-
-pub fn init(conf: &'static PageServerConf) {
-    let mut m = REPOSITORY.lock().unwrap();
-
-    let obj_store = RocksObjectStore::open(conf).unwrap();
-
-    // Set up a WAL redo manager, for applying WAL records.
-    let walredo_mgr = PostgresRedoManager::new(conf);
-
-    // we have already changed current dir to the repository.
-    let repo = ObjectRepository::new(conf, Arc::new(obj_store), Arc::new(walredo_mgr));
-
-    *m = Some(Arc::new(repo));
-}
-
-pub fn get_repository() -> Arc<dyn Repository> {
-    let o = &REPOSITORY.lock().unwrap();
-    Arc::clone(o.as_ref().unwrap())
-}

pageserver/src/relish.rs

@@ -0,0 +1,235 @@
+//!
+//! Zenith stores PostgreSQL relations, and some other files, in the
+//! repository.  The relations (i.e. tables and indexes) take up most
+//! of the space in a typical installation, while the other files are
+//! small. We call each relation and other file that is stored in the
+//! repository a "relish". It comes from "rel"-ish, as in "kind of a
+//! rel", because it covers relations as well as other things that are
+//! not relations, but are treated similarly for the purposes of the
+//! storage layer.
+//!
+//! This source file contains the definition of the RelishTag struct,
+//! which uniquely identifies a relish.
+//!
+//! Relishes come in two flavors: blocky and non-blocky. Relations and
+//! SLRUs are blocky, that is, they are divided into 8k blocks, and
+//! the repository tracks their size. Other relishes are non-blocky:
+//! the content of the whole relish is stored as one blob. Block
+//! number must be passed as 0 for all operations on a non-blocky
+//! relish. The one "block" that you store in a non-blocky relish can
+//! have arbitrary size, but they are expected to be small, or you
+//! will have performance issues.
+//!
+//! All relishes are versioned by LSN in the repository.
+//!
+
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+use postgres_ffi::relfile_utils::forknumber_to_name;
+use postgres_ffi::{Oid, TransactionId};
+
+///
+/// RelishTag identifies one relish.
+///
+#[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub enum RelishTag {
+    // Relations correspond to PostgreSQL relation forks. Each
+    // PostgreSQL relation fork is considered a separate relish.
+    Relation(RelTag),
+
+    // SLRUs include pg_clog, pg_multixact/members, and
+    // pg_multixact/offsets. There are other SLRUs in PostgreSQL, but
+    // they don't need to be stored permanently (e.g. pg_subtrans),
+    // or we do not support them in zenith yet (pg_commit_ts).
+    //
+    // These are currently never requested directly by the compute
+    // nodes, although in principle that would be possible. However,
+    // when a new compute node is created, these are included in the
+    // tarball that we send to the compute node to initialize the
+    // PostgreSQL data directory.
+    //
+    // Each SLRU segment in PostgreSQL is considered a separate
+    // relish. For example, pg_clog/0000, pg_clog/0001, and so forth.
+    //
+    // SLRU segments are divided into blocks, like relations.
+    Slru { slru: SlruKind, segno: u32 },
+
+    // Miscellaneous other files that need to be included in the
+    // tarball at compute node creation. These are non-blocky, and are
+    // expected to be small.
+
+    //
+    // FileNodeMap represents PostgreSQL's 'pg_filenode.map'
+    // files. They are needed to map catalog table OIDs to filenode
+    // numbers. Usually the mapping is done by looking up a relation's
+    // 'relfilenode' field in the 'pg_class' system table, but that
+    // doesn't work for 'pg_class' itself and a few other such system
+    // relations. See PostgreSQL relmapper.c for details.
+    //
+    // Each database has a map file for its local mapped catalogs,
+    // and there is a separate map file for shared catalogs.
+    //
+    // These files are always 512 bytes long (although we don't check
+    // or care about that in the page server).
+    //
+    FileNodeMap { spcnode: Oid, dbnode: Oid },
+
+    //
+    // State files for prepared transactions (e.g pg_twophase/1234)
+    //
+    TwoPhase { xid: TransactionId },
+
+    // The control file, stored in global/pg_control
+    ControlFile,
+
+    // Special entry that represents PostgreSQL checkpoint. It doesn't
+    // correspond to to any physical file in PostgreSQL, but we use it
+    // to track fields needed to restore the checkpoint data in the
+    // control file, when a compute node is created.
+    Checkpoint,
+}
+
+impl RelishTag {
+    pub const fn is_blocky(&self) -> bool {
+        match self {
+            // These relishes work with blocks
+            RelishTag::Relation(_) | RelishTag::Slru { slru: _, segno: _ } => true,
+
+            // and these don't
+            RelishTag::FileNodeMap {
+                spcnode: _,
+                dbnode: _,
+            }
+            | RelishTag::TwoPhase { xid: _ }
+            | RelishTag::ControlFile
+            | RelishTag::Checkpoint => false,
+        }
+    }
+
+    // Physical relishes represent files and use
+    // RelationSizeEntry to track existing and dropped files.
+    // They can be both blocky and non-blocky.
+    pub const fn is_physical(&self) -> bool {
+        match self {
+            // These relishes represent physical files
+            RelishTag::Relation(_)
+            | RelishTag::Slru { .. }
+            | RelishTag::FileNodeMap { .. }
+            | RelishTag::TwoPhase { .. } => true,
+
+            // and these don't
+            RelishTag::ControlFile | RelishTag::Checkpoint => false,
+        }
+    }
+
+    // convenience function to check if this relish is a normal relation.
+    pub const fn is_relation(&self) -> bool {
+        if let RelishTag::Relation(_) = self {
+            true
+        } else {
+            false
+        }
+    }
+}
+
+///
+/// Relation data file segment id throughout the Postgres cluster.
+///
+/// Every data file in Postgres is uniquely identified by 4 numbers:
+/// - relation id / node (`relnode`)
+/// - database id (`dbnode`)
+/// - tablespace id (`spcnode`), in short this is a unique id of a separate
+///   directory to store data files.
+/// - forknumber (`forknum`) is used to split different kinds of data of the same relation
+///   between some set of files (`relnode`, `relnode_fsm`, `relnode_vm`).
+///
+/// In native Postgres code `RelFileNode` structure and individual `ForkNumber` value
+/// are used for the same purpose.
+/// [See more related comments here](https:///github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/relfilenode.h#L57).
+///
+#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy, Serialize, Deserialize)]
+pub struct RelTag {
+    pub forknum: u8,
+    pub spcnode: Oid,
+    pub dbnode: Oid,
+    pub relnode: Oid,
+}
+
+/// Display RelTag in the same format that's used in most PostgreSQL debug messages:
+///
+/// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
+///
+impl fmt::Display for RelTag {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if let Some(forkname) = forknumber_to_name(self.forknum) {
+            write!(
+                f,
+                "{}/{}/{}_{}",
+                self.spcnode, self.dbnode, self.relnode, forkname
+            )
+        } else {
+            write!(f, "{}/{}/{}", self.spcnode, self.dbnode, self.relnode)
+        }
+    }
+}
+
+/// Display RelTag in the same format that's used in most PostgreSQL debug messages:
+///
+/// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
+///
+impl fmt::Display for RelishTag {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            RelishTag::Relation(rel) => rel.fmt(f),
+            RelishTag::Slru { slru, segno } => {
+                // e.g. pg_clog/0001
+                write!(f, "{}/{:04X}", slru.to_str(), segno)
+            }
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                write!(f, "relmapper file for spc {} db {}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => {
+                write!(f, "pg_twophase/{:08X}", xid)
+            }
+            RelishTag::ControlFile => {
+                write!(f, "control file")
+            }
+            RelishTag::Checkpoint => {
+                write!(f, "checkpoint")
+            }
+        }
+    }
+}
+
+///
+/// Non-relation transaction status files (clog (a.k.a. pg_xact) and
+/// pg_multixact) in Postgres are handled by SLRU (Simple LRU) buffer,
+/// hence the name.
+///
+/// These files are global for a postgres instance.
+///
+/// These files are divided into segments, which are divided into
+/// pages of the same BLCKSZ as used for relation files.
+///
+#[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub enum SlruKind {
+    Clog,
+    MultiXactMembers,
+    MultiXactOffsets,
+}
+
+impl SlruKind {
+    pub fn to_str(&self) -> &'static str {
+        match self {
+            Self::Clog => "pg_xact",
+            Self::MultiXactMembers => "pg_multixact/members",
+            Self::MultiXactOffsets => "pg_multixact/offsets",
+        }
+    }
+}
+
+pub const FIRST_NONREL_RELISH_TAG: RelishTag = RelishTag::Slru {
+    slru: SlruKind::Clog,
+    segno: 0,
+};

pageserver/src/repository.rs

@@ -1,12 +1,13 @@
-use crate::ZTimelineId;
+use crate::relish::*;
 use anyhow::Result;
 use bytes::{Buf, BufMut, Bytes, BytesMut};
-use postgres_ffi::relfile_utils::forknumber_to_name;
 use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
-use std::fmt;
+use std::ops::AddAssign;
 use std::sync::Arc;
-use zenith_utils::lsn::Lsn;
+use std::time::Duration;
+use zenith_utils::lsn::{Lsn, RecordLsn};
+use zenith_utils::zid::ZTimelineId;
 
 ///
 /// A repository corresponds to one .zenith directory. One repository holds multiple
@@ -25,26 +26,95 @@ pub trait Repository: Send + Sync {
     /// Branch a timeline
     fn branch_timeline(&self, src: ZTimelineId, dst: ZTimelineId, start_lsn: Lsn) -> Result<()>;
 
+    /// perform one garbage collection iteration.
+    /// garbage collection is periodically performed by gc thread,
+    /// but it can be explicitly requested through page server api.
+    ///
+    /// 'timelineid' specifies the timeline to GC, or None for all.
+    /// `horizon` specifies delta from last lsn to preserve all object versions (pitr interval).
+    /// `compact` parameter is used to force compaction of storage.
+    /// some storage implementation are based on lsm tree and require periodic merge (compaction).
+    /// usually storage implementation determines itself when compaction should be performed.
+    /// but for gc tests it way be useful to force compaction just after completion of gc iteration
+    /// to make sure that all detected garbage is removed.
+    /// so right now `compact` is set to true when gc explicitly requested through page srver api,
+    /// and is st to false in gc threads which infinitely repeats gc iterations in loop.
+    fn gc_iteration(
+        &self,
+        timelineid: Option<ZTimelineId>,
+        horizon: u64,
+        compact: bool,
+    ) -> Result<GcResult>;
+
+    // TODO get timelines?
     //fn get_stats(&self) -> RepositoryStats;
 }
 
+///
+/// Result of performing GC
+///
+#[derive(Default)]
+pub struct GcResult {
+    pub ondisk_relfiles_total: u64,
+    pub ondisk_relfiles_needed_by_cutoff: u64,
+    pub ondisk_relfiles_needed_by_branches: u64,
+    pub ondisk_relfiles_not_updated: u64,
+    pub ondisk_relfiles_removed: u64, // # of layer files removed because they have been made obsolete by newer ondisk files.
+    pub ondisk_relfiles_dropped: u64, // # of layer files removed because the relation was dropped
+
+    pub ondisk_nonrelfiles_total: u64,
+    pub ondisk_nonrelfiles_needed_by_cutoff: u64,
+    pub ondisk_nonrelfiles_needed_by_branches: u64,
+    pub ondisk_nonrelfiles_not_updated: u64,
+    pub ondisk_nonrelfiles_removed: u64, // # of layer files removed because they have been made obsolete by newer ondisk files.
+    pub ondisk_nonrelfiles_dropped: u64, // # of layer files removed because the relation was dropped
+
+    pub elapsed: Duration,
+}
+
+impl AddAssign for GcResult {
+    fn add_assign(&mut self, other: Self) {
+        self.ondisk_relfiles_total += other.ondisk_relfiles_total;
+        self.ondisk_relfiles_needed_by_cutoff += other.ondisk_relfiles_needed_by_cutoff;
+        self.ondisk_relfiles_needed_by_branches += other.ondisk_relfiles_needed_by_branches;
+        self.ondisk_relfiles_not_updated += other.ondisk_relfiles_not_updated;
+        self.ondisk_relfiles_removed += other.ondisk_relfiles_removed;
+        self.ondisk_relfiles_dropped += other.ondisk_relfiles_dropped;
+
+        self.ondisk_nonrelfiles_total += other.ondisk_nonrelfiles_total;
+        self.ondisk_nonrelfiles_needed_by_cutoff += other.ondisk_nonrelfiles_needed_by_cutoff;
+        self.ondisk_nonrelfiles_needed_by_branches += other.ondisk_nonrelfiles_needed_by_branches;
+        self.ondisk_nonrelfiles_not_updated += other.ondisk_nonrelfiles_not_updated;
+        self.ondisk_nonrelfiles_removed += other.ondisk_nonrelfiles_removed;
+        self.ondisk_nonrelfiles_dropped += other.ondisk_nonrelfiles_dropped;
+
+        self.elapsed += other.elapsed;
+    }
+}
+
 pub trait Timeline: Send + Sync {
     //------------------------------------------------------------------------------
     // Public GET functions
     //------------------------------------------------------------------------------
 
     /// Look up given page in the cache.
-    fn get_page_at_lsn(&self, tag: BufferTag, lsn: Lsn) -> Result<Bytes>;
+    fn get_page_at_lsn(&self, tag: RelishTag, blknum: u32, lsn: Lsn) -> Result<Bytes>;
 
-    /// Get size of relation
-    fn get_rel_size(&self, tag: RelTag, lsn: Lsn) -> Result<u32>;
+    /// Look up given page in the cache.
+    fn get_page_at_lsn_nowait(&self, tag: RelishTag, blknum: u32, lsn: Lsn) -> Result<Bytes>;
+
+    /// Get size of a relish
+    fn get_relish_size(&self, tag: RelishTag, lsn: Lsn) -> Result<Option<u32>>;
 
     /// Does relation exist?
-    fn get_rel_exists(&self, tag: RelTag, lsn: Lsn) -> Result<bool>;
+    fn get_rel_exists(&self, tag: RelishTag, lsn: Lsn) -> Result<bool>;
 
     /// Get a list of all distinct relations in given tablespace and database.
     fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashSet<RelTag>>;
 
+    /// Get a list of non-relational objects
+    fn list_nonrels<'a>(&'a self, lsn: Lsn) -> Result<HashSet<RelishTag>>;
+
     //------------------------------------------------------------------------------
     // Public PUT functions, to update the repository with new page versions.
     //
@@ -55,30 +125,27 @@ pub trait Timeline: Send + Sync {
     ///
     /// This will implicitly extend the relation, if the page is beyond the
     /// current end-of-file.
-    fn put_wal_record(&self, tag: BufferTag, rec: WALRecord) -> Result<()>;
+    fn put_wal_record(&self, tag: RelishTag, blknum: u32, rec: WALRecord) -> Result<()>;
 
     /// Like put_wal_record, but with ready-made image of the page.
-    fn put_page_image(&self, tag: BufferTag, lsn: Lsn, img: Bytes) -> Result<()>;
+    fn put_page_image(&self, tag: RelishTag, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()>;
 
     /// Truncate relation
-    fn put_truncation(&self, rel: RelTag, lsn: Lsn, nblocks: u32) -> Result<()>;
+    fn put_truncation(&self, rel: RelishTag, lsn: Lsn, nblocks: u32) -> Result<()>;
 
-    /// Remember the all WAL before the given LSN has been processed.
-    ///
-    /// The WAL receiver calls this after the put_* functions, to indicate that
-    /// all WAL before this point has been digested. Before that, if you call
-    /// GET on an earlier LSN, it will block.
-    fn advance_last_valid_lsn(&self, lsn: Lsn);
-    fn get_last_valid_lsn(&self) -> Lsn;
-    fn init_valid_lsn(&self, lsn: Lsn);
+    /// This method is used for marking dropped relations and truncated SLRU files
+    fn drop_relish(&self, tag: RelishTag, lsn: Lsn) -> Result<()>;
 
-    /// Like `advance_last_valid_lsn`, but this always points to the end of
-    /// a WAL record, not in the middle of one.
+    /// Track end of the latest digested WAL record.
     ///
-    /// This must be <= last valid LSN. This is tracked separately from last
-    /// valid LSN, so that the WAL receiver knows where to restart streaming.
+    /// Advance requires aligned LSN as an argument and would wake wait_lsn() callers.
+    /// Previous last record LSN is stored alongside the latest and can be read.
     fn advance_last_record_lsn(&self, lsn: Lsn);
+    /// Atomically get both last and prev.
+    fn get_last_record_rlsn(&self) -> RecordLsn;
+    /// Get last or prev record separately. Same as get_last_record_rlsn().last/prev.
     fn get_last_record_lsn(&self) -> Lsn;
+    fn get_prev_record_lsn(&self) -> Lsn;
 
     ///
     /// Flush to disk all data that was written with the put_* functions
@@ -96,65 +163,7 @@ pub struct RepositoryStats {
     pub num_getpage_requests: Lsn,
 }
 
-///
-/// Relation data file segment id throughout the Postgres cluster.
-///
-/// Every data file in Postgres is uniquely identified by 4 numbers:
-/// - relation id / node (`relnode`)
-/// - database id (`dbnode`)
-/// - tablespace id (`spcnode`), in short this is a unique id of a separate
-///   directory to store data files.
-/// - forknumber (`forknum`) is used to split different kinds of data of the same relation
-///   between some set of files (`relnode`, `relnode_fsm`, `relnode_vm`).
-///
-/// In native Postgres code `RelFileNode` structure and individual `ForkNumber` value
-/// are used for the same purpose.
-/// [See more related comments here](https:///github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/relfilenode.h#L57).
-///
-/// We use additional fork numbers to logically separate relational and
-/// non-relational data inside pageserver key-value storage.
-/// See, e.g., `ROCKSDB_SPECIAL_FORKNUM`.
-///
-#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy, Serialize, Deserialize)]
-pub struct RelTag {
-    pub forknum: u8,
-    pub spcnode: u32,
-    pub dbnode: u32,
-    pub relnode: u32,
-}
-
-/// Display RelTag in the same format that's used in most PostgreSQL debug messages:
-///
-/// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
-///
-impl fmt::Display for RelTag {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        if let Some(forkname) = forknumber_to_name(self.forknum) {
-            write!(
-                f,
-                "{}/{}/{}_{}",
-                self.spcnode, self.dbnode, self.relnode, forkname
-            )
-        } else {
-            write!(f, "{}/{}/{}", self.spcnode, self.dbnode, self.relnode)
-        }
-    }
-}
-
-///
-/// `RelTag` + block number (`blknum`) gives us a unique id of the page in the cluster.
-/// This is used as a part of the key inside key-value storage (RocksDB currently).
-///
-/// In Postgres `BufferTag` structure is used for exactly the same purpose.
-/// [See more related comments here](https://github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/buf_internals.h#L91).
-///
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Serialize, Deserialize)]
-pub struct BufferTag {
-    pub rel: RelTag,
-    pub blknum: u32,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct WALRecord {
     pub lsn: Lsn, // LSN at the *end* of the record
     pub will_init: bool,
@@ -194,8 +203,7 @@ impl WALRecord {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::object_repository::ObjectRepository;
-    use crate::rocksdb_storage::RocksObjectStore;
+    use crate::layered_repository::LayeredRepository;
     use crate::walredo::{WalRedoError, WalRedoManager};
     use crate::PageServerConf;
     use postgres_ffi::pg_constants;
@@ -203,24 +211,24 @@ mod tests {
     use std::path::PathBuf;
     use std::str::FromStr;
     use std::time::Duration;
+    use zenith_utils::postgres_backend::AuthType;
+    use zenith_utils::zid::ZTenantId;
 
     /// Arbitrary relation tag, for testing.
-    const TESTREL_A: RelTag = RelTag {
+    const TESTREL_A: RelishTag = RelishTag::Relation(RelTag {
         spcnode: 0,
         dbnode: 111,
         relnode: 1000,
         forknum: 0,
-    };
+    });
+    const TESTREL_B: RelishTag = RelishTag::Relation(RelTag {
+        spcnode: 0,
+        dbnode: 111,
+        relnode: 1001,
+        forknum: 0,
+    });
 
-    /// Convenience function to create a BufferTag for testing.
-    /// Helps to keeps the tests shorter.
-    #[allow(non_snake_case)]
-    fn TEST_BUF(blknum: u32) -> BufferTag {
-        BufferTag {
-            rel: TESTREL_A,
-            blknum,
-        }
-    }
+    const TESTDB: u32 = 111;
 
     /// Convenience function to create a page image with given string as the only content
     #[allow(non_snake_case)]
@@ -232,113 +240,121 @@ mod tests {
         buf.freeze()
     }
 
+    static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
+
     fn get_test_repo(test_name: &str) -> Result<Box<dyn Repository>> {
         let repo_dir = PathBuf::from(format!("../tmp_check/test_{}", test_name));
         let _ = fs::remove_dir_all(&repo_dir);
         fs::create_dir_all(&repo_dir)?;
+        fs::create_dir_all(&repo_dir.join("timelines"))?;
 
         let conf = PageServerConf {
             daemonize: false,
-            interactive: false,
             gc_horizon: 64 * 1024 * 1024,
             gc_period: Duration::from_secs(10),
-            listen_addr: "127.0.0.1:5430".parse().unwrap(),
+            listen_addr: "127.0.0.1:5430".to_string(),
+            http_endpoint_addr: "127.0.0.1:9898".to_string(),
+            superuser: "zenith_admin".to_string(),
             workdir: repo_dir,
             pg_distrib_dir: "".into(),
+            auth_type: AuthType::Trust,
+            auth_validation_public_key_path: None,
         };
         // Make a static copy of the config. This can never be free'd, but that's
         // OK in a test.
         let conf: &'static PageServerConf = Box::leak(Box::new(conf));
-
-        let obj_store = RocksObjectStore::create(conf)?;
+        let tenantid = ZTenantId::generate();
+        fs::create_dir_all(conf.tenant_path(&tenantid)).unwrap();
 
         let walredo_mgr = TestRedoManager {};
 
-        let repo = ObjectRepository::new(conf, Arc::new(obj_store), Arc::new(walredo_mgr));
+        let repo = Box::new(LayeredRepository::new(
+            conf,
+            Arc::new(walredo_mgr),
+            tenantid,
+        ));
 
-        Ok(Box::new(repo))
+        Ok(repo)
     }
 
-    /// Test get_relsize() and truncation.
     #[test]
     fn test_relsize() -> Result<()> {
+        let repo = get_test_repo("test_relsize")?;
         // get_timeline() with non-existent timeline id should fail
         //repo.get_timeline("11223344556677881122334455667788");
 
         // Create timeline to work on
-        let repo = get_test_repo("test_relsize")?;
         let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
+        let tline = repo.create_empty_timeline(timelineid, Lsn(0x00))?;
 
-        tline.init_valid_lsn(Lsn(1));
-        tline.put_page_image(TEST_BUF(0), Lsn(2), TEST_IMG("foo blk 0 at 2"))?;
-        tline.put_page_image(TEST_BUF(0), Lsn(2), TEST_IMG("foo blk 0 at 2"))?;
-        tline.put_page_image(TEST_BUF(0), Lsn(3), TEST_IMG("foo blk 0 at 3"))?;
-        tline.put_page_image(TEST_BUF(1), Lsn(4), TEST_IMG("foo blk 1 at 4"))?;
-        tline.put_page_image(TEST_BUF(2), Lsn(5), TEST_IMG("foo blk 2 at 5"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(0x30), TEST_IMG("foo blk 0 at 3"))?;
+        tline.put_page_image(TESTREL_A, 1, Lsn(0x40), TEST_IMG("foo blk 1 at 4"))?;
+        tline.put_page_image(TESTREL_A, 2, Lsn(0x50), TEST_IMG("foo blk 2 at 5"))?;
 
-        tline.advance_last_valid_lsn(Lsn(5));
+        tline.advance_last_record_lsn(Lsn(0x50));
 
         // The relation was created at LSN 2, not visible at LSN 1 yet.
-        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(1))?, false);
-        assert!(tline.get_rel_size(TESTREL_A, Lsn(1)).is_err());
+        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(0x10))?, false);
+        assert!(tline.get_relish_size(TESTREL_A, Lsn(0x10))?.is_none());
 
-        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(2))?, true);
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(2))?, 1);
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(5))?, 3);
+        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(0x20))?, true);
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(0x20))?.unwrap(), 1);
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(0x50))?.unwrap(), 3);
 
         // Check page contents at each LSN
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(0), Lsn(2))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x20))?,
             TEST_IMG("foo blk 0 at 2")
         );
 
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(0), Lsn(3))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x30))?,
             TEST_IMG("foo blk 0 at 3")
         );
 
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(0), Lsn(4))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x40))?,
             TEST_IMG("foo blk 0 at 3")
         );
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(1), Lsn(4))?,
+            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x40))?,
             TEST_IMG("foo blk 1 at 4")
         );
 
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(0), Lsn(5))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x50))?,
             TEST_IMG("foo blk 0 at 3")
         );
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(1), Lsn(5))?,
+            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x50))?,
             TEST_IMG("foo blk 1 at 4")
         );
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(2), Lsn(5))?,
+            tline.get_page_at_lsn(TESTREL_A, 2, Lsn(0x50))?,
             TEST_IMG("foo blk 2 at 5")
         );
 
         // Truncate last block
-        tline.put_truncation(TESTREL_A, Lsn(6), 2)?;
-        tline.advance_last_valid_lsn(Lsn(6));
+        tline.put_truncation(TESTREL_A, Lsn(0x60), 2)?;
+        tline.advance_last_record_lsn(Lsn(0x60));
 
         // Check reported size and contents after truncation
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(6))?, 2);
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(0x60))?.unwrap(), 2);
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(0), Lsn(6))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x60))?,
             TEST_IMG("foo blk 0 at 3")
         );
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(1), Lsn(6))?,
+            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x60))?,
             TEST_IMG("foo blk 1 at 4")
         );
 
         // should still see the truncated block with older LSN
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(5))?, 3);
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(0x50))?.unwrap(), 3);
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(2), Lsn(5))?,
+            tline.get_page_at_lsn(TESTREL_A, 2, Lsn(0x50))?,
             TEST_IMG("foo blk 2 at 5")
         );
 
@@ -347,48 +363,157 @@ mod tests {
 
     /// Test get_relsize() and truncation with a file larger than 1 GB, so that it's
     /// split into multiple 1 GB segments in Postgres.
-    ///
-    /// This isn't very interesting with the RocksDb implementation, as we don't pay
-    /// any attention to Postgres segment boundaries there.
     #[test]
     fn test_large_rel() -> Result<()> {
         let repo = get_test_repo("test_large_rel")?;
         let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
+        let tline = repo.create_empty_timeline(timelineid, Lsn(0x00))?;
 
-        tline.init_valid_lsn(Lsn(1));
-
-        let mut lsn = 0;
-        for i in 0..pg_constants::RELSEG_SIZE + 1 {
-            let img = TEST_IMG(&format!("foo blk {} at {}", i, Lsn(lsn)));
-            lsn += 1;
-            tline.put_page_image(TEST_BUF(i as u32), Lsn(lsn), img)?;
+        let mut lsn = 0x10;
+        for blknum in 0..pg_constants::RELSEG_SIZE + 1 {
+            let img = TEST_IMG(&format!("foo blk {} at {}", blknum, Lsn(lsn)));
+            lsn += 0x10;
+            tline.put_page_image(TESTREL_A, blknum as u32, Lsn(lsn), img)?;
         }
-        tline.advance_last_valid_lsn(Lsn(lsn));
+        tline.advance_last_record_lsn(Lsn(lsn));
 
         assert_eq!(
-            tline.get_rel_size(TESTREL_A, Lsn(lsn))?,
+            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
             pg_constants::RELSEG_SIZE + 1
         );
 
         // Truncate one block
-        lsn += 1;
+        lsn += 0x10;
         tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE)?;
-        tline.advance_last_valid_lsn(Lsn(lsn));
+        tline.advance_last_record_lsn(Lsn(lsn));
         assert_eq!(
-            tline.get_rel_size(TESTREL_A, Lsn(lsn))?,
+            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
             pg_constants::RELSEG_SIZE
         );
 
         // Truncate another block
-        lsn += 1;
+        lsn += 0x10;
         tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE - 1)?;
-        tline.advance_last_valid_lsn(Lsn(lsn));
+        tline.advance_last_record_lsn(Lsn(lsn));
         assert_eq!(
-            tline.get_rel_size(TESTREL_A, Lsn(lsn))?,
+            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
             pg_constants::RELSEG_SIZE - 1
         );
 
+        // Truncate to 1500, and then truncate all the way down to 0, one block at a time
+        // This tests the behavior at segment boundaries
+        let mut size: i32 = 3000;
+        while size >= 0 {
+            lsn += 0x10;
+            tline.put_truncation(TESTREL_A, Lsn(lsn), size as u32)?;
+            tline.advance_last_record_lsn(Lsn(lsn));
+            assert_eq!(
+                tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
+                size as u32
+            );
+
+            size -= 1;
+        }
+
+        Ok(())
+    }
+
+    ///
+    /// Test list_rels() function, with branches and unlinking
+    ///
+    #[test]
+    fn test_list_rels_and_unlink() -> Result<()> {
+        let repo = get_test_repo("test_unlink")?;
+        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
+        let tline = repo.create_empty_timeline(timelineid, Lsn(0x00))?;
+
+        // Import initial dummy checkpoint record, otherwise the get_timeline() call
+        // after branching fails below
+        tline.put_page_image(RelishTag::Checkpoint, 0, Lsn(0x10), ZERO_PAGE.clone())?;
+
+        // Create a relation on the timeline
+        tline.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?;
+
+        // Check that list_rels() lists it after LSN 2, but no before it.
+        let reltag = match TESTREL_A {
+            RelishTag::Relation(reltag) => reltag,
+            _ => panic!("unexpected relish")
+        };
+        assert!(!tline.list_rels(0, TESTDB, Lsn(0x10))?.contains(&reltag));
+        assert!(tline.list_rels(0, TESTDB, Lsn(0x20))?.contains(&reltag));
+        assert!(tline.list_rels(0, TESTDB, Lsn(0x30))?.contains(&reltag));
+
+        // Create a branch, check that the relation is visible there
+        let newtimelineid = ZTimelineId::from_str("AA223344556677881122334455667788").unwrap();
+        repo.branch_timeline(timelineid, newtimelineid, Lsn(0x30))?;
+        let newtline = repo.get_timeline(newtimelineid)?;
+
+        assert!(newtline.list_rels(0, TESTDB, Lsn(0x30))?.contains(&reltag));
+
+        // Unlink it on the branch
+        newtline.drop_relish(TESTREL_A, Lsn(0x40))?;
+
+        // Check that it's no longer listed on the branch after the point where it was unlinked
+        assert!(newtline.list_rels(0, TESTDB, Lsn(0x30))?.contains(&reltag));
+        assert!(!newtline.list_rels(0, TESTDB, Lsn(0x40))?.contains(&reltag));
+
+        // Run checkpoint and garbage collection and check that it's still not visible
+        newtline.checkpoint()?;
+        repo.gc_iteration(Some(newtimelineid), 0, true)?;
+        assert!(!newtline.list_rels(0, TESTDB, Lsn(0x40))?.contains(&reltag));
+
+        Ok(())
+    }
+
+    ///
+    /// Test branch creation
+    ///
+    #[test]
+    fn test_branch() -> Result<()> {
+        let repo = get_test_repo("test_branch")?;
+        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
+        let tline = repo.create_empty_timeline(timelineid, Lsn(0x00))?;
+
+        // Import initial dummy checkpoint record, otherwise the get_timeline() call
+        // after branching fails below
+        tline.put_page_image(RelishTag::Checkpoint, 0, Lsn(0x10), ZERO_PAGE.clone())?;
+
+        // Create a relation on the timeline
+        tline.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(0x30), TEST_IMG("foo blk 0 at 3"))?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(0x40), TEST_IMG("foo blk 0 at 4"))?;
+
+        // Create another relation
+        tline.put_page_image(TESTREL_B, 0, Lsn(0x20), TEST_IMG("foobar blk 0 at 2"))?;
+
+        tline.advance_last_record_lsn(Lsn(0x40));
+
+        // Branch the history, modify relation differently on the new timeline
+        let newtimelineid = ZTimelineId::from_str("AA223344556677881122334455667788").unwrap();
+        repo.branch_timeline(timelineid, newtimelineid, Lsn(0x30))?;
+        let newtline = repo.get_timeline(newtimelineid)?;
+
+        newtline.put_page_image(TESTREL_A, 0, Lsn(0x40), TEST_IMG("bar blk 0 at 4"))?;
+        newtline.advance_last_record_lsn(Lsn(0x40));
+
+        // Check page contents on both branches
+        assert_eq!(
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x40))?,
+            TEST_IMG("foo blk 0 at 4")
+        );
+
+        assert_eq!(
+            newtline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x40))?,
+            TEST_IMG("bar blk 0 at 4")
+        );
+
+        assert_eq!(
+            newtline.get_page_at_lsn(TESTREL_B, 0, Lsn(0x40))?,
+            TEST_IMG("foobar blk 0 at 2")
+        );
+
+        assert_eq!(newtline.get_relish_size(TESTREL_B, Lsn(0x40))?.unwrap(), 1);
+
         Ok(())
     }
 
@@ -398,15 +523,16 @@ mod tests {
     impl WalRedoManager for TestRedoManager {
         fn request_redo(
             &self,
-            tag: BufferTag,
+            rel: RelishTag,
+            blknum: u32,
             lsn: Lsn,
             base_img: Option<Bytes>,
             records: Vec<WALRecord>,
         ) -> Result<Bytes, WalRedoError> {
             let s = format!(
-                "redo for rel {} blk {} to get to {}, with {} and {} records",
-                tag.rel,
-                tag.blknum,
+                "redo for {} blk {} to get to {}, with {} and {} records",
+                rel,
+                blknum,
                 lsn,
                 if base_img.is_some() {
                     "base image"

pageserver/src/restore_local_repo.rs

@@ -1,9 +1,11 @@
 //!
 //! Import data and WAL from a PostgreSQL data directory and WAL segments into
-//! zenith repository
+//! zenith Timeline.
 //!
 use log::*;
-use std::cmp::{max, min};
+use postgres_ffi::nonrelfile_utils::clogpage_precedes;
+use postgres_ffi::nonrelfile_utils::slru_may_delete_clogsegment;
+use std::cmp::min;
 use std::fs;
 use std::fs::File;
 use std::io::Read;
@@ -12,41 +14,22 @@ use std::io::SeekFrom;
 use std::path::{Path, PathBuf};
 
 use anyhow::Result;
-use bytes::Bytes;
+use bytes::{Buf, Bytes};
 
-use crate::repository::{BufferTag, RelTag, Timeline, WALRecord};
-use crate::waldecoder::{decode_wal_record, DecodedWALRecord, Oid, WalStreamDecoder};
-use crate::waldecoder::{XlCreateDatabase, XlSmgrTruncate};
-use crate::PageServerConf;
-use crate::ZTimelineId;
-use postgres_ffi::pg_constants;
+use crate::relish::*;
+use crate::repository::*;
+use crate::waldecoder::*;
+use postgres_ffi::nonrelfile_utils::mx_offset_to_member_segment;
 use postgres_ffi::relfile_utils::*;
 use postgres_ffi::xlog_utils::*;
+use postgres_ffi::Oid;
+use postgres_ffi::{pg_constants, CheckPoint, ControlFileData};
 use zenith_utils::lsn::Lsn;
 
-///
-/// Find latest snapshot in a timeline's 'snapshots' directory
-///
-pub fn find_latest_snapshot(_conf: &PageServerConf, timeline: ZTimelineId) -> Result<Lsn> {
-    let snapshotspath = format!("timelines/{}/snapshots", timeline);
+const MAX_MBR_BLKNO: u32 =
+    pg_constants::MAX_MULTIXACT_ID / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
 
-    let mut last_snapshot_lsn = Lsn(0);
-    for direntry in fs::read_dir(&snapshotspath).unwrap() {
-        let filename = direntry.unwrap().file_name();
-
-        if let Ok(lsn) = Lsn::from_filename(&filename) {
-            last_snapshot_lsn = max(lsn, last_snapshot_lsn);
-        } else {
-            error!("unrecognized file in snapshots directory: {:?}", filename);
-        }
-    }
-
-    if last_snapshot_lsn == Lsn(0) {
-        error!("could not find valid snapshot in {}", &snapshotspath);
-        // TODO return error?
-    }
-    Ok(last_snapshot_lsn)
-}
+const ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
 
 ///
 /// Import all relation data pages from local disk into the repository.
@@ -62,9 +45,24 @@ pub fn import_timeline_from_postgres_datadir(
         match direntry.file_name().to_str() {
             None => continue,
 
-            // These special files appear in the snapshot, but are not needed by the page server
-            Some("pg_control") => continue,
-            Some("pg_filenode.map") => continue,
+            Some("pg_control") => {
+                import_nonrel_file(timeline, lsn, RelishTag::ControlFile, &direntry.path())?;
+                // Extract checkpoint record from pg_control and store is as separate object
+                let pg_control_bytes =
+                    timeline.get_page_at_lsn_nowait(RelishTag::ControlFile, 0, lsn)?;
+                let pg_control = ControlFileData::decode(&pg_control_bytes)?;
+                let checkpoint_bytes = pg_control.checkPointCopy.encode();
+                timeline.put_page_image(RelishTag::Checkpoint, 0, lsn, checkpoint_bytes)?;
+            }
+            Some("pg_filenode.map") => import_nonrel_file(
+                timeline,
+                lsn,
+                RelishTag::FileNodeMap {
+                    spcnode: pg_constants::GLOBALTABLESPACE_OID,
+                    dbnode: 0,
+                },
+                &direntry.path(),
+            )?,
 
             // Load any relation files into the page server
             _ => import_relfile(
@@ -82,6 +80,11 @@ pub fn import_timeline_from_postgres_datadir(
     for direntry in fs::read_dir(path.join("base"))? {
         let direntry = direntry?;
 
+        //skip all temporary files
+        if direntry.file_name().to_str().unwrap() == "pgsql_tmp" {
+            continue;
+        }
+
         let dboid = direntry.file_name().to_str().unwrap().parse::<u32>()?;
 
         for direntry in fs::read_dir(direntry.path())? {
@@ -89,9 +92,16 @@ pub fn import_timeline_from_postgres_datadir(
             match direntry.file_name().to_str() {
                 None => continue,
 
-                // These special files appear in the snapshot, but are not needed by the page server
                 Some("PG_VERSION") => continue,
-                Some("pg_filenode.map") => continue,
+                Some("pg_filenode.map") => import_nonrel_file(
+                    timeline,
+                    lsn,
+                    RelishTag::FileNodeMap {
+                        spcnode: pg_constants::DEFAULTTABLESPACE_OID,
+                        dbnode: dboid,
+                    },
+                    &direntry.path(),
+                )?,
 
                 // Load any relation files into the page server
                 _ => import_relfile(
@@ -104,8 +114,26 @@ pub fn import_timeline_from_postgres_datadir(
             }
         }
     }
+    for entry in fs::read_dir(path.join("pg_xact"))? {
+        let entry = entry?;
+        import_slru_file(timeline, lsn, SlruKind::Clog, &entry.path())?;
+    }
+    for entry in fs::read_dir(path.join("pg_multixact").join("members"))? {
+        let entry = entry?;
+        import_slru_file(timeline, lsn, SlruKind::MultiXactMembers, &entry.path())?;
+    }
+    for entry in fs::read_dir(path.join("pg_multixact").join("offsets"))? {
+        let entry = entry?;
+        import_slru_file(timeline, lsn, SlruKind::MultiXactOffsets, &entry.path())?;
+    }
+    for entry in fs::read_dir(path.join("pg_twophase"))? {
+        let entry = entry?;
+        let xid = u32::from_str_radix(&entry.path().to_str().unwrap(), 16)?;
+        import_nonrel_file(timeline, lsn, RelishTag::TwoPhase { xid }, &entry.path())?;
+    }
     // TODO: Scan pg_tblspc
 
+    timeline.advance_last_record_lsn(lsn.align());
     timeline.checkpoint()?;
 
     Ok(())
@@ -123,7 +151,7 @@ fn import_relfile(
 
     let p = parse_relfilename(path.file_name().unwrap().to_str().unwrap());
     if let Err(e) = p {
-        warn!("unrecognized file in snapshot: {:?} ({})", path, e);
+        warn!("unrecognized file in postgres datadir: {:?} ({})", path, e);
         return Err(e.into());
     }
     let (relnode, forknum, segno) = p.unwrap();
@@ -136,21 +164,14 @@ fn import_relfile(
         let r = file.read_exact(&mut buf);
         match r {
             Ok(_) => {
-                let tag = BufferTag {
-                    rel: RelTag {
-                        spcnode: spcoid,
-                        dbnode: dboid,
-                        relnode,
-                        forknum,
-                    },
-                    blknum,
+                let rel = RelTag {
+                    spcnode: spcoid,
+                    dbnode: dboid,
+                    relnode,
+                    forknum,
                 };
-                timeline.put_page_image(tag, lsn, Bytes::copy_from_slice(&buf))?;
-                /*
-                if oldest_lsn == 0 || p.lsn < oldest_lsn {
-                    oldest_lsn = p.lsn;
-                }
-                 */
+                let tag = RelishTag::Relation(rel);
+                timeline.put_page_image(tag, blknum, lsn, Bytes::copy_from_slice(&buf))?;
             }
 
             // TODO: UnexpectedEof is expected
@@ -172,17 +193,92 @@ fn import_relfile(
     Ok(())
 }
 
-/// Scan PostgreSQL WAL files in given directory, and load all records >= 'startpoint' into
-/// the repository.
+///
+/// Import a "non-blocky" file into the repository
+///
+/// This is used for small files like the control file, twophase files etc. that
+/// are just slurped into the repository as one blob.
+///
+fn import_nonrel_file(
+    timeline: &dyn Timeline,
+    lsn: Lsn,
+    tag: RelishTag,
+    path: &Path,
+) -> Result<()> {
+    let mut file = File::open(path)?;
+    let mut buffer = Vec::new();
+    // read the whole file
+    file.read_to_end(&mut buffer)?;
+
+    info!("importing non-rel file {}", path.display());
+
+    timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buffer[..]))?;
+    Ok(())
+}
+
+///
+/// Import an SLRU segment file
+///
+fn import_slru_file(timeline: &dyn Timeline, lsn: Lsn, slru: SlruKind, path: &Path) -> Result<()> {
+    // Does it look like an SLRU file?
+    let mut file = File::open(path)?;
+    let mut buf: [u8; 8192] = [0u8; 8192];
+    let segno = u32::from_str_radix(path.file_name().unwrap().to_str().unwrap(), 16)?;
+
+    info!("importing slru file {}", path.display());
+
+    let mut rpageno = 0;
+    loop {
+        let r = file.read_exact(&mut buf);
+        match r {
+            Ok(_) => {
+                timeline.put_page_image(
+                    RelishTag::Slru { slru, segno },
+                    rpageno,
+                    lsn,
+                    Bytes::copy_from_slice(&buf),
+                )?;
+            }
+
+            // TODO: UnexpectedEof is expected
+            Err(e) => match e.kind() {
+                std::io::ErrorKind::UnexpectedEof => {
+                    // reached EOF. That's expected.
+                    // FIXME: maybe check that we read the full length of the file?
+                    break;
+                }
+                _ => {
+                    error!("error reading file: {:?} ({})", path, e);
+                    break;
+                }
+            },
+        };
+        rpageno += 1;
+
+        // TODO: Check that the file isn't unexpectedly large, not larger than SLRU_PAGES_PER_SEGMENT pages
+    }
+
+    Ok(())
+}
+
+/// Scan PostgreSQL WAL files in given directory
+/// and load all records >= 'startpoint' into the repository.
 pub fn import_timeline_wal(walpath: &Path, timeline: &dyn Timeline, startpoint: Lsn) -> Result<()> {
     let mut waldecoder = WalStreamDecoder::new(startpoint);
 
     let mut segno = startpoint.segment_number(pg_constants::WAL_SEGMENT_SIZE);
     let mut offset = startpoint.segment_offset(pg_constants::WAL_SEGMENT_SIZE);
     let mut last_lsn = startpoint;
+
+    let checkpoint_bytes = timeline.get_page_at_lsn_nowait(RelishTag::Checkpoint, 0, startpoint)?;
+    let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
+
     loop {
         // FIXME: assume postgresql tli 1 for now
         let filename = XLogFileName(1, segno, pg_constants::WAL_SEGMENT_SIZE);
+        let mut buf = Vec::new();
+
+        //Read local file
         let mut path = walpath.join(&filename);
 
         // It could be as .partial
@@ -203,12 +299,12 @@ pub fn import_timeline_wal(walpath: &Path, timeline: &dyn Timeline, startpoint:
             file.seek(SeekFrom::Start(offset as u64))?;
         }
 
-        let mut buf = Vec::new();
         let nread = file.read_to_end(&mut buf)?;
         if nread != pg_constants::WAL_SEGMENT_SIZE - offset as usize {
             // Maybe allow this for .partial files?
             error!("read only {} bytes from WAL file", nread);
         }
+
         waldecoder.feed_bytes(&buf);
 
         let mut nrecords = 0;
@@ -217,11 +313,12 @@ pub fn import_timeline_wal(walpath: &Path, timeline: &dyn Timeline, startpoint:
             if rec.is_err() {
                 // Assume that an error means we've reached the end of
                 // a partial WAL record. So that's ok.
+                trace!("WAL decoder error {:?}", rec);
                 break;
             }
             if let Some((lsn, recdata)) = rec.unwrap() {
                 let decoded = decode_wal_record(recdata.clone());
-                save_decoded_record(timeline, &decoded, recdata, lsn)?;
+                save_decoded_record(&mut checkpoint, timeline, &decoded, recdata, lsn)?;
                 last_lsn = lsn;
             } else {
                 break;
@@ -229,17 +326,18 @@ pub fn import_timeline_wal(walpath: &Path, timeline: &dyn Timeline, startpoint:
             nrecords += 1;
         }
 
-        info!(
-            "imported {} records from WAL file {} up to {}",
-            nrecords,
-            path.display(),
-            last_lsn
-        );
+        info!("imported {} records up to {}", nrecords, last_lsn);
 
         segno += 1;
         offset = 0;
     }
+
     info!("reached end of WAL at {}", last_lsn);
+    let checkpoint_bytes = checkpoint.encode();
+    timeline.put_page_image(RelishTag::Checkpoint, 0, last_lsn, checkpoint_bytes)?;
+
+    timeline.advance_last_record_lsn(last_lsn.align());
+    timeline.checkpoint()?;
     Ok(())
 }
 
@@ -248,23 +346,23 @@ pub fn import_timeline_wal(walpath: &Path, timeline: &dyn Timeline, startpoint:
 /// relations/pages that the record affects.
 ///
 pub fn save_decoded_record(
+    checkpoint: &mut CheckPoint,
     timeline: &dyn Timeline,
     decoded: &DecodedWALRecord,
     recdata: Bytes,
     lsn: Lsn,
 ) -> Result<()> {
+    checkpoint.update_next_xid(decoded.xl_xid);
+
     // Iterate through all the blocks that the record modifies, and
     // "put" a separate copy of the record for each block.
     for blk in decoded.blocks.iter() {
-        let tag = BufferTag {
-            rel: RelTag {
-                spcnode: blk.rnode_spcnode,
-                dbnode: blk.rnode_dbnode,
-                relnode: blk.rnode_relnode,
-                forknum: blk.forknum as u8,
-            },
-            blknum: blk.blkno,
-        };
+        let tag = RelishTag::Relation(RelTag {
+            spcnode: blk.rnode_spcnode,
+            dbnode: blk.rnode_dbnode,
+            relnode: blk.rnode_relnode,
+            forknum: blk.forknum as u8,
+        });
 
         let rec = WALRecord {
             lsn,
@@ -273,25 +371,170 @@ pub fn save_decoded_record(
             main_data_offset: decoded.main_data_offset as u32,
         };
 
-        timeline.put_wal_record(tag, rec)?;
+        timeline.put_wal_record(tag, blk.blkno, rec)?;
     }
 
+    let mut buf = decoded.record.clone();
+    buf.advance(decoded.main_data_offset);
+
     // Handle a few special record types
     if decoded.xl_rmid == pg_constants::RM_SMGR_ID
         && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) == pg_constants::XLOG_SMGR_TRUNCATE
     {
-        let truncate = XlSmgrTruncate::decode(&decoded);
+        let truncate = XlSmgrTruncate::decode(&mut buf);
         save_xlog_smgr_truncate(timeline, lsn, &truncate)?;
-    } else if decoded.xl_rmid == pg_constants::RM_DBASE_ID
-        && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) == pg_constants::XLOG_DBASE_CREATE
-    {
-        let createdb = XlCreateDatabase::decode(&decoded);
-        save_xlog_dbase_create(timeline, lsn, &createdb)?;
-    }
+    } else if decoded.xl_rmid == pg_constants::RM_DBASE_ID {
+        if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) == pg_constants::XLOG_DBASE_CREATE {
+            let createdb = XlCreateDatabase::decode(&mut buf);
+            save_xlog_dbase_create(timeline, lsn, &createdb)?;
+        } else if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK)
+            == pg_constants::XLOG_DBASE_DROP
+        {
+            let dropdb = XlDropDatabase::decode(&mut buf);
 
+            for tablespace_id in dropdb.tablespace_ids {
+                let rels = timeline.list_rels(tablespace_id, dropdb.db_id, lsn)?;
+                for rel in rels {
+                    timeline.drop_relish(RelishTag::Relation(rel), lsn)?;
+                }
+                trace!(
+                    "Drop FileNodeMap {}, {} at lsn {}",
+                    tablespace_id,
+                    dropdb.db_id,
+                    lsn
+                );
+                timeline.drop_relish(
+                    RelishTag::FileNodeMap {
+                        spcnode: tablespace_id,
+                        dbnode: dropdb.db_id,
+                    },
+                    lsn,
+                )?;
+            }
+        }
+    } else if decoded.xl_rmid == pg_constants::RM_TBLSPC_ID {
+        trace!("XLOG_TBLSPC_CREATE/DROP is not handled yet");
+    } else if decoded.xl_rmid == pg_constants::RM_CLOG_ID {
+        let info = decoded.xl_info & !pg_constants::XLR_INFO_MASK;
+        if info == pg_constants::CLOG_ZEROPAGE {
+            let pageno = buf.get_u32_le();
+            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+            timeline.put_page_image(
+                RelishTag::Slru {
+                    slru: SlruKind::Clog,
+                    segno,
+                },
+                rpageno,
+                lsn,
+                ZERO_PAGE,
+            )?;
+        } else {
+            assert!(info == pg_constants::CLOG_TRUNCATE);
+            let xlrec = XlClogTruncate::decode(&mut buf);
+            save_clog_truncate_record(checkpoint, timeline, lsn, &xlrec)?;
+        }
+    } else if decoded.xl_rmid == pg_constants::RM_XACT_ID {
+        let info = decoded.xl_info & pg_constants::XLOG_XACT_OPMASK;
+        if info == pg_constants::XLOG_XACT_COMMIT || info == pg_constants::XLOG_XACT_ABORT {
+            let parsed_xact = XlXactParsedRecord::decode(&mut buf, decoded.xl_xid, decoded.xl_info);
+            save_xact_record(timeline, lsn, &parsed_xact, decoded)?;
+        } else if info == pg_constants::XLOG_XACT_COMMIT_PREPARED
+            || info == pg_constants::XLOG_XACT_ABORT_PREPARED
+        {
+            let parsed_xact = XlXactParsedRecord::decode(&mut buf, decoded.xl_xid, decoded.xl_info);
+            save_xact_record(timeline, lsn, &parsed_xact, decoded)?;
+            // Remove twophase file. see RemoveTwoPhaseFile() in postgres code
+            trace!(
+                "Drop twophaseFile for xid {} parsed_xact.xid {} here at {}",
+                decoded.xl_xid,
+                parsed_xact.xid,
+                lsn
+            );
+            timeline.drop_relish(
+                RelishTag::TwoPhase {
+                    xid: parsed_xact.xid,
+                },
+                lsn,
+            )?;
+        } else if info == pg_constants::XLOG_XACT_PREPARE {
+            let mut buf = decoded.record.clone();
+            buf.advance(decoded.main_data_offset);
+
+            timeline.put_page_image(
+                RelishTag::TwoPhase {
+                    xid: decoded.xl_xid,
+                },
+                0,
+                lsn,
+                Bytes::copy_from_slice(&buf[..]),
+            )?;
+        }
+    } else if decoded.xl_rmid == pg_constants::RM_MULTIXACT_ID {
+        let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
+
+        if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE {
+            let pageno = buf.get_u32_le();
+            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+            timeline.put_page_image(
+                RelishTag::Slru {
+                    slru: SlruKind::MultiXactOffsets,
+                    segno,
+                },
+                rpageno,
+                lsn,
+                ZERO_PAGE,
+            )?;
+        } else if info == pg_constants::XLOG_MULTIXACT_ZERO_MEM_PAGE {
+            let pageno = buf.get_u32_le();
+            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+            timeline.put_page_image(
+                RelishTag::Slru {
+                    slru: SlruKind::MultiXactMembers,
+                    segno,
+                },
+                rpageno,
+                lsn,
+                ZERO_PAGE,
+            )?;
+        } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
+            let xlrec = XlMultiXactCreate::decode(&mut buf);
+            save_multixact_create_record(checkpoint, timeline, lsn, &xlrec, decoded)?;
+        } else if info == pg_constants::XLOG_MULTIXACT_TRUNCATE_ID {
+            let xlrec = XlMultiXactTruncate::decode(&mut buf);
+            save_multixact_truncate_record(checkpoint, timeline, lsn, &xlrec)?;
+        }
+    } else if decoded.xl_rmid == pg_constants::RM_RELMAP_ID {
+        let xlrec = XlRelmapUpdate::decode(&mut buf);
+        save_relmap_page(timeline, lsn, &xlrec, decoded)?;
+    } else if decoded.xl_rmid == pg_constants::RM_XLOG_ID {
+        let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
+        if info == pg_constants::XLOG_NEXTOID {
+            let next_oid = buf.get_u32_le();
+            checkpoint.nextOid = next_oid;
+        } else if info == pg_constants::XLOG_CHECKPOINT_ONLINE
+            || info == pg_constants::XLOG_CHECKPOINT_SHUTDOWN
+        {
+            let mut checkpoint_bytes = [0u8; SIZEOF_CHECKPOINT];
+            let mut buf = decoded.record.clone();
+            buf.advance(decoded.main_data_offset);
+            buf.copy_to_slice(&mut checkpoint_bytes);
+            let xlog_checkpoint = CheckPoint::decode(&checkpoint_bytes).unwrap();
+            trace!(
+                "xlog_checkpoint.oldestXid={}, checkpoint.oldestXid={}",
+                xlog_checkpoint.oldestXid,
+                checkpoint.oldestXid
+            );
+            if (checkpoint.oldestXid.wrapping_sub(xlog_checkpoint.oldestXid) as i32) < 0 {
+                checkpoint.oldestXid = xlog_checkpoint.oldestXid;
+            }
+        }
+    }
     // Now that this record has been handled, let the repository know that
     // it is up-to-date to this LSN
-    timeline.advance_last_record_lsn(lsn);
+    timeline.advance_last_record_lsn(lsn.align());
     Ok(())
 }
 
@@ -319,7 +562,9 @@ fn save_xlog_dbase_create(timeline: &dyn Timeline, lsn: Lsn, rec: &XlCreateDatab
         assert_eq!(src_rel.spcnode, src_tablespace_id);
         assert_eq!(src_rel.dbnode, src_db_id);
 
-        let nblocks = timeline.get_rel_size(src_rel, req_lsn)?;
+        let nblocks = timeline
+            .get_relish_size(RelishTag::Relation(src_rel), req_lsn)?
+            .unwrap_or(0);
         let dst_rel = RelTag {
             spcnode: tablespace_id,
             dbnode: db_id,
@@ -329,30 +574,42 @@ fn save_xlog_dbase_create(timeline: &dyn Timeline, lsn: Lsn, rec: &XlCreateDatab
 
         // Copy content
         for blknum in 0..nblocks {
-            let src_key = BufferTag {
-                rel: src_rel,
-                blknum,
-            };
-            let dst_key = BufferTag {
-                rel: dst_rel,
-                blknum,
-            };
+            let content =
+                timeline.get_page_at_lsn_nowait(RelishTag::Relation(src_rel), blknum, req_lsn)?;
 
-            let content = timeline.get_page_at_lsn(src_key, req_lsn)?;
+            debug!("copying block {} from {} to {}", blknum, src_rel, dst_rel);
 
-            info!("copying block {:?} to {:?}", src_key, dst_key);
-
-            timeline.put_page_image(dst_key, lsn, content)?;
+            timeline.put_page_image(RelishTag::Relation(dst_rel), blknum, lsn, content)?;
             num_blocks_copied += 1;
         }
 
         if nblocks == 0 {
             // make sure we have some trace of the relation, even if it's empty
-            timeline.put_truncation(dst_rel, lsn, 0)?;
+            timeline.put_truncation(RelishTag::Relation(dst_rel), lsn, 0)?;
         }
 
         num_rels_copied += 1;
     }
+
+    // Copy relfilemap
+    // TODO This implementation is very inefficient -
+    // it scans all non-rels only to find FileNodeMaps
+    for tag in timeline.list_nonrels(req_lsn)? {
+        match tag {
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                if spcnode == src_tablespace_id && dbnode == src_db_id {
+                    let img = timeline.get_page_at_lsn_nowait(tag, 0, req_lsn)?;
+                    let new_tag = RelishTag::FileNodeMap {
+                        spcnode: tablespace_id,
+                        dbnode: db_id,
+                    };
+                    timeline.put_page_image(new_tag, 0, lsn, img)?;
+                    break;
+                }
+            }
+            _ => {} // do nothing
+        }
+    }
     info!(
         "Created database {}/{}, copied {} blocks in {} rels at {}",
         tablespace_id, db_id, num_blocks_copied, num_rels_copied, lsn
@@ -375,7 +632,7 @@ fn save_xlog_smgr_truncate(timeline: &dyn Timeline, lsn: Lsn, rec: &XlSmgrTrunca
             relnode,
             forknum: pg_constants::MAIN_FORKNUM,
         };
-        timeline.put_truncation(rel, lsn, rec.blkno)?;
+        timeline.put_truncation(RelishTag::Relation(rel), lsn, rec.blkno)?;
     }
     if (rec.flags & pg_constants::SMGR_TRUNCATE_FSM) != 0 {
         let rel = RelTag {
@@ -398,7 +655,7 @@ fn save_xlog_smgr_truncate(timeline: &dyn Timeline, lsn: Lsn, rec: &XlSmgrTrunca
             info!("Partial truncation of FSM is not supported");
         }
         let num_fsm_blocks = 0;
-        timeline.put_truncation(rel, lsn, num_fsm_blocks)?;
+        timeline.put_truncation(RelishTag::Relation(rel), lsn, num_fsm_blocks)?;
     }
     if (rec.flags & pg_constants::SMGR_TRUNCATE_VM) != 0 {
         let rel = RelTag {
@@ -417,7 +674,263 @@ fn save_xlog_smgr_truncate(timeline: &dyn Timeline, lsn: Lsn, rec: &XlSmgrTrunca
             info!("Partial truncation of VM is not supported");
         }
         let num_vm_blocks = 0;
-        timeline.put_truncation(rel, lsn, num_vm_blocks)?;
+        timeline.put_truncation(RelishTag::Relation(rel), lsn, num_vm_blocks)?;
     }
     Ok(())
 }
+
+/// Subroutine of save_decoded_record(), to handle an XLOG_XACT_* records.
+///
+fn save_xact_record(
+    timeline: &dyn Timeline,
+    lsn: Lsn,
+    parsed: &XlXactParsedRecord,
+    decoded: &DecodedWALRecord,
+) -> Result<()> {
+    // Record update of CLOG page
+    let mut pageno = parsed.xid / pg_constants::CLOG_XACTS_PER_PAGE;
+
+    let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+    let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+    let rec = WALRecord {
+        lsn,
+        will_init: false,
+        rec: decoded.record.clone(),
+        main_data_offset: decoded.main_data_offset as u32,
+    };
+    timeline.put_wal_record(
+        RelishTag::Slru {
+            slru: SlruKind::Clog,
+            segno,
+        },
+        rpageno,
+        rec.clone(),
+    )?;
+
+    for subxact in &parsed.subxacts {
+        let subxact_pageno = subxact / pg_constants::CLOG_XACTS_PER_PAGE;
+        if subxact_pageno != pageno {
+            pageno = subxact_pageno;
+            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+            timeline.put_wal_record(
+                RelishTag::Slru {
+                    slru: SlruKind::Clog,
+                    segno,
+                },
+                rpageno,
+                rec.clone(),
+            )?;
+        }
+    }
+    for xnode in &parsed.xnodes {
+        for forknum in pg_constants::MAIN_FORKNUM..=pg_constants::VISIBILITYMAP_FORKNUM {
+            let rel = RelTag {
+                forknum,
+                spcnode: xnode.spcnode,
+                dbnode: xnode.dbnode,
+                relnode: xnode.relnode,
+            };
+            timeline.drop_relish(RelishTag::Relation(rel), lsn)?;
+        }
+    }
+    Ok(())
+}
+
+fn save_clog_truncate_record(
+    checkpoint: &mut CheckPoint,
+    timeline: &dyn Timeline,
+    lsn: Lsn,
+    xlrec: &XlClogTruncate,
+) -> Result<()> {
+    info!(
+        "RM_CLOG_ID truncate pageno {} oldestXid {} oldestXidDB {} lsn {}",
+        xlrec.pageno, xlrec.oldest_xid, xlrec.oldest_xid_db, lsn
+    );
+
+    // Here we treat oldestXid and oldestXidDB
+    // differently from postgres redo routines.
+    // In postgres checkpoint.oldestXid lags behind xlrec.oldest_xid
+    // until checkpoint happens and updates the value.
+    // Here we can use the most recent value.
+    // It's just an optimization, though and can be deleted.
+    // TODO Figure out if there will be any issues with replica.
+    checkpoint.oldestXid = xlrec.oldest_xid;
+    checkpoint.oldestXidDB = xlrec.oldest_xid_db;
+
+    // TODO Treat AdvanceOldestClogXid() or write a comment why we don't need it
+
+    let latest_page_number = checkpoint.nextXid.value as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
+
+    // Now delete all segments containing pages between xlrec.pageno
+    // and latest_page_number.
+
+    // First, make an important safety check:
+    // the current endpoint page must not be eligible for removal.
+    // See SimpleLruTruncate() in slru.c
+    if clogpage_precedes(latest_page_number, xlrec.pageno) {
+        info!("could not truncate directory pg_xact apparent wraparound");
+        return Ok(());
+    }
+
+    // Iterate via SLRU CLOG segments and drop segments that we're ready to truncate
+    // TODO This implementation is very inefficient -
+    // it scans all non-rels only to find Clog
+    //
+    // We cannot pass 'lsn' to the Timeline.list_nonrels(), or it
+    // will block waiting for the last valid LSN to advance up to
+    // it. So we use the previous record's LSN in the get calls
+    // instead.
+    let req_lsn = min(timeline.get_last_record_lsn(), lsn);
+    for obj in timeline.list_nonrels(req_lsn)? {
+        match obj {
+            RelishTag::Slru { slru, segno } => {
+                if slru == SlruKind::Clog {
+                    let segpage = segno * pg_constants::SLRU_PAGES_PER_SEGMENT;
+                    if slru_may_delete_clogsegment(segpage, xlrec.pageno) {
+                        timeline.drop_relish(RelishTag::Slru { slru, segno }, lsn)?;
+                        trace!("Drop CLOG segment {:>04X} at lsn {}", segno, lsn);
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
+
+    Ok(())
+}
+
+fn save_multixact_create_record(
+    checkpoint: &mut CheckPoint,
+    timeline: &dyn Timeline,
+    lsn: Lsn,
+    xlrec: &XlMultiXactCreate,
+    decoded: &DecodedWALRecord,
+) -> Result<()> {
+    let rec = WALRecord {
+        lsn,
+        will_init: false,
+        rec: decoded.record.clone(),
+        main_data_offset: decoded.main_data_offset as u32,
+    };
+    let pageno = xlrec.mid / pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32;
+    let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+    let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+    timeline.put_wal_record(
+        RelishTag::Slru {
+            slru: SlruKind::MultiXactOffsets,
+            segno,
+        },
+        rpageno,
+        rec.clone(),
+    )?;
+
+    let first_mbr_pageno = xlrec.moff / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
+    let last_mbr_pageno =
+        (xlrec.moff + xlrec.nmembers - 1) / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
+    // The members SLRU can, in contrast to the offsets one, be filled to almost
+    // the full range at once. So we need to handle wraparound.
+    let mut pageno = first_mbr_pageno;
+    loop {
+        // Update members page
+        let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+        let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+        timeline.put_wal_record(
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno,
+            },
+            rpageno,
+            rec.clone(),
+        )?;
+
+        if pageno == last_mbr_pageno {
+            // last block inclusive
+            break;
+        }
+
+        // handle wraparound
+        if pageno == MAX_MBR_BLKNO {
+            pageno = 0;
+        } else {
+            pageno += 1;
+        }
+    }
+    if xlrec.mid >= checkpoint.nextMulti {
+        checkpoint.nextMulti = xlrec.mid + 1;
+    }
+    if xlrec.moff + xlrec.nmembers > checkpoint.nextMultiOffset {
+        checkpoint.nextMultiOffset = xlrec.moff + xlrec.nmembers;
+    }
+    let max_mbr_xid = xlrec.members.iter().fold(0u32, |acc, mbr| {
+        if mbr.xid.wrapping_sub(acc) as i32 > 0 {
+            mbr.xid
+        } else {
+            acc
+        }
+    });
+
+    checkpoint.update_next_xid(max_mbr_xid);
+    Ok(())
+}
+
+fn save_multixact_truncate_record(
+    checkpoint: &mut CheckPoint,
+    timeline: &dyn Timeline,
+    lsn: Lsn,
+    xlrec: &XlMultiXactTruncate,
+) -> Result<()> {
+    checkpoint.oldestMulti = xlrec.end_trunc_off;
+    checkpoint.oldestMultiDB = xlrec.oldest_multi_db;
+
+    // PerformMembersTruncation
+    let maxsegment: i32 = mx_offset_to_member_segment(pg_constants::MAX_MULTIXACT_OFFSET);
+    let startsegment: i32 = mx_offset_to_member_segment(xlrec.start_trunc_memb);
+    let endsegment: i32 = mx_offset_to_member_segment(xlrec.end_trunc_memb);
+    let mut segment: i32 = startsegment;
+
+    // Delete all the segments except the last one. The last segment can still
+    // contain, possibly partially, valid data.
+    while segment != endsegment {
+        timeline.drop_relish(
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno: segment as u32,
+            },
+            lsn,
+        )?;
+
+        /* move to next segment, handling wraparound correctly */
+        if segment == maxsegment {
+            segment = 0;
+        } else {
+            segment += 1;
+        }
+    }
+
+    // Truncate offsets
+    // FIXME: this did not handle wraparound correctly
+
+    Ok(())
+}
+
+fn save_relmap_page(
+    timeline: &dyn Timeline,
+    lsn: Lsn,
+    xlrec: &XlRelmapUpdate,
+    decoded: &DecodedWALRecord,
+) -> Result<()> {
+    let tag = RelishTag::FileNodeMap {
+        spcnode: xlrec.tsid,
+        dbnode: xlrec.dbid,
+    };
+
+    let mut buf = decoded.record.clone();
+    buf.advance(decoded.main_data_offset);
+    // skip xl_relmap_update
+    buf.advance(12);
+
+    timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buf[..]))?;
+
+    Ok(())
+}

pageserver/src/restore_s3.rs

@@ -1,274 +0,0 @@
-//
-// Restore chunks from S3
-//
-// This runs once at Page Server startup. It loads all the "base images" from
-// S3 into the in-memory page cache. It also initializes the "last valid LSN"
-// in the page cache to the LSN of the base image, so that when the WAL receiver
-// is started, it starts streaming from that LSN.
-//
-
-use bytes::{Buf, BytesMut};
-use log::*;
-use regex::Regex;
-use std::env;
-use std::fmt;
-
-use s3::bucket::Bucket;
-use s3::creds::Credentials;
-use s3::region::Region;
-use s3::S3Error;
-
-use tokio::runtime;
-
-use futures::future;
-
-use crate::{page_cache, PageServerConf};
-use postgres_ffi::pg_constants;
-use postgres_ffi::relfile_utils::*;
-
-struct Storage {
-    region: Region,
-    credentials: Credentials,
-    bucket: String,
-}
-
-pub fn restore_main(conf: &PageServerConf) {
-    // Create a new thread pool
-    let runtime = runtime::Runtime::new().unwrap();
-
-    runtime.block_on(async {
-        let result = restore_chunk(conf).await;
-
-        match result {
-            Ok(_) => {}
-            Err(err) => {
-                error!("S3 error: {}", err);
-            }
-        }
-    });
-}
-
-//
-// Restores one chunk from S3.
-//
-// 1. Fetch the last base image >= given LSN
-// 2. Fetch all WAL
-//
-// Load it all into the page cache.
-//
-async fn restore_chunk(conf: &PageServerConf) -> Result<(), S3Error> {
-    let backend = Storage {
-        region: Region::Custom {
-            region: env::var("S3_REGION").unwrap(),
-            endpoint: env::var("S3_ENDPOINT").unwrap(),
-        },
-        credentials: Credentials::new(
-            Some(&env::var("S3_ACCESSKEY").unwrap()),
-            Some(&env::var("S3_SECRET").unwrap()),
-            None,
-            None,
-            None,
-        )
-        .unwrap(),
-        bucket: "zenith-testbucket".to_string(),
-    };
-
-    info!("Restoring from S3...");
-
-    // Create Bucket in REGION for BUCKET
-    let bucket = Bucket::new_with_path_style(&backend.bucket, backend.region, backend.credentials)?;
-
-    // List out contents of directory
-    let results: Vec<s3::serde_types::ListBucketResult> = bucket
-        .list("relationdata/".to_string(), Some("".to_string()))
-        .await?;
-
-    // TODO: get that from backup
-    let sys_id: u64 = 42;
-    let mut oldest_lsn = 0;
-    let mut slurp_futures: Vec<_> = Vec::new();
-
-    for result in results {
-        for object in result.contents {
-            // Download every relation file, slurping them into memory
-
-            let key = object.key;
-            let relpath = key.strip_prefix("relationdata/").unwrap();
-
-            let parsed = parse_rel_file_path(&relpath);
-
-            match parsed {
-                Ok(p) => {
-                    if oldest_lsn == 0 || p.lsn < oldest_lsn {
-                        oldest_lsn = p.lsn;
-                    }
-                    let b = bucket.clone();
-                    let f = slurp_base_file(conf, sys_id, b, key.to_string(), p);
-
-                    slurp_futures.push(f);
-                }
-                Err(e) => {
-                    warn!("unrecognized file: {} ({})", relpath, e);
-                }
-            };
-        }
-    }
-
-    if oldest_lsn == 0 {
-        panic!("no base backup found");
-    }
-
-    let pcache = page_cache::get_pagecache(conf, sys_id);
-    pcache.init_valid_lsn(oldest_lsn);
-
-    info!("{} files to restore...", slurp_futures.len());
-
-    future::join_all(slurp_futures).await;
-    info!("restored!");
-
-    Ok(())
-}
-
-#[derive(Debug)]
-struct ParsedBaseImageFileName {
-    pub spcnode: u32,
-    pub dbnode: u32,
-    pub relnode: u32,
-    pub forknum: u8,
-    pub segno: u32,
-
-    pub lsn: u64,
-}
-
-// formats:
-// <oid>
-// <oid>_<fork name>
-// <oid>.<segment number>
-// <oid>_<fork name>.<segment number>
-
-fn parse_filename(fname: &str) -> Result<(u32, u8, u32, u64), FilePathError> {
-    let re = Regex::new(r"^(?P<relnode>\d+)(_(?P<forkname>[a-z]+))?(\.(?P<segno>\d+))?_(?P<lsnhi>[[:xdigit:]]{8})(?P<lsnlo>[[:xdigit:]]{8})$").unwrap();
-
-    let caps = re
-        .captures(fname)
-        .ok_or_else(|| FilePathError::new("invalid relation data file name"))?;
-
-    let relnode_str = caps.name("relnode").unwrap().as_str();
-    let relnode: u32 = relnode_str.parse()?;
-
-    let forkname = caps.name("forkname").map(|f| f.as_str());
-    let forknum = forkname_to_forknum(forkname)?;
-
-    let segno_match = caps.name("segno");
-    let segno = if segno_match.is_none() {
-        0
-    } else {
-        segno_match.unwrap().as_str().parse::<u32>()?
-    };
-
-    let lsn_hi: u64 = caps.name("lsnhi").unwrap().as_str().parse()?;
-    let lsn_lo: u64 = caps.name("lsnlo").unwrap().as_str().parse()?;
-    let lsn = lsn_hi << 32 | lsn_lo;
-
-    Ok((relnode, forknum, segno, lsn))
-}
-
-fn parse_rel_file_path(path: &str) -> Result<ParsedBaseImageFileName, FilePathError> {
-    /*
-     * Relation data files can be in one of the following directories:
-     *
-     * global/
-     *		shared relations
-     *
-     * base/<db oid>/
-     *		regular relations, default tablespace
-     *
-     * pg_tblspc/<tblspc oid>/<tblspc version>/
-     *		within a non-default tablespace (the name of the directory
-     *		depends on version)
-     *
-     * And the relation data files themselves have a filename like:
-     *
-     * <oid>.<segment number>
-     */
-    if let Some(fname) = path.strip_prefix("global/") {
-        let (relnode, forknum, segno, lsn) = parse_filename(fname)?;
-
-        Ok(ParsedBaseImageFileName {
-            spcnode: pg_constants::GLOBALTABLESPACE_OID,
-            dbnode: 0,
-            relnode,
-            forknum,
-            segno,
-            lsn,
-        })
-    } else if let Some(dbpath) = path.strip_prefix("base/") {
-        let mut s = dbpath.split("/");
-        let dbnode_str = s
-            .next()
-            .ok_or_else(|| FilePathError::new("invalid relation data file name"))?;
-        let dbnode: u32 = dbnode_str.parse()?;
-        let fname = s
-            .next()
-            .ok_or_else(|| FilePathError::new("invalid relation data file name"))?;
-        if s.next().is_some() {
-            return Err(FilePathError::new("invalid relation data file name"));
-        };
-
-        let (relnode, forknum, segno, lsn) = parse_filename(fname)?;
-
-        Ok(ParsedBaseImageFileName {
-            spcnode: pg_constants::DEFAULTTABLESPACE_OID,
-            dbnode,
-            relnode,
-            forknum,
-            segno,
-            lsn,
-        })
-    } else if let Some(_) = path.strip_prefix("pg_tblspc/") {
-        // TODO
-        Err(FilePathError::new("tablespaces not supported"))
-    } else {
-        Err(FilePathError::new("invalid relation data file name"))
-    }
-}
-
-//
-// Load a base file from S3, and insert it into the page cache
-//
-async fn slurp_base_file(
-    conf: &PageServerConf,
-    sys_id: u64,
-    bucket: Bucket,
-    s3path: String,
-    parsed: ParsedBaseImageFileName,
-) {
-    // FIXME: rust-s3 opens a new connection for each request. Should reuse
-    // the reqwest::Client object. But that requires changes to rust-s3 itself.
-    let (data, code) = bucket.get_object(s3path.clone()).await.unwrap();
-
-    trace!("got response: {} on {}", code, &s3path);
-    assert_eq!(200, code);
-
-    let mut bytes = BytesMut::from(data.as_slice()).freeze();
-
-    let mut blknum: u32 = parsed.segno * (1024 * 1024 * 1024 / pg_constants::BLCKSZ as u32);
-
-    let pcache = page_cache::get_pagecache(conf, sys_id);
-
-    while bytes.remaining() >= 8192 {
-        let tag = page_cache::BufferTag {
-            rel: page_cache::RelTag {
-                spcnode: parsed.spcnode,
-                dbnode: parsed.dbnode,
-                relnode: parsed.relnode,
-                forknum: parsed.forknum,
-            },
-            blknum,
-        };
-
-        pcache.put_page_image(tag, parsed.lsn, bytes.copy_to_bytes(8192));
-
-        blknum += 1;
-    }
-}

pageserver/src/rocksdb_storage.rs

@@ -1,208 +0,0 @@
-//!
-//! An implementation of the ObjectStore interface, backed by RocksDB
-//!
-use crate::object_store::{ObjectKey, ObjectStore};
-use crate::repository::{BufferTag, RelTag};
-use crate::PageServerConf;
-use crate::ZTimelineId;
-use anyhow::{bail, Result};
-use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
-use zenith_utils::bin_ser::BeSer;
-use zenith_utils::lsn::Lsn;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-struct StorageKey {
-    obj_key: ObjectKey,
-    lsn: Lsn,
-}
-
-pub struct RocksObjectStore {
-    _conf: &'static PageServerConf,
-
-    // RocksDB handle
-    db: rocksdb::DB,
-}
-
-impl ObjectStore for RocksObjectStore {
-    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>> {
-        let val = self.db.get(StorageKey::ser(&StorageKey {
-            obj_key: key.clone(),
-            lsn,
-        })?)?;
-        if let Some(val) = val {
-            Ok(val)
-        } else {
-            bail!("could not find page {:?}", key);
-        }
-    }
-
-    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()> {
-        self.db.put(
-            StorageKey::ser(&StorageKey {
-                obj_key: key.clone(),
-                lsn,
-            })?,
-            value,
-        )?;
-        Ok(())
-    }
-
-    /// Iterate through page versions of given page, starting from the given LSN.
-    /// The versions are walked in descending LSN order.
-    fn object_versions<'a>(
-        &'a self,
-        key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>> {
-        let iter = RocksObjectVersionIter::new(&self.db, key, lsn)?;
-        Ok(Box::new(iter))
-    }
-
-    /// Get a list of all distinct relations in given tablespace and database.
-    ///
-    /// TODO: This implementation is very inefficient, it scans
-    /// through all entries in the given database. In practice, this
-    /// is used for CREATE DATABASE, and usually the template database is small.
-    /// But if it's not, this will be slow.
-    fn list_rels(
-        &self,
-        timelineid: ZTimelineId,
-        spcnode: u32,
-        dbnode: u32,
-        lsn: Lsn,
-    ) -> Result<HashSet<RelTag>> {
-        // FIXME: This scans everything. Very slow
-
-        let mut rels: HashSet<RelTag> = HashSet::new();
-
-        let searchkey = StorageKey {
-            obj_key: ObjectKey {
-                timeline: timelineid,
-                buf_tag: BufferTag {
-                    rel: RelTag {
-                        spcnode,
-                        dbnode,
-                        relnode: 0,
-                        forknum: 0u8,
-                    },
-                    blknum: 0,
-                },
-            },
-            lsn: Lsn(0),
-        };
-        let mut iter = self.db.raw_iterator();
-        iter.seek(searchkey.ser()?);
-        while iter.valid() {
-            let key = StorageKey::des(iter.key().unwrap())?;
-            if key.obj_key.buf_tag.rel.spcnode != spcnode
-                || key.obj_key.buf_tag.rel.dbnode != dbnode
-            {
-                break;
-            }
-
-            if key.lsn < lsn {
-                rels.insert(key.obj_key.buf_tag.rel);
-            }
-            iter.next();
-        }
-
-        Ok(rels)
-    }
-}
-
-impl RocksObjectStore {
-    /// Open a RocksDB database.
-    pub fn open(conf: &'static PageServerConf) -> Result<RocksObjectStore> {
-        let path = conf.workdir.join("rocksdb-storage");
-        let db = rocksdb::DB::open(&Self::get_rocksdb_opts(), path)?;
-
-        let storage = RocksObjectStore { _conf: conf, db };
-        Ok(storage)
-    }
-
-    /// Create a new, empty RocksDB database.
-    pub fn create(conf: &'static PageServerConf) -> Result<RocksObjectStore> {
-        let path = conf.workdir.join("rocksdb-storage");
-        std::fs::create_dir(&path)?;
-
-        let mut opts = Self::get_rocksdb_opts();
-        opts.create_if_missing(true);
-        opts.set_error_if_exists(true);
-        let db = rocksdb::DB::open(&opts, &path)?;
-
-        let obj_store = RocksObjectStore { _conf: conf, db };
-        Ok(obj_store)
-    }
-
-    /// common options used by `open` and `create`
-    fn get_rocksdb_opts() -> rocksdb::Options {
-        let mut opts = rocksdb::Options::default();
-        opts.set_use_fsync(true);
-        opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
-
-        // FIXME
-        /*
-                opts.set_compaction_filter("ttl", move |_level: u32, _key: &[u8], val: &[u8]| {
-                    if (val[0] & UNUSED_VERSION_FLAG) != 0 {
-                        rocksdb::compaction_filter::Decision::Remove
-                    } else {
-                        rocksdb::compaction_filter::Decision::Keep
-                    }
-                });
-        */
-        opts
-    }
-}
-
-///
-/// Iterator for `object_versions`. Returns all page versions of a given block, in
-/// reverse LSN order.
-///
-struct RocksObjectVersionIter<'a> {
-    obj_key: ObjectKey,
-    dbiter: rocksdb::DBRawIterator<'a>,
-    first_call: bool,
-}
-impl<'a> RocksObjectVersionIter<'a> {
-    fn new(
-        db: &'a rocksdb::DB,
-        obj_key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<RocksObjectVersionIter<'a>> {
-        let key = StorageKey {
-            obj_key: obj_key.clone(),
-            lsn,
-        };
-        let mut dbiter = db.raw_iterator();
-        dbiter.seek_for_prev(StorageKey::ser(&key)?); // locate last entry
-        Ok(RocksObjectVersionIter {
-            first_call: true,
-            obj_key: obj_key.clone(),
-            dbiter,
-        })
-    }
-}
-impl<'a> Iterator for RocksObjectVersionIter<'a> {
-    type Item = (Lsn, Vec<u8>);
-
-    fn next(&mut self) -> std::option::Option<Self::Item> {
-        if self.first_call {
-            self.first_call = false;
-        } else {
-            self.dbiter.prev(); // walk backwards
-        }
-
-        if !self.dbiter.valid() {
-            return None;
-        }
-        let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
-        if key.obj_key.buf_tag != self.obj_key.buf_tag {
-            return None;
-        }
-        let val = self.dbiter.value().unwrap();
-        let result = val.to_vec();
-
-        Some((key.lsn, result))
-    }
-}

pageserver/src/tenant_mgr.rs

@@ -0,0 +1,74 @@
+//! This module acts as a switchboard to access different repositories managed by this
+//! page server.
+
+use crate::branches;
+use crate::layered_repository::LayeredRepository;
+use crate::repository::Repository;
+use crate::walredo::PostgresRedoManager;
+use crate::PageServerConf;
+use anyhow::{anyhow, bail, Result};
+use lazy_static::lazy_static;
+use log::info;
+use std::collections::HashMap;
+use std::fs;
+use std::str::FromStr;
+use std::sync::{Arc, Mutex};
+use zenith_utils::zid::ZTenantId;
+
+lazy_static! {
+    pub static ref REPOSITORY: Mutex<HashMap<ZTenantId, Arc<dyn Repository>>> =
+        Mutex::new(HashMap::new());
+}
+
+pub fn init(conf: &'static PageServerConf) {
+    let mut m = REPOSITORY.lock().unwrap();
+
+    for dir_entry in fs::read_dir(conf.tenants_path()).unwrap() {
+        let tenantid =
+            ZTenantId::from_str(dir_entry.unwrap().file_name().to_str().unwrap()).unwrap();
+
+        // Set up a WAL redo manager, for applying WAL records.
+        let walredo_mgr = PostgresRedoManager::new(conf, tenantid);
+
+        // Set up an object repository, for actual data storage.
+        let repo = Arc::new(LayeredRepository::new(
+            conf,
+            Arc::new(walredo_mgr),
+            tenantid,
+        ));
+        LayeredRepository::launch_checkpointer_thread(conf, repo.clone());
+
+        info!("initialized storage for tenant: {}", &tenantid);
+        m.insert(tenantid, repo);
+    }
+}
+
+pub fn create_repository_for_tenant(
+    conf: &'static PageServerConf,
+    tenantid: ZTenantId,
+) -> Result<()> {
+    let mut m = REPOSITORY.lock().unwrap();
+
+    // First check that the tenant doesn't exist already
+    if m.get(&tenantid).is_some() {
+        bail!("tenant {} already exists", tenantid);
+    }
+    let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenantid));
+    let repo = branches::create_repo(conf, tenantid, wal_redo_manager)?;
+
+    m.insert(tenantid, repo);
+
+    Ok(())
+}
+
+pub fn insert_repository_for_tenant(tenantid: ZTenantId, repo: Arc<dyn Repository>) {
+    let o = &mut REPOSITORY.lock().unwrap();
+    o.insert(tenantid, repo);
+}
+
+pub fn get_repository_for_tenant(tenantid: &ZTenantId) -> Result<Arc<dyn Repository>> {
+    let o = &REPOSITORY.lock().unwrap();
+    o.get(tenantid)
+        .map(|repo| Arc::clone(repo))
+        .ok_or_else(|| anyhow!("repository not found for tenant name {}", tenantid))
+}

pageserver/src/tui.rs

@@ -1,307 +0,0 @@
-use crate::tui_event::{Event, Events};
-use crate::tui_logger::TuiLogger;
-use crate::tui_logger::TuiLoggerWidget;
-
-use lazy_static::lazy_static;
-use std::sync::Arc;
-use std::{error::Error, io};
-use termion::{event::Key, input::MouseTerminal, raw::IntoRawMode, screen::AlternateScreen};
-use tui::backend::TermionBackend;
-use tui::buffer::Buffer;
-use tui::layout::{Constraint, Direction, Layout, Rect};
-use tui::style::{Color, Modifier, Style};
-use tui::text::{Span, Spans, Text};
-use tui::widgets::{Block, BorderType, Borders, Paragraph, Widget};
-use tui::Terminal;
-
-use slog::Drain;
-
-lazy_static! {
-    pub static ref PAGESERVICE_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
-    pub static ref WALRECEIVER_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
-    pub static ref WALREDO_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
-    pub static ref CATCHALL_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
-}
-
-pub fn init_logging() -> slog_scope::GlobalLoggerGuard {
-    let pageservice_drain =
-        slog::Filter::new(PAGESERVICE_DRAIN.as_ref(), |record: &slog::Record| {
-            if record.level().is_at_least(slog::Level::Debug)
-                && record.module().starts_with("pageserver::page_service")
-            {
-                return true;
-            }
-            false
-        })
-        .fuse();
-
-    let walredo_drain = slog::Filter::new(WALREDO_DRAIN.as_ref(), |record: &slog::Record| {
-        if record.level().is_at_least(slog::Level::Debug)
-            && record.module().starts_with("pageserver::walredo")
-        {
-            return true;
-        }
-        false
-    })
-    .fuse();
-
-    let walreceiver_drain =
-        slog::Filter::new(WALRECEIVER_DRAIN.as_ref(), |record: &slog::Record| {
-            if record.level().is_at_least(slog::Level::Debug)
-                && record.module().starts_with("pageserver::walreceiver")
-            {
-                return true;
-            }
-            false
-        })
-        .fuse();
-
-    let catchall_drain = slog::Filter::new(CATCHALL_DRAIN.as_ref(), |record: &slog::Record| {
-        if record.level().is_at_least(slog::Level::Info) {
-            return true;
-        }
-        if record.level().is_at_least(slog::Level::Debug)
-            && record.module().starts_with("pageserver")
-        {
-            return true;
-        }
-        false
-    })
-    .fuse();
-
-    let drain = pageservice_drain;
-    let drain = slog::Duplicate::new(drain, walreceiver_drain).fuse();
-    let drain = slog::Duplicate::new(drain, walredo_drain).fuse();
-    let drain = slog::Duplicate::new(drain, catchall_drain).fuse();
-    let drain = slog_async::Async::new(drain).chan_size(1000).build().fuse();
-    let drain = slog::Filter::new(drain, |record: &slog::Record| {
-        if record.level().is_at_least(slog::Level::Info) {
-            return true;
-        }
-        if record.level().is_at_least(slog::Level::Debug)
-            && record.module().starts_with("pageserver")
-        {
-            return true;
-        }
-
-        false
-    })
-    .fuse();
-    let logger = slog::Logger::root(drain, slog::o!());
-    slog_scope::set_global_logger(logger)
-}
-
-pub fn ui_main() -> Result<(), Box<dyn Error>> {
-    // Terminal initialization
-    let stdout = io::stdout().into_raw_mode()?;
-    let stdout = MouseTerminal::from(stdout);
-    let stdout = AlternateScreen::from(stdout);
-    let backend = TermionBackend::new(stdout);
-    let mut terminal = Terminal::new(backend)?;
-
-    // Setup event handlers
-    let events = Events::new();
-
-    loop {
-        terminal.draw(|f| {
-            let size = f.size();
-
-            // +----------------+----------------+
-            // |                |                |
-            // |  top_top_left  | top_top_right  |
-            // |                |                |
-            // +----------------+----------------|
-            // |                |                |
-            // |  top_bot_left  | top_left_right |
-            // |                |                |
-            // +----------------+----------------+
-            // |                                 |
-            // |             bottom              |
-            // |                                 |
-            // +---------------------------------+
-            let chunks = Layout::default()
-                .direction(Direction::Vertical)
-                .constraints([Constraint::Percentage(70), Constraint::Percentage(30)].as_ref())
-                .split(size);
-            let top_chunk = chunks[0];
-            let bottom_chunk = chunks[1];
-
-            let top_chunks = Layout::default()
-                .direction(Direction::Horizontal)
-                .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
-                .split(top_chunk);
-            let top_left_chunk = top_chunks[0];
-            let top_right_chunk = top_chunks[1];
-
-            let c = Layout::default()
-                .direction(Direction::Vertical)
-                .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
-                .split(top_left_chunk);
-            let top_top_left_chunk = c[0];
-            let top_bot_left_chunk = c[1];
-
-            let c = Layout::default()
-                .direction(Direction::Vertical)
-                .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
-                .split(top_right_chunk);
-            let top_top_right_chunk = c[0];
-            let top_bot_right_chunk = c[1];
-
-            f.render_widget(
-                LogWidget::new(PAGESERVICE_DRAIN.as_ref(), "Page Service"),
-                top_top_left_chunk,
-            );
-
-            f.render_widget(
-                LogWidget::new(WALREDO_DRAIN.as_ref(), "WAL Redo"),
-                top_bot_left_chunk,
-            );
-
-            f.render_widget(
-                LogWidget::new(WALRECEIVER_DRAIN.as_ref(), "WAL Receiver"),
-                top_top_right_chunk,
-            );
-
-            f.render_widget(MetricsWidget {}, top_bot_right_chunk);
-
-            f.render_widget(
-                LogWidget::new(CATCHALL_DRAIN.as_ref(), "All Log").show_module(true),
-                bottom_chunk,
-            );
-        })?;
-
-        // If ther user presses 'q', quit.
-
-        // silence clippy's suggestion to rewrite this as an if-statement. Match
-        // makes more sense as soon as we get another command than 'q'.
-        #[allow(clippy::single_match)]
-        #[allow(clippy::collapsible_match)]
-        if let Event::Input(key) = events.next()? {
-            match key {
-                Key::Char('q') => {
-                    break;
-                }
-                _ => (),
-            }
-        }
-    }
-
-    terminal.show_cursor().unwrap();
-    terminal.clear().unwrap();
-
-    Ok(())
-}
-
-#[allow(dead_code)]
-struct LogWidget<'a> {
-    logger: &'a TuiLogger,
-    title: &'a str,
-    show_module: bool,
-}
-
-impl<'a> LogWidget<'a> {
-    fn new(logger: &'a TuiLogger, title: &'a str) -> LogWidget<'a> {
-        LogWidget {
-            logger,
-            title,
-            show_module: false,
-        }
-    }
-
-    fn show_module(mut self, b: bool) -> LogWidget<'a> {
-        self.show_module = b;
-        self
-    }
-}
-
-impl<'a> Widget for LogWidget<'a> {
-    fn render(self, area: Rect, buf: &mut Buffer) {
-        let w = TuiLoggerWidget::default(self.logger)
-            .block(
-                Block::default()
-                    .borders(Borders::ALL)
-                    .title(self.title)
-                    .border_type(BorderType::Rounded),
-            )
-            .show_module(true)
-            .style_error(Style::default().fg(Color::Red))
-            .style_warn(Style::default().fg(Color::Yellow))
-            .style_info(Style::default().fg(Color::Green));
-        w.render(area, buf);
-    }
-}
-
-// Render a widget to show some metrics
-struct MetricsWidget {}
-
-fn _get_metric_u64(title: &str, value: u64) -> Spans {
-    Spans::from(vec![
-        Span::styled(format!("{:<20}", title), Style::default()),
-        Span::raw(": "),
-        Span::styled(
-            value.to_string(),
-            Style::default().add_modifier(Modifier::BOLD),
-        ),
-    ])
-}
-
-// This is not used since LSNs were removed from page cache stats.
-// Maybe it will be used in the future?
-fn _get_metric_str<'a>(title: &str, value: &'a str) -> Spans<'a> {
-    Spans::from(vec![
-        Span::styled(format!("{:<20}", title), Style::default()),
-        Span::raw(": "),
-        Span::styled(value, Style::default().add_modifier(Modifier::BOLD)),
-    ])
-}
-
-impl tui::widgets::Widget for MetricsWidget {
-    fn render(self, area: Rect, buf: &mut Buffer) {
-        let block = Block::default()
-            .borders(Borders::ALL)
-            .title("Page Cache Metrics")
-            .border_type(BorderType::Rounded);
-        let inner_area = block.inner(area);
-
-        block.render(area, buf);
-
-        #[allow(unused_mut)]
-        let mut lines: Vec<Spans> = Vec::new();
-
-        // FIXME
-        //let page_cache_stats = crate::page_cache::get_stats();
-
-        // This is not used since LSNs were removed from page cache stats.
-        // Maybe it will be used in the future?
-        /*
-        let lsnrange = format!(
-            "{} - {}",
-            page_cache_stats.first_valid_lsn, page_cache_stats.last_valid_lsn
-        );
-        let last_valid_recordlsn_str = page_cache_stats.last_record_lsn.to_string();
-        lines.push(get_metric_str("Valid LSN range", &lsnrange));
-        lines.push(get_metric_str("Last record LSN", &last_valid_recordlsn_str));
-        */
-        /*
-        lines.push(get_metric_u64(
-            "# of cache entries",
-            page_cache_stats.num_entries,
-        ));
-        lines.push(get_metric_u64(
-            "# of page images",
-            page_cache_stats.num_page_images,
-        ));
-        lines.push(get_metric_u64(
-            "# of WAL records",
-            page_cache_stats.num_wal_records,
-        ));
-        lines.push(get_metric_u64(
-            "# of GetPage@LSN calls",
-            page_cache_stats.num_getpage_requests,
-        ));
-        */
-        let text = Text::from(lines);
-
-        Paragraph::new(text).render(inner_area, buf);
-    }
-}

pageserver/src/tui_event.rs

@@ -1,96 +0,0 @@
-use std::io;
-use std::sync::mpsc;
-use std::sync::{
-    atomic::{AtomicBool, Ordering},
-    Arc,
-};
-use std::thread;
-use std::time::Duration;
-
-use termion::event::Key;
-use termion::input::TermRead;
-
-pub enum Event<I> {
-    Input(I),
-    Tick,
-}
-
-/// A small event handler that wrap termion input and tick events. Each event
-/// type is handled in its own thread and returned to a common `Receiver`
-#[allow(dead_code)]
-pub struct Events {
-    rx: mpsc::Receiver<Event<Key>>,
-    input_handle: thread::JoinHandle<()>,
-    ignore_exit_key: Arc<AtomicBool>,
-    tick_handle: thread::JoinHandle<()>,
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct Config {
-    pub exit_key: Key,
-    pub tick_rate: Duration,
-}
-
-impl Default for Config {
-    fn default() -> Config {
-        Config {
-            exit_key: Key::Char('q'),
-            tick_rate: Duration::from_millis(250),
-        }
-    }
-}
-
-impl Events {
-    pub fn new() -> Events {
-        Events::with_config(Config::default())
-    }
-
-    pub fn with_config(config: Config) -> Events {
-        let (tx, rx) = mpsc::channel();
-        let ignore_exit_key = Arc::new(AtomicBool::new(false));
-        let input_handle = {
-            let tx = tx.clone();
-            let ignore_exit_key = ignore_exit_key.clone();
-            thread::spawn(move || {
-                let stdin = io::stdin();
-                for evt in stdin.keys() {
-                    if let Ok(key) = evt {
-                        if let Err(err) = tx.send(Event::Input(key)) {
-                            eprintln!("{}", err);
-                            return;
-                        }
-                        if !ignore_exit_key.load(Ordering::Relaxed) && key == config.exit_key {
-                            return;
-                        }
-                    }
-                }
-            })
-        };
-        let tick_handle = {
-            thread::spawn(move || loop {
-                if tx.send(Event::Tick).is_err() {
-                    break;
-                }
-                thread::sleep(config.tick_rate);
-            })
-        };
-        Events {
-            rx,
-            input_handle,
-            ignore_exit_key,
-            tick_handle,
-        }
-    }
-
-    pub fn next(&self) -> Result<Event<Key>, mpsc::RecvError> {
-        self.rx.recv()
-    }
-
-    pub fn disable_exit_key(&mut self) {
-        self.ignore_exit_key.store(true, Ordering::Relaxed);
-    }
-
-    pub fn enable_exit_key(&mut self) {
-        self.ignore_exit_key.store(false, Ordering::Relaxed);
-    }
-}

pageserver/src/tui_logger.rs

@@ -1,199 +0,0 @@
-//
-// A TUI Widget that displays log entries
-//
-// This is heavily inspired by gin66's tui_logger crate at https://github.com/gin66/tui-logger,
-// but I wrote this based on the 'slog' module, which simplified things a lot. tui-logger also
-// implemented the slog Drain trait, but it had a model of one global buffer for the records.
-// With this implementation, each TuiLogger is a separate ring buffer and separate slog Drain.
-// Also, I didn't do any of the "hot log" stuff that gin66's implementation had, you can use an
-// AsyncDrain to buffer and handle overflow if desired.
-//
-use chrono::offset::Local;
-use chrono::DateTime;
-use slog::{Drain, Level, OwnedKVList, Record};
-use slog_async::AsyncRecord;
-use std::collections::VecDeque;
-use std::sync::Mutex;
-use std::time::SystemTime;
-use tui::buffer::Buffer;
-use tui::layout::Rect;
-use tui::style::{Modifier, Style};
-use tui::text::{Span, Spans};
-use tui::widgets::{Block, Paragraph, Widget, Wrap};
-
-// Size of the log ring buffer, in # of records
-static BUFFER_SIZE: usize = 1000;
-
-pub struct TuiLogger {
-    events: Mutex<VecDeque<(SystemTime, AsyncRecord)>>,
-}
-
-impl<'a> Default for TuiLogger {
-    fn default() -> TuiLogger {
-        TuiLogger {
-            events: Mutex::new(VecDeque::with_capacity(BUFFER_SIZE)),
-        }
-    }
-}
-
-impl Drain for TuiLogger {
-    type Ok = ();
-    type Err = slog::Error;
-
-    fn log(&self, record: &Record, values: &OwnedKVList) -> Result<Self::Ok, Self::Err> {
-        let mut events = self.events.lock().unwrap();
-
-        let now = SystemTime::now();
-        let asyncrec = AsyncRecord::from(record, values);
-        events.push_front((now, asyncrec));
-
-        if events.len() > BUFFER_SIZE {
-            events.pop_back();
-        }
-
-        Ok(())
-    }
-}
-
-// TuiLoggerWidget renders a TuiLogger ring buffer
-pub struct TuiLoggerWidget<'b> {
-    block: Option<Block<'b>>,
-    /// Base style of the widget
-    style: Style,
-    /// Level based style
-    style_error: Option<Style>,
-    style_warn: Option<Style>,
-    style_debug: Option<Style>,
-    style_trace: Option<Style>,
-    style_info: Option<Style>,
-    show_module: bool,
-    logger: &'b TuiLogger,
-}
-impl<'b> TuiLoggerWidget<'b> {
-    pub fn default(logger: &'b TuiLogger) -> TuiLoggerWidget<'b> {
-        TuiLoggerWidget {
-            block: None,
-            style: Default::default(),
-            style_error: None,
-            style_warn: None,
-            style_debug: None,
-            style_trace: None,
-            style_info: None,
-            show_module: true,
-            logger,
-        }
-    }
-}
-impl<'b> TuiLoggerWidget<'b> {
-    pub fn block(mut self, block: Block<'b>) -> TuiLoggerWidget<'b> {
-        self.block = Some(block);
-        self
-    }
-    #[allow(unused)]
-    pub fn style(mut self, style: Style) -> TuiLoggerWidget<'b> {
-        self.style = style;
-        self
-    }
-    pub fn style_error(mut self, style: Style) -> TuiLoggerWidget<'b> {
-        self.style_error = Some(style);
-        self
-    }
-    pub fn style_warn(mut self, style: Style) -> TuiLoggerWidget<'b> {
-        self.style_warn = Some(style);
-        self
-    }
-    pub fn style_info(mut self, style: Style) -> TuiLoggerWidget<'b> {
-        self.style_info = Some(style);
-        self
-    }
-    #[allow(unused)]
-    pub fn style_trace(mut self, style: Style) -> TuiLoggerWidget<'b> {
-        self.style_trace = Some(style);
-        self
-    }
-    #[allow(unused)]
-    pub fn style_debug(mut self, style: Style) -> TuiLoggerWidget<'b> {
-        self.style_debug = Some(style);
-        self
-    }
-
-    pub fn show_module(mut self, b: bool) -> TuiLoggerWidget<'b> {
-        self.show_module = b;
-        self
-    }
-}
-impl<'b> Widget for TuiLoggerWidget<'b> {
-    fn render(mut self, area: Rect, buf: &mut Buffer) {
-        buf.set_style(area, self.style);
-        let list_area = match self.block.take() {
-            Some(b) => {
-                let inner_area = b.inner(area);
-                b.render(area, buf);
-                inner_area
-            }
-            None => area,
-        };
-        if list_area.width == 0 || list_area.height == 0 {
-            return;
-        }
-
-        let la_height = list_area.height as usize;
-
-        //
-        // Iterate through the records in the buffer. The records are
-        // pushed to the front, so the newest records come first.
-        //
-        let mut lines: Vec<Spans> = Vec::new();
-
-        let style_msg = Style::default().add_modifier(Modifier::BOLD);
-        {
-            let events = self.logger.events.lock().unwrap();
-
-            for evt in events.iter() {
-                let (timestamp, rec) = evt;
-
-                rec.as_record_values(|rec, _kwlist| {
-                    let mut line: Vec<Span> = Vec::new();
-
-                    let datetime: DateTime<Local> = timestamp.clone().into();
-                    let ts = format!("{}", datetime.format("%H:%M:%S%.3f "));
-                    line.push(Span::raw(ts));
-
-                    let (lvl_style, txt, with_loc) = match rec.level() {
-                        Level::Critical => (self.style_error, "CRIT ", true),
-                        Level::Error => (self.style_error, "ERROR", true),
-                        Level::Warning => (self.style_warn, "WARN ", true),
-                        Level::Info => (self.style_info, "INFO ", false),
-                        Level::Debug => (self.style_debug, "DEBUG", true),
-                        Level::Trace => (self.style_trace, "TRACE", true),
-                    };
-                    line.push(Span::styled(txt, lvl_style.unwrap_or_default()));
-
-                    if self.show_module {
-                        line.push(Span::raw(" "));
-                        line.push(Span::raw(rec.module()));
-                    }
-                    if with_loc {
-                        let loc = format!(" {}:{}", rec.file(), rec.line());
-                        line.push(Span::raw(loc));
-                    }
-                    let msg = format!(" {}", rec.msg());
-                    line.push(Span::styled(msg, style_msg));
-
-                    lines.push(Spans::from(line));
-                });
-                if lines.len() == la_height {
-                    break;
-                }
-            }
-        }
-
-        lines.reverse();
-
-        let text = tui::text::Text::from(lines);
-
-        Paragraph::new(text)
-            .wrap(Wrap { trim: true })
-            .render(list_area, buf);
-    }
-}

pageserver/src/waldecoder.rs

@@ -1,49 +1,21 @@
+//!
+//! WAL decoder. For each WAL record, it decodes the record to figure out which data blocks
+//! the record affects, so that they can be stored in repository.
+//!
 use bytes::{Buf, BufMut, Bytes, BytesMut};
+use crc32c::*;
 use log::*;
 use postgres_ffi::pg_constants;
-use postgres_ffi::xlog_utils::XLogRecord;
+use postgres_ffi::xlog_utils::*;
+use postgres_ffi::XLogLongPageHeaderData;
+use postgres_ffi::XLogPageHeaderData;
+use postgres_ffi::XLogRecord;
+use postgres_ffi::{BlockNumber, OffsetNumber};
+use postgres_ffi::{MultiXactId, MultiXactOffset, MultiXactStatus, Oid, TransactionId};
 use std::cmp::min;
-use std::str;
 use thiserror::Error;
 use zenith_utils::lsn::Lsn;
 
-pub type Oid = u32;
-pub type TransactionId = u32;
-pub type BlockNumber = u32;
-pub type OffsetNumber = u16;
-pub type MultiXactId = TransactionId;
-pub type MultiXactOffset = u32;
-pub type MultiXactStatus = u32;
-
-// From PostgreSQL headers
-
-#[repr(C)]
-#[derive(Debug)]
-pub struct XLogPageHeaderData {
-    xlp_magic: u16,    /* magic value for correctness checks */
-    xlp_info: u16,     /* flag bits, see below */
-    xlp_tli: u32,      /* TimeLineID of first record on page */
-    xlp_pageaddr: u64, /* XLOG address of this page */
-    xlp_rem_len: u32,  /* total len of remaining data for record */
-}
-
-// FIXME: this assumes MAXIMUM_ALIGNOF 8. There are 4 padding bytes at end
-#[allow(non_upper_case_globals)]
-const SizeOfXLogShortPHD: usize = 2 + 2 + 4 + 8 + 4 + 4;
-
-#[repr(C)]
-#[derive(Debug)]
-pub struct XLogLongPageHeaderData {
-    std: XLogPageHeaderData, /* standard header fields */
-    xlp_sysid: u64,          /* system identifier from pg_control */
-    xlp_seg_size: u32,       /* just as a cross-check */
-    xlp_xlog_blcksz: u32,    /* just as a cross-check */
-}
-
-// FIXME: this assumes MAXIMUM_ALIGNOF 8.
-#[allow(non_upper_case_globals)]
-const SizeOfXLogLongPHD: usize = (2 + 2 + 4 + 8 + 4) + 4 + 8 + 4 + 4;
-
 #[allow(dead_code)]
 pub struct WalStreamDecoder {
     lsn: Lsn,
@@ -90,7 +62,7 @@ impl WalStreamDecoder {
     /// decoder so far.
     ///
     /// Returns one of the following:
-    ///     Ok((u64, Bytes)): a tuple containing the LSN of next record, and the record itself
+    ///     Ok((Lsn, Bytes)): a tuple containing the LSN of next record, and the record itself
     ///     Ok(None): there is not enough data in the input buffer. Feed more by calling the `feed_bytes` function
     ///     Err(WalDecodeError): an error occured while decoding, meaning the input was invalid.
     ///
@@ -100,11 +72,12 @@ impl WalStreamDecoder {
             if self.lsn.segment_offset(pg_constants::WAL_SEGMENT_SIZE) == 0 {
                 // parse long header
 
-                if self.inputbuf.remaining() < SizeOfXLogLongPHD {
+                if self.inputbuf.remaining() < XLOG_SIZE_OF_XLOG_LONG_PHD {
                     return Ok(None);
                 }
 
-                let hdr = self.decode_XLogLongPageHeaderData();
+                let hdr = XLogLongPageHeaderData::from_bytes(&mut self.inputbuf);
+
                 if hdr.std.xlp_pageaddr != self.lsn.0 {
                     return Err(WalDecodeError {
                         msg: "invalid xlog segment header".into(),
@@ -113,14 +86,15 @@ impl WalStreamDecoder {
                 }
                 // TODO: verify the remaining fields in the header
 
-                self.lsn += SizeOfXLogLongPHD as u64;
+                self.lsn += XLOG_SIZE_OF_XLOG_LONG_PHD as u64;
                 continue;
             } else if self.lsn.block_offset() == 0 {
-                if self.inputbuf.remaining() < SizeOfXLogShortPHD {
+                if self.inputbuf.remaining() < XLOG_SIZE_OF_XLOG_SHORT_PHD {
                     return Ok(None);
                 }
 
-                let hdr = self.decode_XLogPageHeaderData();
+                let hdr = XLogPageHeaderData::from_bytes(&mut self.inputbuf);
+
                 if hdr.xlp_pageaddr != self.lsn.0 {
                     return Err(WalDecodeError {
                         msg: "invalid xlog page header".into(),
@@ -129,7 +103,7 @@ impl WalStreamDecoder {
                 }
                 // TODO: verify the remaining fields in the header
 
-                self.lsn += SizeOfXLogShortPHD as u64;
+                self.lsn += XLOG_SIZE_OF_XLOG_SHORT_PHD as u64;
                 continue;
             } else if self.padlen > 0 {
                 if self.inputbuf.remaining() < self.padlen as usize {
@@ -150,7 +124,7 @@ impl WalStreamDecoder {
                 // read xl_tot_len FIXME: assumes little-endian
                 self.startlsn = self.lsn;
                 let xl_tot_len = self.inputbuf.get_u32_le();
-                if xl_tot_len < SizeOfXLogRecord {
+                if (xl_tot_len as usize) < XLOG_SIZE_OF_XLOG_RECORD {
                     return Err(WalDecodeError {
                         msg: format!("invalid xl_tot_len {}", xl_tot_len),
                         lsn: self.lsn,
@@ -188,6 +162,14 @@ impl WalStreamDecoder {
                     // XLOG_SWITCH records are special. If we see one, we need to skip
                     // to the next WAL segment.
                     let xlogrec = XLogRecord::from_bytes(&mut buf);
+                    let mut crc = crc32c_append(0, &recordbuf[XLOG_RECORD_CRC_OFFS + 4..]);
+                    crc = crc32c_append(crc, &recordbuf[0..XLOG_RECORD_CRC_OFFS]);
+                    if crc != xlogrec.xl_crc {
+                        return Err(WalDecodeError {
+                            msg: "WAL record crc mismatch".into(),
+                            lsn: self.lsn,
+                        });
+                    }
                     if xlogrec.is_xlog_switch_record() {
                         trace!("saw xlog switch record at {}", self.lsn);
                         self.padlen =
@@ -197,7 +179,10 @@ impl WalStreamDecoder {
                         self.padlen = self.lsn.calc_padding(8u32) as u32;
                     }
 
-                    let result = (self.lsn, recordbuf);
+                    // Always align resulting LSN on 0x8 boundary -- that is important for getPage()
+                    // and WalReceiver integration. Since this code is used both for WalReceiver and
+                    // initial WAL import let's force alignment right here.
+                    let result = (self.lsn.align(), recordbuf);
                     return Ok(Some(result));
                 }
                 continue;
@@ -209,40 +194,6 @@ impl WalStreamDecoder {
 
         // deal with xlog_switch records
     }
-
-    #[allow(non_snake_case)]
-    fn decode_XLogPageHeaderData(&mut self) -> XLogPageHeaderData {
-        let buf = &mut self.inputbuf;
-
-        // FIXME: Assume little-endian
-
-        let hdr: XLogPageHeaderData = XLogPageHeaderData {
-            xlp_magic: buf.get_u16_le(),
-            xlp_info: buf.get_u16_le(),
-            xlp_tli: buf.get_u32_le(),
-            xlp_pageaddr: buf.get_u64_le(),
-            xlp_rem_len: buf.get_u32_le(),
-        };
-        // 4 bytes of padding, on 64-bit systems
-        buf.advance(4);
-
-        // FIXME: check that hdr.xlp_rem_len matches self.contlen
-        //println!("next xlog page (xlp_rem_len: {})", hdr.xlp_rem_len);
-
-        hdr
-    }
-
-    #[allow(non_snake_case)]
-    fn decode_XLogLongPageHeaderData(&mut self) -> XLogLongPageHeaderData {
-        let hdr: XLogLongPageHeaderData = XLogLongPageHeaderData {
-            std: self.decode_XLogPageHeaderData(),
-            xlp_sysid: self.inputbuf.get_u64_le(),
-            xlp_seg_size: self.inputbuf.get_u32_le(),
-            xlp_xlog_blcksz: self.inputbuf.get_u32_le(),
-        };
-
-        hdr
-    }
 }
 
 #[allow(dead_code)]
@@ -264,7 +215,7 @@ pub struct DecodedBkpBlock {
     /* Information on full-page image, if any */
     has_image: bool,       /* has image, even for consistency checking */
     pub apply_image: bool, /* has image that should be restored */
-    pub will_init: bool,   /* record intialize page content */
+    pub will_init: bool,   /* record doesn't need previous page version to apply */
     //char	   *bkp_image;
     hole_offset: u16,
     hole_length: u16,
@@ -300,10 +251,8 @@ impl DecodedBkpBlock {
     }
 }
 
-#[allow(non_upper_case_globals)]
-const SizeOfXLogRecord: u32 = 24;
-
 pub struct DecodedWALRecord {
+    pub xl_xid: TransactionId,
     pub xl_info: u8,
     pub xl_rmid: u8,
     pub record: Bytes, // raw XLogRecord
@@ -347,9 +296,7 @@ pub struct XlSmgrTruncate {
 }
 
 impl XlSmgrTruncate {
-    pub fn decode(decoded: &DecodedWALRecord) -> XlSmgrTruncate {
-        let mut buf = decoded.record.clone();
-        buf.advance((SizeOfXLogRecord + 2) as usize);
+    pub fn decode(buf: &mut Bytes) -> XlSmgrTruncate {
         XlSmgrTruncate {
             blkno: buf.get_u32_le(),
             rnode: RelFileNode {
@@ -372,9 +319,7 @@ pub struct XlCreateDatabase {
 }
 
 impl XlCreateDatabase {
-    pub fn decode(decoded: &DecodedWALRecord) -> XlCreateDatabase {
-        let mut buf = decoded.record.clone();
-        buf.advance((SizeOfXLogRecord + 2) as usize);
+    pub fn decode(buf: &mut Bytes) -> XlCreateDatabase {
         XlCreateDatabase {
             db_id: buf.get_u32_le(),
             tablespace_id: buf.get_u32_le(),
@@ -384,6 +329,31 @@ impl XlCreateDatabase {
     }
 }
 
+#[repr(C)]
+#[derive(Debug)]
+pub struct XlDropDatabase {
+    pub db_id: Oid,
+    pub n_tablespaces: Oid, /* number of tablespace IDs */
+    pub tablespace_ids: Vec<Oid>,
+}
+
+impl XlDropDatabase {
+    pub fn decode(buf: &mut Bytes) -> XlDropDatabase {
+        let mut rec = XlDropDatabase {
+            db_id: buf.get_u32_le(),
+            n_tablespaces: buf.get_u32_le(),
+            tablespace_ids: Vec::<Oid>::new(),
+        };
+
+        for _i in 0..rec.n_tablespaces {
+            let id = buf.get_u32_le();
+            rec.tablespace_ids.push(id);
+        }
+
+        rec
+    }
+}
+
 #[repr(C)]
 #[derive(Debug)]
 pub struct XlHeapInsert {
@@ -404,6 +374,7 @@ impl XlHeapInsert {
 #[derive(Debug)]
 pub struct XlHeapMultiInsert {
     pub flags: u8,
+    pub _padding: u8,
     pub ntuples: u16,
 }
 
@@ -411,6 +382,7 @@ impl XlHeapMultiInsert {
     pub fn decode(buf: &mut Bytes) -> XlHeapMultiInsert {
         XlHeapMultiInsert {
             flags: buf.get_u8(),
+            _padding: buf.get_u8(),
             ntuples: buf.get_u16_le(),
         }
     }
@@ -421,6 +393,8 @@ impl XlHeapMultiInsert {
 pub struct XlHeapDelete {
     pub xmax: TransactionId,
     pub offnum: OffsetNumber,
+    pub _padding: u16,
+    pub t_cid: u32,
     pub infobits_set: u8,
     pub flags: u8,
 }
@@ -430,6 +404,8 @@ impl XlHeapDelete {
         XlHeapDelete {
             xmax: buf.get_u32_le(),
             offnum: buf.get_u16_le(),
+            _padding: buf.get_u16_le(),
+            t_cid: buf.get_u32_le(),
             infobits_set: buf.get_u8(),
             flags: buf.get_u8(),
         }
@@ -443,6 +419,7 @@ pub struct XlHeapUpdate {
     pub old_offnum: OffsetNumber,
     pub old_infobits_set: u8,
     pub flags: u8,
+    pub t_cid: u32,
     pub new_xmax: TransactionId,
     pub new_offnum: OffsetNumber,
 }
@@ -454,12 +431,128 @@ impl XlHeapUpdate {
             old_offnum: buf.get_u16_le(),
             old_infobits_set: buf.get_u8(),
             flags: buf.get_u8(),
+            t_cid: buf.get_u32(),
             new_xmax: buf.get_u32_le(),
             new_offnum: buf.get_u16_le(),
         }
     }
 }
 
+///
+/// Note: Parsing some fields is missing, because they're not needed.
+///
+/// This is similar to the xl_xact_parsed_commit and
+/// xl_xact_parsed_abort structs in PostgreSQL, but we use the same
+/// struct for commits and aborts.
+///
+#[derive(Debug)]
+pub struct XlXactParsedRecord {
+    pub xid: TransactionId,
+    pub info: u8,
+    pub xact_time: TimestampTz,
+    pub xinfo: u32,
+
+    pub db_id: Oid, /* MyDatabaseId */
+    pub ts_id: Oid, /* MyDatabaseTableSpace */
+
+    pub subxacts: Vec<TransactionId>,
+
+    pub xnodes: Vec<RelFileNode>,
+}
+
+impl XlXactParsedRecord {
+    /// Decode a XLOG_XACT_COMMIT/ABORT/COMMIT_PREPARED/ABORT_PREPARED
+    /// record. This should agree with the ParseCommitRecord and ParseAbortRecord
+    /// functions in PostgreSQL (in src/backend/access/rmgr/xactdesc.c)
+    pub fn decode(buf: &mut Bytes, mut xid: TransactionId, xl_info: u8) -> XlXactParsedRecord {
+        let info = xl_info & pg_constants::XLOG_XACT_OPMASK;
+        // The record starts with time of commit/abort
+        let xact_time = buf.get_i64_le();
+        let xinfo;
+        if xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
+            xinfo = buf.get_u32_le();
+        } else {
+            xinfo = 0;
+        }
+        let db_id;
+        let ts_id;
+        if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
+            db_id = buf.get_u32_le();
+            ts_id = buf.get_u32_le();
+        } else {
+            db_id = 0;
+            ts_id = 0;
+        }
+        let mut subxacts = Vec::<TransactionId>::new();
+        if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
+            let nsubxacts = buf.get_i32_le();
+            for _i in 0..nsubxacts {
+                let subxact = buf.get_u32_le();
+                subxacts.push(subxact);
+            }
+        }
+        let mut xnodes = Vec::<RelFileNode>::new();
+        if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
+            let nrels = buf.get_i32_le();
+            for _i in 0..nrels {
+                let spcnode = buf.get_u32_le();
+                let dbnode = buf.get_u32_le();
+                let relnode = buf.get_u32_le();
+                trace!(
+                    "XLOG_XACT_COMMIT relfilenode {}/{}/{}",
+                    spcnode,
+                    dbnode,
+                    relnode
+                );
+                xnodes.push(RelFileNode {
+                    spcnode,
+                    dbnode,
+                    relnode,
+                });
+            }
+        }
+        if xinfo & pg_constants::XACT_XINFO_HAS_INVALS != 0 {
+            let nmsgs = buf.get_i32_le();
+            for _i in 0..nmsgs {
+                let sizeof_shared_invalidation_message = 0;
+                buf.advance(sizeof_shared_invalidation_message);
+            }
+        }
+        if xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE != 0 {
+            xid = buf.get_u32_le();
+            trace!("XLOG_XACT_COMMIT-XACT_XINFO_HAS_TWOPHASE");
+        }
+        XlXactParsedRecord {
+            xid,
+            info,
+            xact_time,
+            xinfo,
+            db_id,
+            ts_id,
+            subxacts,
+            xnodes,
+        }
+    }
+}
+
+#[repr(C)]
+#[derive(Debug)]
+pub struct XlClogTruncate {
+    pub pageno: u32,
+    pub oldest_xid: TransactionId,
+    pub oldest_xid_db: Oid,
+}
+
+impl XlClogTruncate {
+    pub fn decode(buf: &mut Bytes) -> XlClogTruncate {
+        XlClogTruncate {
+            pageno: buf.get_u32_le(),
+            oldest_xid: buf.get_u32_le(),
+            oldest_xid_db: buf.get_u32_le(),
+        }
+    }
+}
+
 #[repr(C)]
 #[derive(Debug)]
 pub struct MultiXactMember {
@@ -506,14 +599,14 @@ impl XlMultiXactCreate {
 #[repr(C)]
 #[derive(Debug)]
 pub struct XlMultiXactTruncate {
-    oldest_multi_db: Oid,
+    pub oldest_multi_db: Oid,
     /* to-be-truncated range of multixact offsets */
-    start_trunc_off: MultiXactId, /* just for completeness' sake */
-    end_trunc_off: MultiXactId,
+    pub start_trunc_off: MultiXactId, /* just for completeness' sake */
+    pub end_trunc_off: MultiXactId,
 
     /* to-be-truncated range of multixact members */
-    start_trunc_memb: MultiXactOffset,
-    end_trunc_memb: MultiXactOffset,
+    pub start_trunc_memb: MultiXactOffset,
+    pub end_trunc_memb: MultiXactOffset,
 }
 
 impl XlMultiXactTruncate {
@@ -528,8 +621,7 @@ impl XlMultiXactTruncate {
     }
 }
 
-//
-// Routines to decode a WAL record and figure out which blocks are modified
+/// Main routine to decode a WAL record and figure out which blocks are modified
 //
 // See xlogrecord.h for details
 // The overall layout of an XLOG record is:
@@ -566,9 +658,9 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
         xlogrec.xl_info
     );
 
-    let remaining = xlogrec.xl_tot_len - SizeOfXLogRecord;
+    let remaining: usize = xlogrec.xl_tot_len as usize - XLOG_SIZE_OF_XLOG_RECORD;
 
-    if buf.remaining() != remaining as usize {
+    if buf.remaining() != remaining {
         //TODO error
     }
 
@@ -776,129 +868,14 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
         assert_eq!(buf.remaining(), main_data_len as usize);
     }
 
-    //5. Handle special XACT records
-    if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
-        let info = xlogrec.xl_info & pg_constants::XLOG_XACT_OPMASK;
-        if info == pg_constants::XLOG_XACT_COMMIT {
-            //parse commit record to extract subtrans entries
-            // xl_xact_commit starts with time of commit
-            let _xact_time = buf.get_i64_le();
-
-            let mut xinfo = 0;
-            if xlogrec.xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
-                xinfo = buf.get_u32_le();
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
-                let _dbid = buf.get_u32_le();
-                let _tsid = buf.get_u32_le();
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
-                let nsubxacts = buf.get_i32_le();
-                for _i in 0..nsubxacts {
-                    let _subxact = buf.get_u32_le();
-                }
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
-                let nrels = buf.get_i32_le();
-                for _i in 0..nrels {
-                    let spcnode = buf.get_u32_le();
-                    let dbnode = buf.get_u32_le();
-                    let relnode = buf.get_u32_le();
-                    //TODO handle this too?
-                    trace!(
-                        "XLOG_XACT_COMMIT relfilenode {}/{}/{}",
-                        spcnode,
-                        dbnode,
-                        relnode
-                    );
-                }
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_INVALS != 0 {
-                let nmsgs = buf.get_i32_le();
-                for _i in 0..nmsgs {
-                    let sizeof_shared_invalidation_message = 0;
-                    buf.advance(sizeof_shared_invalidation_message);
-                }
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE != 0 {
-                let _xid = buf.get_u32_le();
-                trace!("XLOG_XACT_COMMIT-XACT_XINFO_HAS_TWOPHASE");
-                //TODO handle this to be able to restore pg_twophase on node start
-            }
-        } else if info == pg_constants::XLOG_XACT_ABORT {
-            //parse abort record to extract subtrans entries
-            // xl_xact_abort starts with time of commit
-            let _xact_time = buf.get_i64_le();
-
-            let mut xinfo = 0;
-            if xlogrec.xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
-                xinfo = buf.get_u32_le();
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
-                let _dbid = buf.get_u32_le();
-                let _tsid = buf.get_u32_le();
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
-                let nsubxacts = buf.get_i32_le();
-                for _i in 0..nsubxacts {
-                    let _subxact = buf.get_u32_le();
-                }
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
-                let nrels = buf.get_i32_le();
-                for _i in 0..nrels {
-                    let spcnode = buf.get_u32_le();
-                    let dbnode = buf.get_u32_le();
-                    let relnode = buf.get_u32_le();
-                    //TODO handle this too?
-                    trace!(
-                        "XLOG_XACT_ABORT relfilenode {}/{}/{}",
-                        spcnode,
-                        dbnode,
-                        relnode
-                    );
-                }
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE != 0 {
-                let _xid = buf.get_u32_le();
-                trace!("XLOG_XACT_ABORT-XACT_XINFO_HAS_TWOPHASE");
-            }
-        }
-    } else if xlogrec.xl_rmid == pg_constants::RM_DBASE_ID {
-        let info = xlogrec.xl_info & !pg_constants::XLR_INFO_MASK;
-        if info == pg_constants::XLOG_DBASE_CREATE {
-            //buf points to main_data
-            let db_id = buf.get_u32_le();
-            let tablespace_id = buf.get_u32_le();
-            let src_db_id = buf.get_u32_le();
-            let src_tablespace_id = buf.get_u32_le();
-            trace!(
-                "XLOG_DBASE_CREATE tablespace_id/db_id {}/{} src_db_id {}/{}",
-                tablespace_id,
-                db_id,
-                src_tablespace_id,
-                src_db_id
-            );
-            // in postgres it is implemented as copydir
-            // we need to copy all pages in page_cache
-        } else {
-            trace!("XLOG_DBASE_DROP is not handled yet");
-        }
-    } else if xlogrec.xl_rmid == pg_constants::RM_TBLSPC_ID {
-        let info = xlogrec.xl_info & !pg_constants::XLR_INFO_MASK;
-        if info == pg_constants::XLOG_TBLSPC_CREATE {
-            //buf points to main_data
-            let ts_id = buf.get_u32_le();
-            let ts_path = str::from_utf8(&buf).unwrap();
-            trace!("XLOG_TBLSPC_CREATE ts_id {} ts_path {}", ts_id, ts_path);
-        } else {
-            trace!("XLOG_TBLSPC_DROP is not handled yet");
-        }
-    } else if xlogrec.xl_rmid == pg_constants::RM_HEAP_ID {
-        let info = xlogrec.xl_info & pg_constants::XLOG_XACT_OPMASK;
+    // 5. Handle a few special record types that modify blocks without registering
+    // them with the standard mechanism.
+    if xlogrec.xl_rmid == pg_constants::RM_HEAP_ID {
+        let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
         let blkno = blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32;
         if info == pg_constants::XLOG_HEAP_INSERT {
             let xlrec = XlHeapInsert::decode(&mut buf);
+            assert_eq!(0, buf.remaining());
             if (xlrec.flags
                 & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
                     | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
@@ -914,6 +891,7 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
             }
         } else if info == pg_constants::XLOG_HEAP_DELETE {
             let xlrec = XlHeapDelete::decode(&mut buf);
+            assert_eq!(0, buf.remaining());
             if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 {
                 let mut blk = DecodedBkpBlock::new();
                 blk.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
@@ -927,6 +905,9 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
             || info == pg_constants::XLOG_HEAP_HOT_UPDATE
         {
             let xlrec = XlHeapUpdate::decode(&mut buf);
+            // the size of tuple data is inferred from the size of the record.
+            // we can't validate the remaining number of bytes without parsing
+            // the tuple data.
             if (xlrec.flags & pg_constants::XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0 {
                 let mut blk = DecodedBkpBlock::new();
                 blk.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
@@ -949,9 +930,18 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
             }
         }
     } else if xlogrec.xl_rmid == pg_constants::RM_HEAP2_ID {
-        let info = xlogrec.xl_info & pg_constants::XLOG_XACT_OPMASK;
+        let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
         if info == pg_constants::XLOG_HEAP2_MULTI_INSERT {
             let xlrec = XlHeapMultiInsert::decode(&mut buf);
+
+            let offset_array_len = if xlogrec.xl_info & pg_constants::XLOG_HEAP_INIT_PAGE > 0 {
+                // the offsets array is omitted if XLOG_HEAP_INIT_PAGE is set
+                0
+            } else {
+                std::mem::size_of::<u16>() * xlrec.ntuples as usize
+            };
+            assert_eq!(offset_array_len, buf.remaining());
+
             if (xlrec.flags
                 & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
                     | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
@@ -970,6 +960,7 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
     }
 
     DecodedWALRecord {
+        xl_xid: xlogrec.xl_xid,
         xl_info: xlogrec.xl_info,
         xl_rmid: xlogrec.xl_rmid,
         record,
@@ -977,3 +968,71 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
         main_data_offset,
     }
 }
+
+///
+/// Build a human-readable string to describe a WAL record
+///
+/// For debugging purposes
+pub fn describe_wal_record(record: &Bytes) -> String {
+    // TODO: It would be nice to use the PostgreSQL rmgrdesc infrastructure for this.
+    // Maybe use the postgres wal redo process, the same used for replaying WAL records?
+    // Or could we compile the rmgrdesc routines into the dump_layer_file() binary directly,
+    // without worrying about security?
+    //
+    // But for now, we have a hand-written code for a few common WAL record types here.
+
+    let mut buf = record.clone();
+
+    // 1. Parse XLogRecord struct
+
+    // FIXME: assume little-endian here
+    let xlogrec = XLogRecord::from_bytes(&mut buf);
+
+    let unknown_str: String;
+
+    let result: &str = match xlogrec.xl_rmid {
+        pg_constants::RM_HEAP2_ID => {
+            let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
+            match info {
+                pg_constants::XLOG_HEAP2_MULTI_INSERT => "HEAP2 MULTI_INSERT",
+                pg_constants::XLOG_HEAP2_VISIBLE => "HEAP2 VISIBLE",
+                _ => {
+                    unknown_str = format!("HEAP2 UNKNOWN_0x{:02x}", info);
+                    &unknown_str
+                }
+            }
+        }
+        pg_constants::RM_HEAP_ID => {
+            let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
+            match info {
+                pg_constants::XLOG_HEAP_INSERT => "HEAP INSERT",
+                pg_constants::XLOG_HEAP_DELETE => "HEAP DELETE",
+                pg_constants::XLOG_HEAP_UPDATE => "HEAP UPDATE",
+                pg_constants::XLOG_HEAP_HOT_UPDATE => "HEAP HOT_UPDATE",
+                _ => {
+                    unknown_str = format!("HEAP2 UNKNOWN_0x{:02x}", info);
+                    &unknown_str
+                }
+            }
+        }
+        pg_constants::RM_XLOG_ID => {
+            let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
+            match info {
+                pg_constants::XLOG_FPI => "XLOG FPI",
+                pg_constants::XLOG_FPI_FOR_HINT => "XLOG FPI_FOR_HINT",
+                _ => {
+                    unknown_str = format!("XLOG UNKNOWN_0x{:02x}", info);
+                    &unknown_str
+                }
+            }
+        }
+        rmid => {
+            let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
+
+            unknown_str = format!("UNKNOWN_RM_{} INFO_0x{:02x}", rmid, info);
+            &unknown_str
+        }
+    };
+
+    String::from(result)
+}

pageserver/src/walreceiver.rs

@@ -1,24 +1,23 @@
 //!
-//! WAL receiver
-//!
-//! The WAL receiver connects to the WAL safekeeper service, and streams WAL.
-//! For each WAL record, it decodes the record to figure out which data blocks
-//! the record affects, and adds the records to the page cache.
+//! WAL receiver connects to the WAL safekeeper service, streams WAL,
+//! decodes records and saves them in the repository for the correct
+//! timeline.
 //!
+//! We keep one WAL receiver active per timeline.
 
-use crate::page_cache;
+use crate::relish::*;
 use crate::restore_local_repo;
+use crate::tenant_mgr;
 use crate::waldecoder::*;
 use crate::PageServerConf;
-use crate::ZTimelineId;
 use anyhow::{Error, Result};
 use lazy_static::lazy_static;
 use log::*;
 use postgres::fallible_iterator::FallibleIterator;
 use postgres::replication::ReplicationIter;
 use postgres::{Client, NoTls, SimpleQueryMessage, SimpleQueryRow};
-use postgres_ffi::pg_constants;
 use postgres_ffi::xlog_utils::*;
+use postgres_ffi::*;
 use postgres_protocol::message::backend::ReplicationMessage;
 use postgres_types::PgLsn;
 use std::cmp::{max, min};
@@ -26,13 +25,14 @@ use std::collections::HashMap;
 use std::fs;
 use std::fs::{File, OpenOptions};
 use std::io::{Seek, SeekFrom, Write};
-use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Mutex;
 use std::thread;
 use std::thread::sleep;
 use std::time::{Duration, SystemTime};
 use zenith_utils::lsn::Lsn;
+use zenith_utils::zid::ZTenantId;
+use zenith_utils::zid::ZTimelineId;
 
 //
 // We keep one WAL Receiver active per timeline.
@@ -51,6 +51,7 @@ pub fn launch_wal_receiver(
     conf: &'static PageServerConf,
     timelineid: ZTimelineId,
     wal_producer_connstr: &str,
+    tenantid: ZTenantId,
 ) {
     let mut receivers = WAL_RECEIVERS.lock().unwrap();
 
@@ -68,7 +69,7 @@ pub fn launch_wal_receiver(
             let _walreceiver_thread = thread::Builder::new()
                 .name("WAL receiver thread".into())
                 .spawn(move || {
-                    thread_main(conf, timelineid);
+                    thread_main(conf, timelineid, &tenantid);
                 })
                 .unwrap();
         }
@@ -89,7 +90,7 @@ fn get_wal_producer_connstr(timelineid: ZTimelineId) -> String {
 //
 // This is the entry point for the WAL receiver thread.
 //
-fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId) {
+fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId, tenantid: &ZTenantId) {
     info!(
         "WAL receiver thread started for timeline : '{}'",
         timelineid
@@ -103,7 +104,7 @@ fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId) {
         // Look up the current WAL producer address
         let wal_producer_connstr = get_wal_producer_connstr(timelineid);
 
-        let res = walreceiver_main(conf, timelineid, &wal_producer_connstr);
+        let res = walreceiver_main(conf, timelineid, &wal_producer_connstr, tenantid);
 
         if let Err(e) = res {
             info!(
@@ -116,31 +117,43 @@ fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId) {
 }
 
 fn walreceiver_main(
-    _conf: &PageServerConf,
+    conf: &PageServerConf,
     timelineid: ZTimelineId,
     wal_producer_connstr: &str,
+    tenantid: &ZTenantId,
 ) -> Result<(), Error> {
     // Connect to the database in replication mode.
     info!("connecting to {:?}", wal_producer_connstr);
-    let connect_cfg = format!("{} replication=true", wal_producer_connstr);
+    let connect_cfg = format!(
+        "{} application_name=pageserver replication=true",
+        wal_producer_connstr
+    );
 
     let mut rclient = Client::connect(&connect_cfg, NoTls)?;
     info!("connected!");
 
+    // Immediately increment the gauge, then create a job to decrement it on thread exit.
+    // One of the pros of `defer!` is that this will *most probably*
+    // get called, even in presence of panics.
+    let gauge = crate::LIVE_CONNECTIONS_COUNT.with_label_values(&["wal_receiver"]);
+    gauge.inc();
+    scopeguard::defer! {
+        gauge.dec();
+    }
+
     let identify = identify_system(&mut rclient)?;
     info!("{:?}", identify);
     let end_of_wal = Lsn::from(u64::from(identify.xlogpos));
     let mut caught_up = false;
 
-    let repository = page_cache::get_repository();
+    let repository = tenant_mgr::get_repository_for_tenant(tenantid)?;
     let timeline = repository.get_timeline(timelineid).unwrap();
 
     //
     // Start streaming the WAL, from where we left off previously.
     //
     // If we had previously received WAL up to some point in the middle of a WAL record, we
-    // better start from the end of last full WAL record, not in the middle of one. Hence,
-    // use 'last_record_lsn' rather than 'last_valid_lsn' here.
+    // better start from the end of last full WAL record, not in the middle of one.
     let mut last_rec_lsn = timeline.get_last_record_lsn();
     let mut startpoint = last_rec_lsn;
 
@@ -149,10 +162,6 @@ fn walreceiver_main(
     }
 
     // There might be some padding after the last full record, skip it.
-    //
-    // FIXME: It probably would be better to always start streaming from the beginning
-    // of the page, or the segment, so that we could check the page/segment headers
-    // too. Just for the sake of paranoia.
     startpoint += startpoint.calc_padding(8u32);
 
     debug!(
@@ -167,8 +176,12 @@ fn walreceiver_main(
 
     let mut waldecoder = WalStreamDecoder::new(startpoint);
 
+    let checkpoint_bytes = timeline.get_page_at_lsn_nowait(RelishTag::Checkpoint, 0, startpoint)?;
+    let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
+    trace!("CheckPoint.nextXid = {}", checkpoint.nextXid.value);
+
     while let Some(replication_message) = physical_stream.next()? {
-        match replication_message {
+        let status_update = match replication_message {
             ReplicationMessage::XLogData(xlog_data) => {
                 // Pass the WAL data to the decoder, and see if we can decode
                 // more records as a result.
@@ -177,33 +190,58 @@ fn walreceiver_main(
                 let endlsn = startlsn + data.len() as u64;
                 let prev_last_rec_lsn = last_rec_lsn;
 
-                write_wal_file(startlsn, timelineid, pg_constants::WAL_SEGMENT_SIZE, data)?;
+                write_wal_file(
+                    conf,
+                    startlsn,
+                    &timelineid,
+                    pg_constants::WAL_SEGMENT_SIZE,
+                    data,
+                    tenantid,
+                )?;
 
                 trace!("received XLogData between {} and {}", startlsn, endlsn);
 
                 waldecoder.feed_bytes(data);
 
                 while let Some((lsn, recdata)) = waldecoder.poll_decode()? {
+                    // Save old checkpoint value to compare with it after decoding WAL record
+                    let old_checkpoint_bytes = checkpoint.encode();
                     let decoded = decode_wal_record(recdata.clone());
-                    restore_local_repo::save_decoded_record(&*timeline, &decoded, recdata, lsn)?;
-                    last_rec_lsn = lsn;
-                }
 
-                // Update the last_valid LSN value in the page cache one more time. We updated
-                // it in the loop above, between each WAL record, but we might have received
-                // a partial record after the last completed record. Our page cache's value
-                // better reflect that, because GetPage@LSN requests might also point in the
-                // middle of a record, if the request LSN was taken from the server's current
-                // flush ptr.
-                timeline.advance_last_valid_lsn(endlsn);
+                    // It is important to deal with the aligned records as lsn in getPage@LSN is
+                    // aligned and can be several bytes bigger. Without this alignment we are
+                    // at risk of hittind a deadlock.
+                    assert!(lsn.is_aligned());
+
+                    restore_local_repo::save_decoded_record(
+                        &mut checkpoint,
+                        &*timeline,
+                        &decoded,
+                        recdata,
+                        lsn,
+                    )?;
+                    last_rec_lsn = lsn;
+
+                    let new_checkpoint_bytes = checkpoint.encode();
+                    // Check if checkpoint data was updated by save_decoded_record
+                    if new_checkpoint_bytes != old_checkpoint_bytes {
+                        timeline.put_page_image(
+                            RelishTag::Checkpoint,
+                            0,
+                            lsn,
+                            new_checkpoint_bytes,
+                        )?;
+                    }
+                }
 
                 // Somewhat arbitrarily, if we have at least 10 complete wal segments (16 MB each),
                 // "checkpoint" the repository to flush all the changes from WAL we've processed
                 // so far to disk. After this, we don't need the original WAL anymore, and it
-                // can be removed.
+                // can be removed. This is probably too aggressive for production, but it's useful
+                // to expose bugs now.
                 //
                 // TODO: We don't actually dare to remove the WAL. It's useful for debugging,
-                // and we might it for logical decoiding other things in the future. Although
+                // and we might it for logical decoding other things in the future. Although
                 // we should also be able to fetch it back from the WAL safekeepers or S3 if
                 // needed.
                 if prev_last_rec_lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE)
@@ -211,14 +249,14 @@ fn walreceiver_main(
                 {
                     info!("switched segment {} to {}", prev_last_rec_lsn, last_rec_lsn);
                     let (oldest_segno, newest_segno) = find_wal_file_range(
-                        timelineid,
+                        conf,
+                        &timelineid,
                         pg_constants::WAL_SEGMENT_SIZE,
                         last_rec_lsn,
+                        tenantid,
                     )?;
 
                     if newest_segno - oldest_segno >= 10 {
-                        timeline.checkpoint()?;
-
                         // TODO: This is where we could remove WAL older than last_rec_lsn.
                         //remove_wal_files(timelineid, pg_constants::WAL_SEGMENT_SIZE, last_rec_lsn)?;
                     }
@@ -228,12 +266,14 @@ fn walreceiver_main(
                     info!("caught up at LSN {}", endlsn);
                     caught_up = true;
                 }
+
+                Some(endlsn)
             }
 
             ReplicationMessage::PrimaryKeepAlive(keepalive) => {
                 let wal_end = keepalive.wal_end();
                 let timestamp = keepalive.timestamp();
-                let reply_requested: bool = keepalive.reply() != 0;
+                let reply_requested = keepalive.reply() != 0;
 
                 trace!(
                     "received PrimaryKeepAlive(wal_end: {}, timestamp: {:?} reply: {})",
@@ -241,36 +281,45 @@ fn walreceiver_main(
                     timestamp,
                     reply_requested,
                 );
-                if reply_requested {
-                    // TODO: More thought should go into what values are sent here.
-                    let last_lsn = PgLsn::from(u64::from(timeline.get_last_valid_lsn()));
-                    let write_lsn = last_lsn;
-                    let flush_lsn = last_lsn;
-                    let apply_lsn = PgLsn::from(0);
-                    let ts = SystemTime::now();
-                    const NO_REPLY: u8 = 0u8;
 
-                    physical_stream
-                        .standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY)?;
+                if reply_requested {
+                    Some(timeline.get_last_record_lsn())
+                } else {
+                    None
                 }
             }
-            _ => (),
+
+            _ => None,
+        };
+
+        if let Some(last_lsn) = status_update {
+            // TODO: More thought should go into what values are sent here.
+            let last_lsn = PgLsn::from(u64::from(last_lsn));
+            let write_lsn = last_lsn;
+            let flush_lsn = last_lsn;
+            let apply_lsn = PgLsn::from(0);
+            let ts = SystemTime::now();
+            const NO_REPLY: u8 = 0;
+
+            physical_stream.standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY)?;
         }
     }
     Ok(())
 }
 
 fn find_wal_file_range(
-    timeline: ZTimelineId,
+    conf: &PageServerConf,
+    timeline: &ZTimelineId,
     wal_seg_size: usize,
     written_upto: Lsn,
+    tenant: &ZTenantId,
 ) -> Result<(u64, u64)> {
     let written_upto_segno = written_upto.segment_number(wal_seg_size);
 
     let mut oldest_segno = written_upto_segno;
     let mut newest_segno = written_upto_segno;
     // Scan the wal directory, and count how many WAL filed we could remove
-    let wal_dir = PathBuf::from(format!("timelines/{}/wal", timeline));
+    let wal_dir = conf.wal_dir_path(timeline, tenant);
     for entry in fs::read_dir(wal_dir)? {
         let entry = entry?;
         let path = entry.path();
@@ -347,10 +396,12 @@ pub fn identify_system(client: &mut Client) -> Result<IdentifySystem, Error> {
 }
 
 fn write_wal_file(
+    conf: &PageServerConf,
     startpos: Lsn,
-    timeline: ZTimelineId,
+    timelineid: &ZTimelineId,
     wal_seg_size: usize,
     buf: &[u8],
+    tenantid: &ZTenantId,
 ) -> anyhow::Result<()> {
     let mut bytes_left: usize = buf.len();
     let mut bytes_written: usize = 0;
@@ -358,7 +409,7 @@ fn write_wal_file(
     let mut start_pos = startpos;
     const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ];
 
-    let wal_dir = PathBuf::from(format!("timelines/{}/wal", timeline));
+    let wal_dir = conf.wal_dir_path(timelineid, tenantid);
 
     /* Extract WAL location for this block */
     let mut xlogoff = start_pos.segment_offset(wal_seg_size);

pageserver/src/walredo.rs

@@ -1,5 +1,7 @@
 //!
-//! WAL redo
+//! WAL redo. This service runs PostgreSQL in a special wal_redo mode
+//! to apply given WAL records over an old page image and return new
+//! page image.
 //!
 //! We rely on Postgres to perform WAL redo for us. We launch a
 //! postgres process in special "wal redo" mode that's similar to
@@ -11,19 +13,23 @@
 //! See src/backend/tcop/zenith_wal_redo.c for the other side of
 //! this communication.
 //!
-//! TODO: Even though the postgres code runs in a separate process,
-//! it's not a secure sandbox.
+//! The Postgres process is assumed to be secure against malicious WAL
+//! records. It achieves it by dropping privileges before replaying
+//! any WAL records, so that even if an attacker hijacks the Postgres
+//! process, he cannot escape out of it.
 //!
-use bytes::{BufMut, Bytes, BytesMut};
+use byteorder::{ByteOrder, LittleEndian};
+use bytes::{Buf, BufMut, Bytes, BytesMut};
+use lazy_static::lazy_static;
 use log::*;
+use serde::{Deserialize, Serialize};
 use std::cell::RefCell;
 use std::fs;
 use std::fs::OpenOptions;
 use std::io::prelude::*;
 use std::io::Error;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::process::Stdio;
-use std::sync::mpsc;
 use std::sync::Mutex;
 use std::time::Duration;
 use std::time::Instant;
@@ -31,12 +37,34 @@ use tokio::io::AsyncBufReadExt;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::process::{ChildStdin, ChildStdout, Command};
 use tokio::time::timeout;
+use zenith_metrics::{register_histogram, register_int_counter, Histogram, IntCounter};
 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;
+use zenith_utils::zid::ZTenantId;
 
-use crate::repository::BufferTag;
+use crate::relish::*;
 use crate::repository::WALRecord;
+use crate::waldecoder::XlMultiXactCreate;
+use crate::waldecoder::XlXactParsedRecord;
 use crate::PageServerConf;
+use postgres_ffi::nonrelfile_utils::mx_offset_to_flags_bitshift;
+use postgres_ffi::nonrelfile_utils::mx_offset_to_flags_offset;
+use postgres_ffi::nonrelfile_utils::mx_offset_to_member_offset;
+use postgres_ffi::nonrelfile_utils::transaction_id_set_status;
+use postgres_ffi::pg_constants;
+use postgres_ffi::XLogRecord;
+
+///
+/// `RelTag` + block number (`blknum`) gives us a unique id of the page in the cluster.
+///
+/// In Postgres `BufferTag` structure is used for exactly the same purpose.
+/// [See more related comments here](https://github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/buf_internals.h#L91).
+///
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Serialize, Deserialize)]
+pub struct BufferTag {
+    pub rel: RelTag,
+    pub blknum: u32,
+}
 
 ///
 /// WAL Redo Manager is responsible for replaying WAL records.
@@ -51,7 +79,8 @@ pub trait WalRedoManager: Send + Sync {
     /// the reords.
     fn request_redo(
         &self,
-        tag: BufferTag,
+        rel: RelishTag,
+        blknum: u32,
         lsn: Lsn,
         base_img: Option<Bytes>,
         records: Vec<WALRecord>,
@@ -67,7 +96,8 @@ pub struct DummyRedoManager {}
 impl crate::walredo::WalRedoManager for DummyRedoManager {
     fn request_redo(
         &self,
-        _tag: BufferTag,
+        _rel: RelishTag,
+        _blknum: u32,
         _lsn: Lsn,
         _base_img: Option<Bytes>,
         _records: Vec<WALRecord>,
@@ -78,31 +108,50 @@ impl crate::walredo::WalRedoManager for DummyRedoManager {
 
 static TIMEOUT: Duration = Duration::from_secs(20);
 
-///
-/// The implementation consists of two parts: PostgresRedoManager, and
-/// PostgresRedoManagerInternal. PostgresRedoManager is the public struct
-/// that can be used to send redo requests to the manager.
-/// PostgresRedoManagerInternal is used by the manager thread itself.
-///
-pub struct PostgresRedoManager {
-    request_tx: Mutex<mpsc::Sender<WalRedoRequest>>,
+// Metrics collected on WAL redo operations
+//
+// We collect the time spent in actual WAL redo ('redo'), and time waiting
+// for access to the postgres process ('wait') since there is only one for
+// each tenant.
+lazy_static! {
+    static ref WAL_REDO_TIME: Histogram =
+        register_histogram!("pageserver_wal_redo_time", "Time spent on WAL redo")
+            .expect("failed to define a metric");
+    static ref WAL_REDO_WAIT_TIME: Histogram = register_histogram!(
+        "pageserver_wal_redo_wait_time",
+        "Time spent waiting for access to the WAL redo process"
+    )
+    .expect("failed to define a metric");
+    static ref WAL_REDO_RECORD_COUNTER: IntCounter = register_int_counter!(
+        "pageserver_wal_records_replayed",
+        "Number of WAL records replayed"
+    )
+    .unwrap();
 }
 
-struct PostgresRedoManagerInternal {
+///
+/// This is the real implementation that uses a Postgres process to
+/// perform WAL replay. Only one thread can use the processs at a time,
+/// that is controlled by the Mutex. In the future, we might want to
+/// launch a pool of processes to allow concurrent replay of multiple
+/// records.
+///
+pub struct PostgresRedoManager {
+    tenantid: ZTenantId,
     conf: &'static PageServerConf,
 
-    request_rx: mpsc::Receiver<WalRedoRequest>,
+    runtime: tokio::runtime::Runtime,
+    process: Mutex<Option<PostgresRedoProcess>>,
 }
 
 #[derive(Debug)]
 struct WalRedoRequest {
-    tag: BufferTag,
+    rel: RelishTag,
+    blknum: u32,
     lsn: Lsn,
 
     base_img: Option<Bytes>,
     records: Vec<WALRecord>,
-
-    response_channel: mpsc::Sender<Result<Bytes, WalRedoError>>,
 }
 
 /// An error happened in WAL redo
@@ -118,37 +167,6 @@ pub enum WalRedoError {
 ///
 /// Public interface of WAL redo manager
 ///
-impl PostgresRedoManager {
-    ///
-    /// Create a new PostgresRedoManager.
-    ///
-    /// This launches a new thread to handle the requests.
-    pub fn new(conf: &'static PageServerConf) -> PostgresRedoManager {
-        let (tx, rx) = mpsc::channel();
-
-        //
-        // Launch the WAL redo thread
-        //
-        // Get mutable references to the values that we need to pass to the
-        // thread.
-        let request_rx = rx;
-
-        // Currently, the join handle is not saved anywhere and we
-        // won't try restart the thread if it dies.
-        let _walredo_thread = std::thread::Builder::new()
-            .name("WAL redo thread".into())
-            .spawn(move || {
-                let mut internal = PostgresRedoManagerInternal { conf, request_rx };
-                internal.wal_redo_main();
-            })
-            .unwrap();
-
-        PostgresRedoManager {
-            request_tx: Mutex::new(tx),
-        }
-    }
-}
-
 impl WalRedoManager for PostgresRedoManager {
     ///
     /// Request the WAL redo manager to apply some WAL records
@@ -158,43 +176,55 @@ impl WalRedoManager for PostgresRedoManager {
     ///
     fn request_redo(
         &self,
-        tag: BufferTag,
+        rel: RelishTag,
+        blknum: u32,
         lsn: Lsn,
         base_img: Option<Bytes>,
         records: Vec<WALRecord>,
     ) -> Result<Bytes, WalRedoError> {
-        // Create a channel where to receive the response
-        let (tx, rx) = mpsc::channel::<Result<Bytes, WalRedoError>>();
+        let start_time;
+        let lock_time;
+        let end_time;
 
         let request = WalRedoRequest {
-            tag,
+            rel,
+            blknum,
             lsn,
             base_img,
             records,
-            response_channel: tx,
         };
 
-        self.request_tx
-            .lock()
-            .unwrap()
-            .send(request)
-            .expect("could not send WAL redo request");
+        start_time = Instant::now();
+        let result = {
+            let mut process_guard = self.process.lock().unwrap();
+            lock_time = Instant::now();
 
-        rx.recv()
-            .expect("could not receive response to WAL redo request")
+            // launch the WAL redo process on first use
+            if process_guard.is_none() {
+                let p = self
+                    .runtime
+                    .block_on(PostgresRedoProcess::launch(self.conf, &self.tenantid))?;
+                *process_guard = Some(p);
+            }
+            let process = (*process_guard).as_ref().unwrap();
+
+            self.runtime
+                .block_on(self.handle_apply_request(&process, &request))
+        };
+        end_time = Instant::now();
+
+        WAL_REDO_WAIT_TIME.observe(lock_time.duration_since(start_time).as_secs_f64());
+        WAL_REDO_TIME.observe(end_time.duration_since(lock_time).as_secs_f64());
+
+        result
     }
 }
 
-///
-/// WAL redo thread
-///
-impl PostgresRedoManagerInternal {
-    //
-    // Main entry point for the WAL applicator thread.
-    //
-    fn wal_redo_main(&mut self) {
-        info!("WAL redo thread started");
-
+impl PostgresRedoManager {
+    ///
+    /// Create a new PostgresRedoManager.
+    ///
+    pub fn new(conf: &'static PageServerConf, tenantid: ZTenantId) -> PostgresRedoManager {
         // We block on waiting for requests on the walredo request channel, but
         // use async I/O to communicate with the child process. Initialize the
         // runtime for the async part.
@@ -203,35 +233,12 @@ impl PostgresRedoManagerInternal {
             .build()
             .unwrap();
 
-        let process: PostgresRedoProcess;
-
-        // FIXME: We need a dummy Postgres cluster to run the process in. Currently, we
-        // just create one with constant name. That fails if you try to launch more than
-        // one WAL redo manager concurrently.
-        let datadir = self.conf.workdir.join("wal-redo-datadir");
-
-        info!("launching WAL redo postgres process");
-
-        process = runtime
-            .block_on(PostgresRedoProcess::launch(&datadir))
-            .unwrap();
-
-        // Loop forever, handling requests as they come.
-        loop {
-            let request = self
-                .request_rx
-                .recv()
-                .expect("WAL redo request channel was closed");
-
-            let result = runtime.block_on(self.handle_apply_request(&process, &request));
-            let result_ok = result.is_ok();
-
-            // Send the result to the requester
-            let _ = request.response_channel.send(result);
-
-            if !result_ok {
-                error!("wal-redo-postgres failed to apply request {:?}", request);
-            }
+        // The actual process is launched lazily, on first request.
+        PostgresRedoManager {
+            runtime,
+            tenantid,
+            conf,
+            process: Mutex::new(None),
         }
     }
 
@@ -243,7 +250,8 @@ impl PostgresRedoManagerInternal {
         process: &PostgresRedoProcess,
         request: &WalRedoRequest,
     ) -> Result<Bytes, WalRedoError> {
-        let tag = request.tag;
+        let rel = request.rel;
+        let blknum = request.blknum;
         let lsn = request.lsn;
         let base_img = request.base_img.clone();
         let records = &request.records;
@@ -253,7 +261,153 @@ impl PostgresRedoManagerInternal {
         let start = Instant::now();
 
         let apply_result: Result<Bytes, Error>;
-        apply_result = process.apply_wal_records(tag, base_img, records).await;
+        if let RelishTag::Relation(rel) = rel {
+            // Relational WAL records are applied using wal-redo-postgres
+            let buf_tag = BufferTag { rel, blknum };
+            apply_result = process.apply_wal_records(buf_tag, base_img, records).await;
+        } else {
+            // Non-relational WAL records are handled here, with custom code that has the
+            // same effects as the corresponding Postgres WAL redo function.
+            const ZERO_PAGE: [u8; 8192] = [0u8; 8192];
+            let mut page = BytesMut::new();
+            if let Some(fpi) = base_img {
+                // If full-page image is provided, then use it...
+                page.extend_from_slice(&fpi[..]);
+            } else {
+                // otherwise initialize page with zeros
+                page.extend_from_slice(&ZERO_PAGE);
+            }
+            // Apply all collected WAL records
+            for record in records {
+                let mut buf = record.rec.clone();
+
+                WAL_REDO_RECORD_COUNTER.inc();
+
+                // 1. Parse XLogRecord struct
+                // FIXME: refactor to avoid code duplication.
+                let xlogrec = XLogRecord::from_bytes(&mut buf);
+
+                //move to main data
+                // TODO probably, we should store some records in our special format
+                // to avoid this weird parsing on replay
+                let skip = (record.main_data_offset - pg_constants::SIZEOF_XLOGRECORD) as usize;
+                if buf.remaining() > skip {
+                    buf.advance(skip);
+                }
+
+                if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
+                    // Transaction manager stuff
+                    let rec_segno = match rel {
+                        RelishTag::Slru { slru, segno } => {
+                            if slru != SlruKind::Clog {
+                                panic!("Not valid XACT relish tag {:?}", rel);
+                            }
+                            segno
+                        }
+                        _ => panic!("Not valid XACT relish tag {:?}", rel),
+                    };
+                    let parsed_xact =
+                        XlXactParsedRecord::decode(&mut buf, xlogrec.xl_xid, xlogrec.xl_info);
+                    if parsed_xact.info == pg_constants::XLOG_XACT_COMMIT
+                        || parsed_xact.info == pg_constants::XLOG_XACT_COMMIT_PREPARED
+                    {
+                        transaction_id_set_status(
+                            parsed_xact.xid,
+                            pg_constants::TRANSACTION_STATUS_COMMITTED,
+                            &mut page,
+                        );
+                        for subxact in &parsed_xact.subxacts {
+                            let pageno = *subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
+                            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+                            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+                            // only update xids on the requested page
+                            if rec_segno == segno && blknum == rpageno {
+                                transaction_id_set_status(
+                                    *subxact,
+                                    pg_constants::TRANSACTION_STATUS_SUB_COMMITTED,
+                                    &mut page,
+                                );
+                            }
+                        }
+                    } else if parsed_xact.info == pg_constants::XLOG_XACT_ABORT
+                        || parsed_xact.info == pg_constants::XLOG_XACT_ABORT_PREPARED
+                    {
+                        transaction_id_set_status(
+                            parsed_xact.xid,
+                            pg_constants::TRANSACTION_STATUS_ABORTED,
+                            &mut page,
+                        );
+                        for subxact in &parsed_xact.subxacts {
+                            let pageno = *subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
+                            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+                            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+                            // only update xids on the requested page
+                            if rec_segno == segno && blknum == rpageno {
+                                transaction_id_set_status(
+                                    *subxact,
+                                    pg_constants::TRANSACTION_STATUS_ABORTED,
+                                    &mut page,
+                                );
+                            }
+                        }
+                    }
+                } else if xlogrec.xl_rmid == pg_constants::RM_MULTIXACT_ID {
+                    // Multixact operations
+                    let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
+                    if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
+                        let xlrec = XlMultiXactCreate::decode(&mut buf);
+                        if let RelishTag::Slru {
+                            slru,
+                            segno: rec_segno,
+                        } = rel
+                        {
+                            if slru == SlruKind::MultiXactMembers {
+                                for i in 0..xlrec.nmembers {
+                                    let pageno =
+                                        i / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
+                                    let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+                                    let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+                                    if segno == rec_segno && rpageno == blknum {
+                                        // update only target block
+                                        let offset = xlrec.moff + i;
+                                        let memberoff = mx_offset_to_member_offset(offset);
+                                        let flagsoff = mx_offset_to_flags_offset(offset);
+                                        let bshift = mx_offset_to_flags_bitshift(offset);
+                                        let mut flagsval =
+                                            LittleEndian::read_u32(&page[flagsoff..flagsoff + 4]);
+                                        flagsval &= !(((1
+                                            << pg_constants::MXACT_MEMBER_BITS_PER_XACT)
+                                            - 1)
+                                            << bshift);
+                                        flagsval |= xlrec.members[i as usize].status << bshift;
+                                        LittleEndian::write_u32(
+                                            &mut page[flagsoff..flagsoff + 4],
+                                            flagsval,
+                                        );
+                                        LittleEndian::write_u32(
+                                            &mut page[memberoff..memberoff + 4],
+                                            xlrec.members[i as usize].xid,
+                                        );
+                                    }
+                                }
+                            } else {
+                                // Multixact offsets SLRU
+                                let offs = (xlrec.mid
+                                    % pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32
+                                    * 4) as usize;
+                                LittleEndian::write_u32(&mut page[offs..offs + 4], xlrec.moff);
+                            }
+                        } else {
+                            panic!();
+                        }
+                    } else {
+                        panic!();
+                    }
+                }
+            }
+
+            apply_result = Ok::<Bytes, Error>(page.freeze());
+        }
 
         let duration = start.elapsed();
 
@@ -280,6 +434,9 @@ impl PostgresRedoManagerInternal {
     }
 }
 
+///
+/// Handle to the Postgres WAL redo process
+///
 struct PostgresRedoProcess {
     stdin: RefCell<ChildStdin>,
     stdout: RefCell<ChildStdout>,
@@ -289,12 +446,15 @@ impl PostgresRedoProcess {
     //
     // Start postgres binary in special WAL redo mode.
     //
-    // Tests who run pageserver binary are setting proper PG_BIN_DIR
-    // and PG_LIB_DIR so that WalRedo would start right postgres.
+    async fn launch(
+        conf: &PageServerConf,
+        tenantid: &ZTenantId,
+    ) -> Result<PostgresRedoProcess, Error> {
+        // FIXME: We need a dummy Postgres cluster to run the process in. Currently, we
+        // just create one with constant name. That fails if you try to launch more than
+        // one WAL redo manager concurrently.
+        let datadir = conf.tenant_path(&tenantid).join("wal-redo-datadir");
 
-    // do that: We may later
-    // switch to setting same things in pageserver config file.
-    async fn launch(datadir: &Path) -> Result<PostgresRedoProcess, Error> {
         // Create empty data directory for wal-redo postgres, deleting old one first.
         if datadir.exists() {
             info!("directory {:?} exists, removing", &datadir);
@@ -303,9 +463,12 @@ impl PostgresRedoProcess {
             }
         }
         info!("running initdb in {:?}", datadir.display());
-        let initdb = Command::new("initdb")
+        let initdb = Command::new(conf.pg_bin_dir().join("initdb"))
             .args(&["-D", datadir.to_str().unwrap()])
             .arg("-N")
+            .env_clear()
+            .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
+            .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
             .output()
             .await
             .expect("failed to execute initdb");
@@ -327,12 +490,15 @@ impl PostgresRedoProcess {
             config.write_all(b"zenith.wal_redo=on\n")?;
         }
         // Start postgres itself
-        let mut child = Command::new("postgres")
+        let mut child = Command::new(conf.pg_bin_dir().join("postgres"))
             .arg("--wal-redo")
             .stdin(Stdio::piped())
             .stderr(Stdio::piped())
             .stdout(Stdio::piped())
-            .env("PGDATA", datadir)
+            .env_clear()
+            .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
+            .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
+            .env("PGDATA", &datadir)
             .spawn()
             .expect("postgres --wal-redo command failed to start");
 
@@ -412,6 +578,8 @@ impl PostgresRedoProcess {
             for rec in records.iter() {
                 let r = rec.clone();
 
+                WAL_REDO_RECORD_COUNTER.inc();
+
                 stdin
                     .write_all(&build_apply_record_msg(r.lsn, r.rec))
                     .await?;

postgres_ffi/Cargo.toml

@@ -17,8 +17,11 @@ crc32c = "0.6.0"
 hex = "0.4.3"
 lazy_static = "1.4"
 log = "0.4.14"
+memoffset = "0.6.2"
 thiserror = "1.0"
+serde = { version = "1.0", features = ["derive"] }
 workspace_hack = { path = "../workspace_hack" }
+zenith_utils = { path = "../zenith_utils" }
 
 [build-dependencies]
-bindgen = "0.57"
+bindgen = "0.59.1"

postgres_ffi/README

@@ -1,3 +1,20 @@
-This module contains utility functions for interacting with PostgreSQL
-file formats.
+This module contains utilities for working with PostgreSQL file
+formats. It's a collection of structs that are auto-generated from the
+PostgreSQL header files using bindgen, and Rust functions to read and
+manipulate them.
 
+There are also a bunch of constants in `pg_constants.rs` that are copied
+from various PostgreSQL headers, rather than auto-generated. They mostly
+should be auto-generated too, but that's a TODO.
+
+The PostgreSQL on-disk file format is not portable across different
+CPU architectures and operating systems. It is also subject to change
+in each major PostgreSQL version. Currently, this module is based on
+PostgreSQL v14, but in the future we will probably need a separate
+copy for each PostgreSQL version.
+
+TODO: Currently, there is also some code that deals with WAL records
+in pageserver/src/waldecoder.rs.  That should be moved into this
+module. The rest of the codebase should not have intimate knowledge of
+PostgreSQL file formats or WAL layout, that knowledge should be
+encapsulated in this module.

postgres_ffi/build.rs

@@ -3,6 +3,44 @@ extern crate bindgen;
 use std::env;
 use std::path::PathBuf;
 
+use bindgen::callbacks::ParseCallbacks;
+
+#[derive(Debug)]
+struct PostgresFfiCallbacks;
+
+impl ParseCallbacks for PostgresFfiCallbacks {
+    fn include_file(&self, filename: &str) {
+        // This does the equivalent of passing bindgen::CargoCallbacks
+        // to the builder .parse_callbacks() method.
+        let cargo_callbacks = bindgen::CargoCallbacks;
+        cargo_callbacks.include_file(filename)
+    }
+
+    // Add any custom #[derive] attributes to the data structures that bindgen
+    // creates.
+    fn add_derives(&self, name: &str) -> Vec<String> {
+        // This is the list of data structures that we want to serialize/deserialize.
+        let serde_list = [
+            "XLogRecord",
+            "XLogPageHeaderData",
+            "XLogLongPageHeaderData",
+            "CheckPoint",
+            "FullTransactionId",
+            "ControlFileData",
+        ];
+
+        if serde_list.contains(&name) {
+            vec![
+                "Default".into(), // Default allows us to easily fill the padding fields with 0.
+                "Serialize".into(),
+                "Deserialize".into(),
+            ]
+        } else {
+            vec![]
+        }
+    }
+}
+
 fn main() {
     // Tell cargo to invalidate the built crate whenever the wrapper changes
     println!("cargo:rerun-if-changed=pg_control_ffi.h");
@@ -11,27 +49,50 @@ fn main() {
     // to bindgen, and lets you build up options for
     // the resulting bindings.
     let bindings = bindgen::Builder::default()
-        // The input header we would like to generate
-        // bindings for.
+        //
+        // All the needed PostgreSQL headers are included from 'pg_control_ffi.h'
+        //
         .header("pg_control_ffi.h")
+        //
         // Tell cargo to invalidate the built crate whenever any of the
         // included header files changed.
-        .parse_callbacks(Box::new(bindgen::CargoCallbacks))
-        .whitelist_type("ControlFileData")
-        .whitelist_var("PG_CONTROL_FILE_SIZE")
-        .whitelist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
-        .whitelist_type("DBState")
+        //
+        .parse_callbacks(Box::new(PostgresFfiCallbacks))
+        //
+        // These are the types and constants that we want to generate bindings for
+        //
+        .allowlist_type("BlockNumber")
+        .allowlist_type("OffsetNumber")
+        .allowlist_type("MultiXactId")
+        .allowlist_type("MultiXactOffset")
+        .allowlist_type("MultiXactStatus")
+        .allowlist_type("ControlFileData")
+        .allowlist_type("CheckPoint")
+        .allowlist_type("FullTransactionId")
+        .allowlist_type("XLogRecord")
+        .allowlist_type("XLogPageHeaderData")
+        .allowlist_type("XLogLongPageHeaderData")
+        .allowlist_var("XLOG_PAGE_MAGIC")
+        .allowlist_var("PG_CONTROL_FILE_SIZE")
+        .allowlist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
+        .allowlist_type("DBState")
+        // Because structs are used for serialization, tell bindgen to emit
+        // explicit padding fields.
+        .explicit_padding(true)
+        //
         // Path the server include dir. It is in tmp_install/include/server, if you did
         // "configure --prefix=<path to tmp_install>". But if you used "configure --prefix=/",
         // and used DESTDIR to move it into tmp_install, then it's in
         // tmp_install/include/postgres/server
         // 'pg_config --includedir-server' would perhaps be the more proper way to find it,
         // but this will do for now.
+        //
         .clang_arg("-I../tmp_install/include/server")
         .clang_arg("-I../tmp_install/include/postgresql/server")
+        //
         // Finish the builder and generate the bindings.
+        //
         .generate()
-        // Unwrap the Result and panic on failure.
         .expect("Unable to generate bindings");
 
     // Write the bindings to the $OUT_DIR/bindings.rs file.

postgres_ffi/pg_control_ffi.h

@@ -1,4 +1,13 @@
+/*
+ * This header file is the input to bindgen. It includes all the
+ * PostgreSQL headers that we need to auto-generate Rust structs
+ * from. If you need to expose a new struct to Rust code, add the
+ * header here, and whitelist the struct in the build.rs file.
+ */
 #include "c.h"
 #include "catalog/pg_control.h"
+#include "access/xlog_internal.h"
 
-const uint32 PG_CONTROLFILEDATA_OFFSETOF_CRC = offsetof(ControlFileData, crc);
+#include "storage/block.h"
+#include "storage/off.h"
+#include "access/multixact.h"

postgres_ffi/samples/pg_hba.conf

@@ -0,0 +1,98 @@
+# PostgreSQL Client Authentication Configuration File
+# ===================================================
+#
+# Refer to the "Client Authentication" section in the PostgreSQL
+# documentation for a complete description of this file.  A short
+# synopsis follows.
+#
+# This file controls: which hosts are allowed to connect, how clients
+# are authenticated, which PostgreSQL user names they can use, which
+# databases they can access.  Records take one of these forms:
+#
+# local         DATABASE  USER  METHOD  [OPTIONS]
+# host          DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+# hostssl       DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+# hostnossl     DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+# hostgssenc    DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+# hostnogssenc  DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+#
+# (The uppercase items must be replaced by actual values.)
+#
+# The first field is the connection type:
+# - "local" is a Unix-domain socket
+# - "host" is a TCP/IP socket (encrypted or not)
+# - "hostssl" is a TCP/IP socket that is SSL-encrypted
+# - "hostnossl" is a TCP/IP socket that is not SSL-encrypted
+# - "hostgssenc" is a TCP/IP socket that is GSSAPI-encrypted
+# - "hostnogssenc" is a TCP/IP socket that is not GSSAPI-encrypted
+#
+# DATABASE can be "all", "sameuser", "samerole", "replication", a
+# database name, or a comma-separated list thereof. The "all"
+# keyword does not match "replication". Access to replication
+# must be enabled in a separate record (see example below).
+#
+# USER can be "all", a user name, a group name prefixed with "+", or a
+# comma-separated list thereof.  In both the DATABASE and USER fields
+# you can also write a file name prefixed with "@" to include names
+# from a separate file.
+#
+# ADDRESS specifies the set of hosts the record matches.  It can be a
+# host name, or it is made up of an IP address and a CIDR mask that is
+# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that
+# specifies the number of significant bits in the mask.  A host name
+# that starts with a dot (.) matches a suffix of the actual host name.
+# Alternatively, you can write an IP address and netmask in separate
+# columns to specify the set of hosts.  Instead of a CIDR-address, you
+# can write "samehost" to match any of the server's own IP addresses,
+# or "samenet" to match any address in any subnet that the server is
+# directly connected to.
+#
+# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256",
+# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert".
+# Note that "password" sends passwords in clear text; "md5" or
+# "scram-sha-256" are preferred since they send encrypted passwords.
+#
+# OPTIONS are a set of options for the authentication in the format
+# NAME=VALUE.  The available options depend on the different
+# authentication methods -- refer to the "Client Authentication"
+# section in the documentation for a list of which options are
+# available for which authentication methods.
+#
+# Database and user names containing spaces, commas, quotes and other
+# special characters must be quoted.  Quoting one of the keywords
+# "all", "sameuser", "samerole" or "replication" makes the name lose
+# its special character, and just match a database or username with
+# that name.
+#
+# This file is read on server startup and when the server receives a
+# SIGHUP signal.  If you edit the file on a running system, you have to
+# SIGHUP the server for the changes to take effect, run "pg_ctl reload",
+# or execute "SELECT pg_reload_conf()".
+#
+# Put your actual configuration here
+# ----------------------------------
+#
+# If you want to allow non-local connections, you need to add more
+# "host" records.  In that case you will also need to make PostgreSQL
+# listen on a non-local interface via the listen_addresses
+# configuration parameter, or via the -i or -h command line switches.
+
+# CAUTION: Configuring the system for local "trust" authentication
+# allows any local user to connect as any PostgreSQL user, including
+# the database superuser.  If you do not trust all your local users,
+# use another authentication method.
+
+
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+
+# "local" is for Unix domain socket connections only
+local   all             all                                     trust
+# IPv4 local connections:
+host    all             all             127.0.0.1/32            trust
+# IPv6 local connections:
+host    all             all             ::1/128                 trust
+# Allow replication connections from localhost, by a user with the
+# replication privilege.
+local   replication     all                                     trust
+host    replication     all             127.0.0.1/32            trust
+host    replication     all             ::1/128                 trust

postgres_ffi/src/controlfile_utils.rs

@@ -0,0 +1,97 @@
+//!
+//! Utilities for reading and writing the PostgreSQL control file.
+//!
+//! The PostgreSQL control file is one the first things that the PostgreSQL
+//! server reads when it starts up. It indicates whether the server was shut
+//! down cleanly, or if it crashed or was restored from online backup so that
+//! WAL recovery needs to be performed. It also contains a copy of the latest
+//! checkpoint record and its location in the WAL.
+//!
+//! The control file also contains fields for detecting whether the
+//! data directory is compatible with a postgres binary. That includes
+//! a version number, configuration options that can be set at
+//! compilation time like the block size, and the platform's alignment
+//! and endianess information. (The PostgreSQL on-disk file format is
+//! not portable across platforms.)
+//!
+//! The control file is stored in the PostgreSQL data directory, as
+//! `global/pg_control`. The data stored in it is designed to be smaller than
+//! 512 bytes, on the assumption that it can be updated atomically. The actual
+//! file is larger, 8192 bytes, but the rest of it is just filled with zeros.
+//!
+//! See src/include/catalog/pg_control.h in the PostgreSQL sources for more
+//! information. You can use PostgreSQL's pg_controldata utility to view its
+//! contents.
+//!
+use crate::{ControlFileData, PG_CONTROL_FILE_SIZE};
+
+use anyhow::{bail, Result};
+use bytes::{Bytes, BytesMut};
+
+/// Equivalent to sizeof(ControlFileData) in C
+const SIZEOF_CONTROLDATA: usize = std::mem::size_of::<ControlFileData>();
+
+impl ControlFileData {
+    /// Compute the offset of the `crc` field within the `ControlFileData` struct.
+    /// Equivalent to offsetof(ControlFileData, crc) in C.
+    // Someday this can be const when the right compiler features land.
+    fn pg_control_crc_offset() -> usize {
+        memoffset::offset_of!(ControlFileData, crc)
+    }
+
+    ///
+    /// Interpret a slice of bytes as a Postgres control file.
+    ///
+    pub fn decode(buf: &[u8]) -> Result<ControlFileData> {
+        use zenith_utils::bin_ser::LeSer;
+
+        // Check that the slice has the expected size. The control file is
+        // padded with zeros up to a 512 byte sector size, so accept a
+        // larger size too, so that the caller can just the whole file
+        // contents without knowing the exact size of the struct.
+        if buf.len() < SIZEOF_CONTROLDATA {
+            bail!("control file is too short");
+        }
+
+        // Compute the expected CRC of the content.
+        let OFFSETOF_CRC = Self::pg_control_crc_offset();
+        let expectedcrc = crc32c::crc32c(&buf[0..OFFSETOF_CRC]);
+
+        // Use serde to deserialize the input as a ControlFileData struct.
+        let controlfile = ControlFileData::des(buf)?;
+
+        // Check the CRC
+        if expectedcrc != controlfile.crc {
+            bail!(
+                "invalid CRC in control file: expected {:08X}, was {:08X}",
+                expectedcrc,
+                controlfile.crc
+            );
+        }
+
+        Ok(controlfile)
+    }
+
+    ///
+    /// Convert a struct representing a Postgres control file into raw bytes.
+    ///
+    /// The CRC is recomputed to match the contents of the fields.
+    pub fn encode(&self) -> Bytes {
+        use zenith_utils::bin_ser::LeSer;
+
+        // Serialize into a new buffer.
+        let b = self.ser().unwrap();
+
+        // Recompute the CRC
+        let OFFSETOF_CRC = Self::pg_control_crc_offset();
+        let newcrc = crc32c::crc32c(&b[0..OFFSETOF_CRC]);
+
+        let mut buf = BytesMut::with_capacity(PG_CONTROL_FILE_SIZE as usize);
+        buf.extend_from_slice(&b[0..OFFSETOF_CRC]);
+        buf.extend_from_slice(&newcrc.to_ne_bytes());
+        // Fill the rest of the control file with zeros.
+        buf.resize(PG_CONTROL_FILE_SIZE as usize, 0);
+
+        buf.into()
+    }
+}

postgres_ffi/src/lib.rs

@@ -1,71 +1,36 @@
 #![allow(non_upper_case_globals)]
 #![allow(non_camel_case_types)]
 #![allow(non_snake_case)]
+// suppress warnings on rust 1.53 due to bindgen unit tests.
+// https://github.com/rust-lang/rust-bindgen/issues/1651
+#![allow(deref_nullptr)]
+
+use serde::{Deserialize, Serialize};
+
 include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
 
+pub mod controlfile_utils;
+pub mod nonrelfile_utils;
 pub mod pg_constants;
 pub mod relfile_utils;
 pub mod xlog_utils;
 
-use bytes::{Buf, Bytes, BytesMut};
-
-// sizeof(ControlFileData)
-const SIZEOF_CONTROLDATA: usize = std::mem::size_of::<ControlFileData>();
-const OFFSETOF_CRC: usize = PG_CONTROLFILEDATA_OFFSETOF_CRC as usize;
-
-impl ControlFileData {
-    // Initialize an all-zeros ControlFileData struct
-    pub fn new() -> ControlFileData {
-        let controlfile: ControlFileData;
-
-        let b = [0u8; SIZEOF_CONTROLDATA];
-        controlfile =
-            unsafe { std::mem::transmute::<[u8; SIZEOF_CONTROLDATA], ControlFileData>(b) };
-
-        controlfile
-    }
+//  See TransactionIdIsNormal in transam.h
+pub const fn transaction_id_is_normal(id: TransactionId) -> bool {
+    id > pg_constants::FIRST_NORMAL_TRANSACTION_ID
 }
 
-pub fn decode_pg_control(mut buf: Bytes) -> Result<ControlFileData, anyhow::Error> {
-    let mut b: [u8; SIZEOF_CONTROLDATA] = [0u8; SIZEOF_CONTROLDATA];
-    buf.copy_to_slice(&mut b);
+// See TransactionIdPrecedes in transam.c
+pub const fn transaction_id_precedes(id1: TransactionId, id2: TransactionId) -> bool {
+    /*
+     * If either ID is a permanent XID then we can just do unsigned
+     * comparison.  If both are normal, do a modulo-2^32 comparison.
+     */
 
-    let controlfile: ControlFileData;
-
-    // TODO: verify CRC
-    let mut data_without_crc: [u8; OFFSETOF_CRC] = [0u8; OFFSETOF_CRC];
-    data_without_crc.copy_from_slice(&b[0..OFFSETOF_CRC]);
-    let expectedcrc = crc32c::crc32c(&data_without_crc);
-
-    controlfile = unsafe { std::mem::transmute::<[u8; SIZEOF_CONTROLDATA], ControlFileData>(b) };
-
-    if expectedcrc != controlfile.crc {
-        anyhow::bail!(
-            "invalid CRC in control file: expected {:08X}, was {:08X}",
-            expectedcrc,
-            controlfile.crc
-        );
+    if !(transaction_id_is_normal(id1)) || !transaction_id_is_normal(id2) {
+        return id1 < id2;
     }
 
-    Ok(controlfile)
-}
-
-pub fn encode_pg_control(controlfile: ControlFileData) -> Bytes {
-    let b: [u8; SIZEOF_CONTROLDATA];
-
-    b = unsafe { std::mem::transmute::<ControlFileData, [u8; SIZEOF_CONTROLDATA]>(controlfile) };
-
-    // Recompute the CRC
-    let mut data_without_crc: [u8; OFFSETOF_CRC] = [0u8; OFFSETOF_CRC];
-    data_without_crc.copy_from_slice(&b[0..OFFSETOF_CRC]);
-    let newcrc = crc32c::crc32c(&data_without_crc);
-
-    let mut buf = BytesMut::with_capacity(PG_CONTROL_FILE_SIZE as usize);
-
-    buf.extend_from_slice(&b[0..OFFSETOF_CRC]);
-    buf.extend_from_slice(&newcrc.to_ne_bytes());
-    // Fill the rest of the control file with zeros.
-    buf.resize(PG_CONTROL_FILE_SIZE as usize, 0);
-
-    buf.into()
+    let diff = id1.wrapping_sub(id2) as i32;
+    return diff < 0;
 }

postgres_ffi/src/nonrelfile_utils.rs

@@ -1,10 +1,12 @@
 //!
 //! Common utilities for dealing with PostgreSQL non-relation files.
 //!
-use crate::pg_constants;
+use crate::{pg_constants, transaction_id_precedes};
 use bytes::BytesMut;
 use log::*;
 
+use crate::MultiXactId;
+
 pub fn transaction_id_set_status(xid: u32, status: u8, page: &mut BytesMut) {
     trace!(
         "handle_apply_request for RM_XACT_ID-{} (1-commit, 2-abort, 3-sub_commit)",
@@ -30,3 +32,51 @@ pub fn transaction_id_get_status(xid: u32, page: &[u8]) -> u8 {
 
     ((page[byteno] >> bshift) & pg_constants::CLOG_XACT_BITMASK) as u8
 }
+
+// See CLOGPagePrecedes in clog.c
+pub const fn clogpage_precedes(page1: u32, page2: u32) -> bool {
+    let mut xid1 = page1 * pg_constants::CLOG_XACTS_PER_PAGE;
+    xid1 += pg_constants::FIRST_NORMAL_TRANSACTION_ID + 1;
+    let mut xid2 = page2 * pg_constants::CLOG_XACTS_PER_PAGE;
+    xid2 += pg_constants::FIRST_NORMAL_TRANSACTION_ID + 1;
+
+    transaction_id_precedes(xid1, xid2)
+        && transaction_id_precedes(xid1, xid2 + pg_constants::CLOG_XACTS_PER_PAGE - 1)
+}
+
+// See SlruMayDeleteSegment() in slru.c
+pub fn slru_may_delete_clogsegment(segpage: u32, cutoff_page: u32) -> bool {
+    let seg_last_page = segpage + pg_constants::SLRU_PAGES_PER_SEGMENT - 1;
+
+    assert_eq!(segpage % pg_constants::SLRU_PAGES_PER_SEGMENT, 0);
+
+    clogpage_precedes(segpage, cutoff_page) && clogpage_precedes(seg_last_page, cutoff_page)
+}
+
+// Multixact utils
+
+pub fn mx_offset_to_flags_offset(xid: MultiXactId) -> usize {
+    ((xid / pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP as u32) as u16
+        % pg_constants::MULTIXACT_MEMBERGROUPS_PER_PAGE
+        * pg_constants::MULTIXACT_MEMBERGROUP_SIZE) as usize
+}
+
+pub fn mx_offset_to_flags_bitshift(xid: MultiXactId) -> u16 {
+    (xid as u16) % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP
+        * pg_constants::MXACT_MEMBER_BITS_PER_XACT
+}
+
+/* Location (byte offset within page) of TransactionId of given member */
+pub fn mx_offset_to_member_offset(xid: MultiXactId) -> usize {
+    mx_offset_to_flags_offset(xid)
+        + (pg_constants::MULTIXACT_FLAGBYTES_PER_GROUP
+            + (xid as u16 % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP) * 4) as usize
+}
+
+fn mx_offset_to_member_page(xid: u32) -> u32 {
+    xid / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32
+}
+
+pub fn mx_offset_to_member_segment(xid: u32) -> i32 {
+    (mx_offset_to_member_page(xid) / pg_constants::SLRU_PAGES_PER_SEGMENT) as i32
+}

postgres_ffi/src/pg_constants.rs

@@ -1,6 +1,11 @@
 //!
 //! Misc constants, copied from PostgreSQL headers.
 //!
+//! TODO: These probably should be auto-generated using bindgen,
+//! rather than copied by hand. Although on the other hand, it's nice
+//! to have them all here in one place, and have the ability to add
+//! comments on them.
+//!
 
 //
 // From pg_tablespace_d.h
@@ -16,13 +21,24 @@ pub const FSM_FORKNUM: u8 = 1;
 pub const VISIBILITYMAP_FORKNUM: u8 = 2;
 pub const INIT_FORKNUM: u8 = 3;
 
-pub const ROCKSDB_SPECIAL_FORKNUM: u8 = 50;
-
 // From storage_xlog.h
 pub const SMGR_TRUNCATE_HEAP: u32 = 0x0001;
 pub const SMGR_TRUNCATE_VM: u32 = 0x0002;
 pub const SMGR_TRUNCATE_FSM: u32 = 0x0004;
 
+// from pg_config.h. These can be changed with configure options --with-blocksize=BLOCKSIZE and
+// --with-segsize=SEGSIZE, but assume the defaults for now.
+pub const BLCKSZ: u16 = 8192;
+pub const RELSEG_SIZE: u32 = 1024 * 1024 * 1024 / (BLCKSZ as u32);
+
+//
+// constants from clog.h
+//
+pub const CLOG_XACTS_PER_BYTE: u32 = 4;
+pub const CLOG_XACTS_PER_PAGE: u32 = BLCKSZ as u32 * CLOG_XACTS_PER_BYTE;
+pub const CLOG_BITS_PER_XACT: u8 = 2;
+pub const CLOG_XACT_BITMASK: u8 = (1 << CLOG_BITS_PER_XACT) - 1;
+
 //
 // Constants from visbilitymap.h
 //
@@ -34,13 +50,23 @@ pub const TRANSACTION_STATUS_COMMITTED: u8 = 0x01;
 pub const TRANSACTION_STATUS_ABORTED: u8 = 0x02;
 pub const TRANSACTION_STATUS_SUB_COMMITTED: u8 = 0x03;
 
+pub const CLOG_ZEROPAGE: u8 = 0x00;
+pub const CLOG_TRUNCATE: u8 = 0x10;
+
 // From xact.h
 pub const XLOG_XACT_COMMIT: u8 = 0x00;
 pub const XLOG_XACT_PREPARE: u8 = 0x10;
 pub const XLOG_XACT_ABORT: u8 = 0x20;
+pub const XLOG_XACT_COMMIT_PREPARED: u8 = 0x30;
+pub const XLOG_XACT_ABORT_PREPARED: u8 = 0x40;
+
+// From srlu.h
+pub const SLRU_PAGES_PER_SEGMENT: u32 = 32;
+pub const SLRU_SEG_SIZE: usize = BLCKSZ as usize * SLRU_PAGES_PER_SEGMENT as usize;
 
 /* mask for filtering opcodes out of xl_info */
 pub const XLOG_XACT_OPMASK: u8 = 0x70;
+pub const XLOG_HEAP_OPMASK: u8 = 0x70;
 /* does this record have a 'xinfo' field or not */
 pub const XLOG_XACT_HAS_INFO: u8 = 0x80;
 
@@ -58,14 +84,42 @@ pub const XACT_XINFO_HAS_TWOPHASE: u32 = 1u32 << 4;
 // pub const XACT_XINFO_HAS_GID: u32 = 1u32 << 7;
 
 // From pg_control.h and rmgrlist.h
+pub const XLOG_NEXTOID: u8 = 0x30;
 pub const XLOG_SWITCH: u8 = 0x40;
 pub const XLOG_SMGR_TRUNCATE: u8 = 0x20;
+pub const XLOG_FPI_FOR_HINT: u8 = 0xA0;
+pub const XLOG_FPI: u8 = 0xB0;
+pub const DB_SHUTDOWNED: u32 = 1;
+
+// From multixact.h
+pub const FIRST_MULTIXACT_ID: u32 = 1;
+pub const MAX_MULTIXACT_ID: u32 = 0xFFFFFFFF;
+pub const MAX_MULTIXACT_OFFSET: u32 = 0xFFFFFFFF;
+
+pub const XLOG_MULTIXACT_ZERO_OFF_PAGE: u8 = 0x00;
+pub const XLOG_MULTIXACT_ZERO_MEM_PAGE: u8 = 0x10;
+pub const XLOG_MULTIXACT_CREATE_ID: u8 = 0x20;
+pub const XLOG_MULTIXACT_TRUNCATE_ID: u8 = 0x30;
+
+pub const MULTIXACT_OFFSETS_PER_PAGE: u16 = BLCKSZ / 4;
+pub const MXACT_MEMBER_BITS_PER_XACT: u16 = 8;
+pub const MXACT_MEMBER_FLAGS_PER_BYTE: u16 = 1;
+pub const MULTIXACT_FLAGBYTES_PER_GROUP: u16 = 4;
+pub const MULTIXACT_MEMBERS_PER_MEMBERGROUP: u16 =
+    MULTIXACT_FLAGBYTES_PER_GROUP * MXACT_MEMBER_FLAGS_PER_BYTE;
+/* size in bytes of a complete group */
+pub const MULTIXACT_MEMBERGROUP_SIZE: u16 =
+    4 * MULTIXACT_MEMBERS_PER_MEMBERGROUP + MULTIXACT_FLAGBYTES_PER_GROUP;
+pub const MULTIXACT_MEMBERGROUPS_PER_PAGE: u16 = BLCKSZ / MULTIXACT_MEMBERGROUP_SIZE;
+pub const MULTIXACT_MEMBERS_PER_PAGE: u16 =
+    MULTIXACT_MEMBERGROUPS_PER_PAGE * MULTIXACT_MEMBERS_PER_MEMBERGROUP;
 
 // From heapam_xlog.h
 pub const XLOG_HEAP_INSERT: u8 = 0x00;
 pub const XLOG_HEAP_DELETE: u8 = 0x10;
 pub const XLOG_HEAP_UPDATE: u8 = 0x20;
 pub const XLOG_HEAP_HOT_UPDATE: u8 = 0x40;
+pub const XLOG_HEAP_INIT_PAGE: u8 = 0x80;
 pub const XLOG_HEAP2_VISIBLE: u8 = 0x40;
 pub const XLOG_HEAP2_MULTI_INSERT: u8 = 0x50;
 pub const XLH_INSERT_ALL_FROZEN_SET: u8 = (1 << 5) as u8;
@@ -99,11 +153,6 @@ pub const XLOG_TBLSPC_DROP: u8 = 0x10;
 
 pub const SIZEOF_XLOGRECORD: u32 = 24;
 
-// from pg_config.h. These can be changed with configure options --with-blocksize=BLOCKSIZE and
-// --with-segsize=SEGSIZE, but assume the defaults for now.
-pub const BLCKSZ: u16 = 8192;
-pub const RELSEG_SIZE: u32 = 1024 * 1024 * 1024 / (BLCKSZ as u32);
-
 //
 // from xlogrecord.h
 //
@@ -126,5 +175,54 @@ pub const BKPIMAGE_HAS_HOLE: u8 = 0x01; /* page image has "hole" */
 pub const BKPIMAGE_IS_COMPRESSED: u8 = 0x02; /* page image is compressed */
 pub const BKPIMAGE_APPLY: u8 = 0x04; /* page image should be restored during replay */
 
+/* From transam.h */
+pub const FIRST_NORMAL_TRANSACTION_ID: u32 = 3;
+pub const INVALID_TRANSACTION_ID: u32 = 0;
+pub const FIRST_BOOTSTRAP_OBJECT_ID: u32 = 12000;
+pub const FIRST_NORMAL_OBJECT_ID: u32 = 16384;
+
 /* FIXME: pageserver should request wal_seg_size from compute node */
 pub const WAL_SEGMENT_SIZE: usize = 16 * 1024 * 1024;
+
+pub const XLOG_BLCKSZ: usize = 8192;
+pub const XLOG_CHECKPOINT_SHUTDOWN: u8 = 0x00;
+pub const XLOG_CHECKPOINT_ONLINE: u8 = 0x10;
+pub const XLP_LONG_HEADER: u16 = 0x0002;
+
+pub const PG_MAJORVERSION: &'static str = "14";
+
+// List of subdirectories inside pgdata.
+// Copied from src/bin/initdb/initdb.c
+pub const PGDATA_SUBDIRS: [&'static str; 22] = [
+    "global",
+    "pg_wal/archive_status",
+    "pg_commit_ts",
+    "pg_dynshmem",
+    "pg_notify",
+    "pg_serial",
+    "pg_snapshots",
+    "pg_subtrans",
+    "pg_twophase",
+    "pg_multixact",
+    "pg_multixact/members",
+    "pg_multixact/offsets",
+    "base",
+    "base/1",
+    "pg_replslot",
+    "pg_tblspc",
+    "pg_stat",
+    "pg_stat_tmp",
+    "pg_xact",
+    "pg_logical",
+    "pg_logical/snapshots",
+    "pg_logical/mappings",
+];
+
+pub const PGDATA_SPECIAL_FILES: [&'static str; 4] = [
+    "pg_hba.conf",
+    "pg_ident.conf",
+    "postgresql.conf",
+    "postgresql.auto.conf",
+];
+
+pub static PG_HBA: &'static str = include_str!("../samples/pg_hba.conf");

postgres_ffi/src/xlog_utils.rs

@@ -8,8 +8,17 @@
 //
 
 use crate::pg_constants;
+use crate::CheckPoint;
+use crate::ControlFileData;
+use crate::FullTransactionId;
+use crate::XLogLongPageHeaderData;
+use crate::XLogPageHeaderData;
+use crate::XLogRecord;
+use crate::XLOG_PAGE_MAGIC;
+
 use byteorder::{ByteOrder, LittleEndian};
 use bytes::{Buf, Bytes};
+use bytes::{BufMut, BytesMut};
 use crc32c::*;
 use log::*;
 use std::cmp::min;
@@ -21,19 +30,22 @@ use std::time::SystemTime;
 pub const XLOG_FNAME_LEN: usize = 24;
 pub const XLOG_BLCKSZ: usize = 8192;
 pub const XLP_FIRST_IS_CONTRECORD: u16 = 0x0001;
-pub const XLOG_PAGE_MAGIC: u16 = 0xD109;
 pub const XLP_REM_LEN_OFFS: usize = 2 + 2 + 4 + 8;
-pub const XLOG_SIZE_OF_XLOG_SHORT_PHD: usize = XLP_REM_LEN_OFFS + 4 + 4;
-pub const XLOG_SIZE_OF_XLOG_LONG_PHD: usize = XLOG_SIZE_OF_XLOG_SHORT_PHD + 8 + 4 + 4;
 pub const XLOG_RECORD_CRC_OFFS: usize = 4 + 4 + 8 + 1 + 1 + 2;
-pub const XLOG_SIZE_OF_XLOG_RECORD: usize = XLOG_RECORD_CRC_OFFS + 4;
 pub const MAX_SEND_SIZE: usize = XLOG_BLCKSZ * 16;
 
+pub const XLOG_SIZE_OF_XLOG_SHORT_PHD: usize = std::mem::size_of::<XLogPageHeaderData>();
+pub const XLOG_SIZE_OF_XLOG_LONG_PHD: usize = std::mem::size_of::<XLogLongPageHeaderData>();
+pub const XLOG_SIZE_OF_XLOG_RECORD: usize = std::mem::size_of::<XLogRecord>();
+pub const SIZE_OF_XLOG_RECORD_DATA_HEADER_SHORT: usize = 1 * 2;
+
 pub type XLogRecPtr = u64;
 pub type TimeLineID = u32;
-pub type TimestampTz = u64;
+pub type TimestampTz = i64;
 pub type XLogSegNo = u64;
 
+const XID_CHECKPOINT_INTERVAL: u32 = 1024;
+
 #[allow(non_snake_case)]
 pub fn XLogSegmentsPerXLogId(wal_segsz_bytes: usize) -> XLogSegNo {
     (0x100000000u64 / wal_segsz_bytes as u64) as XLogSegNo
@@ -83,9 +95,9 @@ pub fn get_current_timestamp() -> TimestampTz {
     const USECS_PER_SEC: u64 = 1000000;
     match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) {
         Ok(n) => {
-            (n.as_secs() - ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY))
+            ((n.as_secs() - ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY))
                 * USECS_PER_SEC
-                + n.subsec_micros() as u64
+                + n.subsec_micros() as u64) as i64
         }
         Err(_) => panic!("SystemTime before UNIX EPOCH!"),
     }
@@ -109,6 +121,7 @@ fn find_end_of_wal_segment(
     let mut rec_hdr = [0u8; XLOG_RECORD_CRC_OFFS];
 
     while offs < wal_seg_size {
+        // we are at the beginning of the page; read it in
         if offs % XLOG_BLCKSZ == 0 {
             if let Ok(bytes_read) = file.read(&mut buf) {
                 if bytes_read != buf.len() {
@@ -120,7 +133,7 @@ fn find_end_of_wal_segment(
             let xlp_magic = LittleEndian::read_u16(&buf[0..2]);
             let xlp_info = LittleEndian::read_u16(&buf[2..4]);
             let xlp_rem_len = LittleEndian::read_u32(&buf[XLP_REM_LEN_OFFS..XLP_REM_LEN_OFFS + 4]);
-            if xlp_magic != XLOG_PAGE_MAGIC {
+            if xlp_magic != XLOG_PAGE_MAGIC as u16 {
                 info!("Invalid WAL file {}.partial magic {}", file_name, xlp_magic);
                 break;
             }
@@ -132,11 +145,12 @@ fn find_end_of_wal_segment(
             } else {
                 offs += XLOG_SIZE_OF_XLOG_SHORT_PHD;
             }
+        // beginning of the next record
         } else if contlen == 0 {
             let page_offs = offs % XLOG_BLCKSZ;
             let xl_tot_len = LittleEndian::read_u32(&buf[page_offs..page_offs + 4]) as usize;
             if xl_tot_len == 0 {
-                break;
+                break; // zeros, reached the end
             }
             last_valid_rec_pos = offs;
             offs += 4;
@@ -144,12 +158,13 @@ fn find_end_of_wal_segment(
             contlen = xl_tot_len - 4;
             rec_hdr[0..4].copy_from_slice(&buf[page_offs..page_offs + 4]);
         } else {
-            let page_offs = offs % XLOG_BLCKSZ;
             // we're continuing a record, possibly from previous page.
+            let page_offs = offs % XLOG_BLCKSZ;
             let pageleft = XLOG_BLCKSZ - page_offs;
 
             // read the rest of the record, or as much as fits on this page.
             let n = min(contlen, pageleft);
+            // fill rec_hdr (header up to (but not including) xl_crc field)
             if rec_offs < XLOG_RECORD_CRC_OFFS {
                 let len = min(XLOG_RECORD_CRC_OFFS - rec_offs, n);
                 rec_hdr[rec_offs..rec_offs + len].copy_from_slice(&buf[page_offs..page_offs + len]);
@@ -173,6 +188,8 @@ fn find_end_of_wal_segment(
                 crc = crc32c_append(crc, &rec_hdr);
                 offs = (offs + 7) & !7; // pad on 8 bytes boundary */
                 if crc == wal_crc {
+                    // record is valid, advance the result to its end (with
+                    // alignment to the next record taken into account)
                     last_valid_rec_pos = offs;
                 } else {
                     info!(
@@ -189,6 +206,9 @@ fn find_end_of_wal_segment(
 
 ///
 /// Scan a directory that contains PostgreSQL WAL files, for the end of WAL.
+/// If precise, returns end LSN (next insertion point, basically);
+/// otherwise, start of the last segment.
+/// Returns (0, 0) if there is no WAL.
 ///
 pub fn find_end_of_wal(
     data_dir: &Path,
@@ -257,34 +277,15 @@ pub fn main() {
     );
 }
 
-//
-// Xlog record parsing routines
-// TODO move here other related code from waldecoder.rs
-//
-#[repr(C)]
-#[derive(Debug)]
-pub struct XLogRecord {
-    pub xl_tot_len: u32,
-    pub xl_xid: u32,
-    pub xl_prev: u64,
-    pub xl_info: u8,
-    pub xl_rmid: u8,
-    pub xl_crc: u32,
-}
-
 impl XLogRecord {
     pub fn from_bytes(buf: &mut Bytes) -> XLogRecord {
-        XLogRecord {
-            xl_tot_len: buf.get_u32_le(),
-            xl_xid: buf.get_u32_le(),
-            xl_prev: buf.get_u64_le(),
-            xl_info: buf.get_u8(),
-            xl_rmid: buf.get_u8(),
-            xl_crc: {
-                buf.advance(2);
-                buf.get_u32_le()
-            },
-        }
+        use zenith_utils::bin_ser::LeSer;
+        XLogRecord::des_from(&mut buf.reader()).unwrap()
+    }
+
+    pub fn encode(&self) -> Bytes {
+        use zenith_utils::bin_ser::LeSer;
+        self.ser().unwrap().into()
     }
 
     // Is this record an XLOG_SWITCH record? They need some special processing,
@@ -292,3 +293,190 @@ impl XLogRecord {
         self.xl_info == pg_constants::XLOG_SWITCH && self.xl_rmid == pg_constants::RM_XLOG_ID
     }
 }
+
+impl XLogPageHeaderData {
+    pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogPageHeaderData {
+        use zenith_utils::bin_ser::LeSer;
+        XLogPageHeaderData::des_from(&mut buf.reader()).unwrap()
+    }
+}
+
+impl XLogLongPageHeaderData {
+    pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogLongPageHeaderData {
+        use zenith_utils::bin_ser::LeSer;
+        XLogLongPageHeaderData::des_from(&mut buf.reader()).unwrap()
+    }
+
+    pub fn encode(&self) -> Bytes {
+        use zenith_utils::bin_ser::LeSer;
+        self.ser().unwrap().into()
+    }
+}
+
+pub const SIZEOF_CHECKPOINT: usize = std::mem::size_of::<CheckPoint>();
+
+impl CheckPoint {
+    pub fn encode(&self) -> Bytes {
+        use zenith_utils::bin_ser::LeSer;
+        self.ser().unwrap().into()
+    }
+
+    pub fn decode(buf: &[u8]) -> Result<CheckPoint, anyhow::Error> {
+        use zenith_utils::bin_ser::LeSer;
+        Ok(CheckPoint::des(buf)?)
+    }
+
+    // Update next XID based on provided new_xid and stored epoch.
+    // Next XID should be greater than new_xid.
+    // Also take in account 32-bit wrap-around.
+    pub fn update_next_xid(&mut self, xid: u32) {
+        let xid = xid.wrapping_add(XID_CHECKPOINT_INTERVAL - 1) & !(XID_CHECKPOINT_INTERVAL - 1);
+        let full_xid = self.nextXid.value;
+        let new_xid = std::cmp::max(xid + 1, pg_constants::FIRST_NORMAL_TRANSACTION_ID);
+        let old_xid = full_xid as u32;
+        if new_xid.wrapping_sub(old_xid) as i32 > 0 {
+            let mut epoch = full_xid >> 32;
+            if new_xid < old_xid {
+                // wrap-around
+                epoch += 1;
+            }
+            self.nextXid = FullTransactionId {
+                value: (epoch << 32) | new_xid as u64,
+            };
+        }
+    }
+}
+
+//
+// Generate new WAL segment with single XLOG_CHECKPOINT_SHUTDOWN record.
+// We need this segment to start compute node.
+// In order to minimize changes in Postgres core, we prefer to
+// provide WAL segment from which is can extract checkpoint record in standard way,
+// rather then implement some alternative mechanism.
+//
+pub fn generate_wal_segment(pg_control: &ControlFileData) -> Bytes {
+    let mut seg_buf = BytesMut::with_capacity(pg_constants::WAL_SEGMENT_SIZE as usize);
+
+    let hdr = XLogLongPageHeaderData {
+        std: {
+            XLogPageHeaderData {
+                xlp_magic: XLOG_PAGE_MAGIC as u16,
+                xlp_info: pg_constants::XLP_LONG_HEADER,
+                xlp_tli: 1, // FIXME: always use Postgres timeline 1
+                xlp_pageaddr: pg_control.checkPoint - XLOG_SIZE_OF_XLOG_LONG_PHD as u64,
+                xlp_rem_len: 0,
+                ..Default::default() // Put 0 in padding fields.
+            }
+        },
+        xlp_sysid: pg_control.system_identifier,
+        xlp_seg_size: pg_constants::WAL_SEGMENT_SIZE as u32,
+        xlp_xlog_blcksz: XLOG_BLCKSZ as u32,
+    };
+
+    let hdr_bytes = hdr.encode();
+    seg_buf.extend_from_slice(&hdr_bytes);
+
+    let rec_hdr = XLogRecord {
+        xl_tot_len: (XLOG_SIZE_OF_XLOG_RECORD
+            + SIZE_OF_XLOG_RECORD_DATA_HEADER_SHORT
+            + SIZEOF_CHECKPOINT) as u32,
+        xl_xid: 0, //0 is for InvalidTransactionId
+        xl_prev: 0,
+        xl_info: pg_constants::XLOG_CHECKPOINT_SHUTDOWN,
+        xl_rmid: pg_constants::RM_XLOG_ID,
+        xl_crc: 0,
+        ..Default::default() // Put 0 in padding fields.
+    };
+
+    let mut rec_shord_hdr_bytes = BytesMut::new();
+    rec_shord_hdr_bytes.put_u8(pg_constants::XLR_BLOCK_ID_DATA_SHORT);
+    rec_shord_hdr_bytes.put_u8(SIZEOF_CHECKPOINT as u8);
+
+    let rec_bytes = rec_hdr.encode();
+    let checkpoint_bytes = pg_control.checkPointCopy.encode();
+
+    //calculate record checksum
+    let mut crc = 0;
+    crc = crc32c_append(crc, &rec_shord_hdr_bytes[..]);
+    crc = crc32c_append(crc, &checkpoint_bytes[..]);
+    crc = crc32c_append(crc, &rec_bytes[0..XLOG_RECORD_CRC_OFFS]);
+
+    seg_buf.extend_from_slice(&rec_bytes[0..XLOG_RECORD_CRC_OFFS]);
+    seg_buf.put_u32_le(crc);
+    seg_buf.extend_from_slice(&rec_shord_hdr_bytes);
+    seg_buf.extend_from_slice(&checkpoint_bytes);
+
+    //zero out the rest of the file
+    seg_buf.resize(pg_constants::WAL_SEGMENT_SIZE, 0);
+    seg_buf.freeze()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use regex::Regex;
+    use std::{env, process::Command, str::FromStr};
+    use zenith_utils::lsn::Lsn;
+
+    // Run find_end_of_wal against file in test_wal dir
+    // Ensure that it finds last record correctly
+    #[test]
+    pub fn test_find_end_of_wal() {
+        // 1. Run initdb to generate some WAL
+        let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
+        let data_dir = top_path.join("test_output/test_find_end_of_wal");
+        let initdb_path = top_path.join("tmp_install/bin/initdb");
+        let lib_path = top_path.join("tmp_install/lib");
+        if data_dir.exists() {
+            fs::remove_dir_all(&data_dir).unwrap();
+        }
+        println!("Using initdb from '{}'", initdb_path.display());
+        println!("Data directory '{}'", data_dir.display());
+        let initdb_output = Command::new(initdb_path)
+            .args(&["-D", data_dir.to_str().unwrap()])
+            .arg("--no-instructions")
+            .arg("--no-sync")
+            .env_clear()
+            .env("LD_LIBRARY_PATH", &lib_path)
+            .env("DYLD_LIBRARY_PATH", &lib_path)
+            .output()
+            .unwrap();
+        assert!(initdb_output.status.success());
+
+        // 2. Pick WAL generated by initdb
+        let wal_dir = data_dir.join("pg_wal");
+        let wal_seg_size = 16 * 1024 * 1024;
+
+        // 3. Check end_of_wal on non-partial WAL segment (we treat it as fully populated)
+        let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true);
+        let wal_end = Lsn(wal_end);
+        println!("wal_end={}, tli={}", wal_end, tli);
+        assert_eq!(wal_end, "0/2000000".parse::<Lsn>().unwrap());
+
+        // 4. Get the actual end of WAL by pg_waldump
+        let waldump_path = top_path.join("tmp_install/bin/pg_waldump");
+        let waldump_output = Command::new(waldump_path)
+            .arg(wal_dir.join("000000010000000000000001"))
+            .env_clear()
+            .env("LD_LIBRARY_PATH", &lib_path)
+            .env("DYLD_LIBRARY_PATH", &lib_path)
+            .output()
+            .unwrap();
+        let waldump_output = std::str::from_utf8(&waldump_output.stderr).unwrap();
+        println!("waldump_output = '{}'", &waldump_output);
+        let re = Regex::new(r"invalid record length at (.+):").unwrap();
+        let caps = re.captures(&waldump_output).unwrap();
+        let waldump_wal_end = Lsn::from_str(caps.get(1).unwrap().as_str()).unwrap();
+
+        // 5. Rename file to partial to actually find last valid lsn
+        fs::rename(
+            wal_dir.join("000000010000000000000001"),
+            wal_dir.join("000000010000000000000001.partial"),
+        )
+        .unwrap();
+        let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true);
+        let wal_end = Lsn(wal_end);
+        println!("wal_end={}, tli={}", wal_end, tli);
+        assert_eq!(wal_end, waldump_wal_end);
+    }
+}

pre-commit.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+from typing import List
+import subprocess
+import sys
+import enum
+import argparse
+import os
+
+
+@enum.unique
+class Color(enum.Enum):
+    RED = "\033[0;31m"
+    GREEN = "\033[0;33m"
+    CYAN = "\033[0;36m"
+
+
+NC = "\033[0m"  # No Color
+
+
+def colorify(
+    s: str,
+    color: Color,
+    no_color: bool = False,
+):
+    if no_color:
+        return s
+    return f"{color.value}{s}{NC}"
+
+
+def rustfmt(fix_inplace: bool = False, no_color: bool = False) -> str:
+    cmd = "rustfmt --edition=2018"
+    if not fix_inplace:
+        cmd += " --check"
+    if no_color:
+        cmd += " --color=never"
+    return cmd
+
+
+def get_commit_files() -> List[str]:
+    files = subprocess.check_output(
+        "git diff --cached --name-only --diff-filter=ACM".split()
+    )
+    return files.decode().splitlines()
+
+
+def check(
+    name: str, suffix: str, cmd: str, changed_files: List[str], no_color: bool = False
+):
+    print(f"Checking: {name} ", end="")
+    applicable_files = list(
+        filter(lambda fname: fname.strip().endswith(suffix), changed_files)
+    )
+    if not applicable_files:
+        print(colorify("[NOT APPLICABLE]", Color.CYAN, no_color))
+        return
+
+    cmd = f'{cmd} {" ".join(applicable_files)}'
+    res = subprocess.run(cmd.split(), capture_output=True)
+    if res.returncode != 0:
+        print(colorify("[FAILED]", Color.RED, no_color))
+        print("Please inspect the output below and run make fmt to fix automatically\n")
+        print(res.stdout.decode())
+        exit(1)
+
+    print(colorify("[OK]", Color.GREEN, no_color))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--fix-inplace", action="store_true", help="apply fixes inplace"
+    )
+    parser.add_argument(
+        "--no-color", action="store_true", help="disable colored output", default=not sys.stdout.isatty()
+    )
+    args = parser.parse_args()
+
+    files = get_commit_files()
+    # we use rustfmt here because cargo fmt does not accept list of files
+    # it internally gathers project files and feeds them to rustfmt
+    # so because we want to check only files included in the commit we use rustfmt directly
+    check(
+        name="rustfmt",
+        suffix=".rs",
+        cmd=rustfmt(fix_inplace=args.fix_inplace, no_color=args.no_color),
+        changed_files=files,
+        no_color=args.no_color,
+    )

proxy/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "proxy"
+version = "0.1.0"
+authors = ["Stas Kelvich <stas.kelvich@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0"
+bytes = { version = "1.0.1", features = ['serde'] }
+md5 = "0.7.0"
+rand = "0.8.3"
+hex = "0.4.3"
+serde = "1"
+serde_json = "1"
+tokio = { version = "1.7.1", features = ["full"] }
+tokio-postgres = "0.7.2"
+clap = "2.33.0"
+rustls = "0.19.1"
+
+zenith_utils = { path = "../zenith_utils" }

proxy/src/cplane_api.rs

@@ -0,0 +1,92 @@
+use anyhow::{bail, Result};
+use serde::{Deserialize, Serialize};
+use std::{
+    collections::HashMap,
+    net::{IpAddr, SocketAddr},
+};
+
+pub struct CPlaneApi {
+    // address: SocketAddr,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct DatabaseInfo {
+    pub host: IpAddr, // TODO: allow host name here too
+    pub port: u16,
+    pub dbname: String,
+    pub user: String,
+    pub password: String,
+}
+
+impl DatabaseInfo {
+    pub fn socket_addr(&self) -> SocketAddr {
+        SocketAddr::new(self.host, self.port)
+    }
+
+    pub fn conn_string(&self) -> String {
+        format!(
+            "dbname={} user={} password={}",
+            self.dbname, self.user, self.password
+        )
+    }
+}
+
+// mock cplane api
+impl CPlaneApi {
+    pub fn new(_address: &SocketAddr) -> CPlaneApi {
+        CPlaneApi {
+            // address: address.clone(),
+        }
+    }
+
+    pub fn check_auth(&self, user: &str, md5_response: &[u8], salt: &[u8; 4]) -> Result<()> {
+        // passwords for both is "mypass"
+        let auth_map: HashMap<_, &str> = vec![
+            ("stas@zenith", "716ee6e1c4a9364d66285452c47402b1"),
+            ("stas2@zenith", "3996f75df64c16a8bfaf01301b61d582"),
+        ]
+        .into_iter()
+        .collect();
+
+        let stored_hash = auth_map
+            .get(&user)
+            .ok_or_else(|| anyhow::Error::msg("user not found"))?;
+        let salted_stored_hash = format!(
+            "md5{:x}",
+            md5::compute([stored_hash.as_bytes(), salt].concat())
+        );
+
+        let received_hash = std::str::from_utf8(&md5_response)?;
+
+        println!(
+            "auth: {} rh={} sh={} ssh={} {:?}",
+            user, received_hash, stored_hash, salted_stored_hash, salt
+        );
+
+        if received_hash == salted_stored_hash {
+            Ok(())
+        } else {
+            bail!("Auth failed")
+        }
+    }
+
+    pub fn get_database_uri(&self, _user: &str, _database: &str) -> Result<DatabaseInfo> {
+        Ok(DatabaseInfo {
+            host: "127.0.0.1".parse()?,
+            port: 5432,
+            dbname: "stas".to_string(),
+            user: "stas".to_string(),
+            password: "mypass".to_string(),
+        })
+    }
+
+    // pub fn create_database(&self, _user: &String, _database: &String) -> Result<DatabaseInfo> {
+    //     Ok(DatabaseInfo {
+    //         host: "127.0.0.1".parse()?,
+    //         port: 5432,
+    //         dbname: "stas".to_string(),
+    //         user: "stas".to_string(),
+    //         password: "mypass".to_string(),
+    //     })
+    // }
+}

proxy/src/main.rs

@@ -0,0 +1,157 @@
+///
+/// Postgres protocol proxy/router.
+///
+/// This service listens psql port and can check auth via external service
+/// (control plane API in our case) and can create new databases and accounts
+/// in somewhat transparent manner (again via communication with control plane API).
+///
+use std::{
+    collections::HashMap,
+    net::{SocketAddr, TcpListener},
+    sync::{mpsc, Arc, Mutex},
+    thread,
+};
+
+use anyhow::{anyhow, bail, ensure, Context};
+use clap::{App, Arg, ArgMatches};
+
+use cplane_api::DatabaseInfo;
+use rustls::{internal::pemfile, NoClientAuth, ProtocolVersion, ServerConfig};
+
+mod cplane_api;
+mod mgmt;
+mod proxy;
+
+pub struct ProxyConf {
+    /// main entrypoint for users to connect to
+    pub proxy_address: SocketAddr,
+
+    /// http management endpoint. Upon user account creation control plane
+    /// will notify us here, so that we can 'unfreeze' user session.
+    pub mgmt_address: SocketAddr,
+
+    /// send unauthenticated users to this URI
+    pub redirect_uri: String,
+
+    /// control plane address where we would check auth.
+    pub cplane_address: SocketAddr,
+
+    pub ssl_config: Option<Arc<ServerConfig>>,
+}
+
+pub struct ProxyState {
+    pub conf: ProxyConf,
+    pub waiters: Mutex<HashMap<String, mpsc::Sender<anyhow::Result<DatabaseInfo>>>>,
+}
+
+fn configure_ssl(arg_matches: &ArgMatches) -> anyhow::Result<Option<Arc<ServerConfig>>> {
+    let (key_path, cert_path) = match (
+        arg_matches.value_of("ssl-key"),
+        arg_matches.value_of("ssl-cert"),
+    ) {
+        (Some(key_path), Some(cert_path)) => (key_path, cert_path),
+        (None, None) => return Ok(None),
+        _ => bail!("either both or neither ssl-key and ssl-cert must be specified"),
+    };
+
+    let key = {
+        let key_bytes = std::fs::read(key_path).context("SSL key file")?;
+        let mut keys = pemfile::rsa_private_keys(&mut &key_bytes[..])
+            .or_else(|_| pemfile::pkcs8_private_keys(&mut &key_bytes[..]))
+            .map_err(|_| anyhow!("couldn't read TLS keys"))?;
+        ensure!(keys.len() == 1, "keys.len() = {} (should be 1)", keys.len());
+        keys.pop().unwrap()
+    };
+
+    let cert_chain = {
+        let cert_chain_bytes = std::fs::read(cert_path).context("SSL cert file")?;
+        pemfile::certs(&mut &cert_chain_bytes[..])
+            .map_err(|_| anyhow!("couldn't read TLS certificates"))?
+    };
+
+    let mut config = ServerConfig::new(NoClientAuth::new());
+    config.set_single_cert(cert_chain, key)?;
+    config.versions = vec![ProtocolVersion::TLSv1_3];
+
+    Ok(Some(Arc::new(config)))
+}
+
+fn main() -> anyhow::Result<()> {
+    let arg_matches = App::new("Zenith proxy/router")
+        .arg(
+            Arg::with_name("proxy")
+                .short("p")
+                .long("proxy")
+                .takes_value(true)
+                .help("listen for incoming client connections on ip:port")
+                .default_value("127.0.0.1:4432"),
+        )
+        .arg(
+            Arg::with_name("mgmt")
+                .short("m")
+                .long("mgmt")
+                .takes_value(true)
+                .help("listen for management callback connection on ip:port")
+                .default_value("127.0.0.1:7000"),
+        )
+        .arg(
+            Arg::with_name("uri")
+                .short("u")
+                .long("uri")
+                .takes_value(true)
+                .help("redirect unauthenticated users to given uri")
+                .default_value("http://localhost:3000/psql_session/"),
+        )
+        .arg(
+            Arg::with_name("ssl-key")
+                .short("k")
+                .long("ssl-key")
+                .takes_value(true)
+                .help("path to SSL key for client postgres connections"),
+        )
+        .arg(
+            Arg::with_name("ssl-cert")
+                .short("c")
+                .long("ssl-cert")
+                .takes_value(true)
+                .help("path to SSL cert for client postgres connections"),
+        )
+        .get_matches();
+
+    let conf = ProxyConf {
+        proxy_address: arg_matches.value_of("proxy").unwrap().parse()?,
+        mgmt_address: arg_matches.value_of("mgmt").unwrap().parse()?,
+        redirect_uri: arg_matches.value_of("uri").unwrap().parse()?,
+        cplane_address: "127.0.0.1:3000".parse()?,
+        ssl_config: configure_ssl(&arg_matches)?,
+    };
+    let state = ProxyState {
+        conf,
+        waiters: Mutex::new(HashMap::new()),
+    };
+    let state: &'static ProxyState = Box::leak(Box::new(state));
+
+    // Check that we can bind to address before further initialization
+    println!("Starting proxy on {}", state.conf.proxy_address);
+    let pageserver_listener = TcpListener::bind(state.conf.proxy_address)?;
+
+    println!("Starting mgmt on {}", state.conf.mgmt_address);
+    let mgmt_listener = TcpListener::bind(state.conf.mgmt_address)?;
+
+    let threads = vec![
+        // Spawn a thread to listen for connections. It will spawn further threads
+        // for each connection.
+        thread::Builder::new()
+            .name("Proxy thread".into())
+            .spawn(move || proxy::thread_main(&state, pageserver_listener))?,
+        thread::Builder::new()
+            .name("Mgmt thread".into())
+            .spawn(move || mgmt::thread_main(&state, mgmt_listener))?,
+    ];
+
+    for t in threads.into_iter() {
+        t.join().unwrap()?;
+    }
+
+    Ok(())
+}

proxy/src/mgmt.rs

@@ -0,0 +1,111 @@
+use std::{
+    net::{TcpListener, TcpStream},
+    thread,
+};
+
+use anyhow::bail;
+use bytes::Bytes;
+use serde::{Deserialize, Serialize};
+use zenith_utils::{
+    postgres_backend::{self, query_from_cstring, AuthType, PostgresBackend},
+    pq_proto::{BeMessage, SINGLE_COL_ROWDESC},
+};
+
+use crate::{cplane_api::DatabaseInfo, ProxyState};
+
+///
+/// Main proxy listener loop.
+///
+/// Listens for connections, and launches a new handler thread for each.
+///
+pub fn thread_main(state: &'static ProxyState, listener: TcpListener) -> anyhow::Result<()> {
+    loop {
+        let (socket, peer_addr) = listener.accept()?;
+        println!("accepted connection from {}", peer_addr);
+        socket.set_nodelay(true).unwrap();
+
+        thread::spawn(move || {
+            if let Err(err) = mgmt_conn_main(state, socket) {
+                println!("error: {}", err);
+            }
+        });
+    }
+}
+
+pub fn mgmt_conn_main(state: &'static ProxyState, socket: TcpStream) -> anyhow::Result<()> {
+    let mut conn_handler = MgmtHandler { state };
+    let pgbackend = PostgresBackend::new(socket, AuthType::Trust, None)?;
+    pgbackend.run(&mut conn_handler)
+}
+
+struct MgmtHandler {
+    state: &'static ProxyState,
+}
+/// Serialized examples:
+// {
+//     "session_id": "71d6d03e6d93d99a",
+//     "result": {
+//         "Success": {
+//             "host": "127.0.0.1",
+//             "port": 5432,
+//             "dbname": "stas",
+//             "user": "stas"
+//             "password": "mypass"
+//         }
+//     }
+// }
+// {
+//     "session_id": "71d6d03e6d93d99a",
+//     "result": {
+//         "Failure": "oops"
+//     }
+// }
+#[derive(Serialize, Deserialize)]
+pub struct PsqlSessionResponse {
+    session_id: String,
+    result: PsqlSessionResult,
+}
+
+#[derive(Serialize, Deserialize)]
+pub enum PsqlSessionResult {
+    Success(DatabaseInfo),
+    Failure(String),
+}
+
+impl postgres_backend::Handler for MgmtHandler {
+    fn process_query(
+        &mut self,
+        pgb: &mut PostgresBackend,
+        query_string: Bytes,
+    ) -> anyhow::Result<()> {
+        let query_string = query_from_cstring(query_string);
+
+        println!("Got mgmt query: '{}'", std::str::from_utf8(&query_string)?);
+
+        let resp: PsqlSessionResponse = serde_json::from_slice(&query_string)?;
+
+        let waiters = self.state.waiters.lock().unwrap();
+
+        let sender = waiters
+            .get(&resp.session_id)
+            .ok_or_else(|| anyhow::Error::msg("psql_session_id is not found"))?;
+
+        match resp.result {
+            PsqlSessionResult::Success(db_info) => {
+                sender.send(Ok(db_info))?;
+
+                pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
+                    .write_message_noflush(&BeMessage::DataRow(&[Some(b"ok")]))?
+                    .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
+                pgb.flush()?;
+                Ok(())
+            }
+
+            PsqlSessionResult::Failure(message) => {
+                sender.send(Err(anyhow::Error::msg(message.clone())))?;
+
+                bail!("psql session request failed: {}", message)
+            }
+        }
+    }
+}

proxy/src/proxy.rs

@@ -0,0 +1,291 @@
+use crate::cplane_api::CPlaneApi;
+use crate::cplane_api::DatabaseInfo;
+use crate::ProxyState;
+
+use anyhow::bail;
+use tokio_postgres::NoTls;
+
+use rand::Rng;
+use std::io::Write;
+use std::{io, sync::mpsc::channel, thread};
+use zenith_utils::postgres_backend::Stream;
+use zenith_utils::postgres_backend::{PostgresBackend, ProtoState};
+use zenith_utils::pq_proto::*;
+use zenith_utils::sock_split::{ReadStream, WriteStream};
+use zenith_utils::{postgres_backend, pq_proto::BeMessage};
+
+///
+/// Main proxy listener loop.
+///
+/// Listens for connections, and launches a new handler thread for each.
+///
+pub fn thread_main(
+    state: &'static ProxyState,
+    listener: std::net::TcpListener,
+) -> anyhow::Result<()> {
+    loop {
+        let (socket, peer_addr) = listener.accept()?;
+        println!("accepted connection from {}", peer_addr);
+        socket.set_nodelay(true).unwrap();
+
+        thread::spawn(move || {
+            if let Err(err) = proxy_conn_main(state, socket) {
+                println!("error: {}", err);
+            }
+        });
+    }
+}
+
+// XXX: clean up fields
+struct ProxyConnection {
+    state: &'static ProxyState,
+
+    cplane: CPlaneApi,
+
+    user: String,
+    database: String,
+
+    pgb: PostgresBackend,
+    md5_salt: [u8; 4],
+
+    psql_session_id: String,
+}
+
+pub fn proxy_conn_main(
+    state: &'static ProxyState,
+    socket: std::net::TcpStream,
+) -> anyhow::Result<()> {
+    let mut conn = ProxyConnection {
+        state,
+        cplane: CPlaneApi::new(&state.conf.cplane_address),
+        user: "".into(),
+        database: "".into(),
+        pgb: PostgresBackend::new(
+            socket,
+            postgres_backend::AuthType::MD5,
+            state.conf.ssl_config.clone(),
+        )?,
+        md5_salt: [0u8; 4],
+        psql_session_id: "".into(),
+    };
+
+    // Check StartupMessage
+    // This will set conn.existing_user and we can decide on next actions
+    conn.handle_startup()?;
+
+    // both scenarious here should end up producing database connection string
+    let db_info = if conn.is_existing_user() {
+        conn.handle_existing_user()?
+    } else {
+        conn.handle_new_user()?
+    };
+
+    proxy_pass(conn.pgb, db_info)
+}
+
+impl ProxyConnection {
+    fn is_existing_user(&self) -> bool {
+        self.user.ends_with("@zenith")
+    }
+
+    fn handle_startup(&mut self) -> anyhow::Result<()> {
+        let mut encrypted = false;
+        loop {
+            let msg = self.pgb.read_message()?;
+            println!("got message {:?}", msg);
+            match msg {
+                Some(FeMessage::StartupMessage(m)) => {
+                    println!("got startup message {:?}", m);
+
+                    match m.kind {
+                        StartupRequestCode::NegotiateGss => {
+                            self.pgb
+                                .write_message(&BeMessage::EncryptionResponse(false))?;
+                        }
+                        StartupRequestCode::NegotiateSsl => {
+                            println!("SSL requested");
+                            if self.pgb.tls_config.is_some() {
+                                self.pgb
+                                    .write_message(&BeMessage::EncryptionResponse(true))?;
+                                self.pgb.start_tls()?;
+                                encrypted = true;
+                            } else {
+                                self.pgb
+                                    .write_message(&BeMessage::EncryptionResponse(false))?;
+                            }
+                        }
+                        StartupRequestCode::Normal => {
+                            if self.state.conf.ssl_config.is_some() && !encrypted {
+                                self.pgb.write_message(&BeMessage::ErrorResponse(
+                                    "must connect with TLS".to_string(),
+                                ))?;
+                                bail!("client did not connect with TLS");
+                            }
+                            self.user = m
+                                .params
+                                .get("user")
+                                .ok_or_else(|| {
+                                    anyhow::Error::msg("user is required in startup packet")
+                                })?
+                                .into();
+                            self.database = m
+                                .params
+                                .get("database")
+                                .ok_or_else(|| {
+                                    anyhow::Error::msg("database is required in startup packet")
+                                })?
+                                .into();
+
+                            break;
+                        }
+                        StartupRequestCode::Cancel => break,
+                    }
+                }
+                None => {
+                    bail!("connection closed")
+                }
+                unexpected => {
+                    bail!("unexpected message type : {:?}", unexpected)
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn handle_existing_user(&mut self) -> anyhow::Result<DatabaseInfo> {
+        // ask password
+        rand::thread_rng().fill(&mut self.md5_salt);
+        self.pgb
+            .write_message(&BeMessage::AuthenticationMD5Password(&self.md5_salt))?;
+        self.pgb.state = ProtoState::Authentication; // XXX
+
+        // check password
+        println!("handle_existing_user");
+        let msg = self.pgb.read_message()?;
+        println!("got message {:?}", msg);
+        if let Some(FeMessage::PasswordMessage(m)) = msg {
+            println!("got password message '{:?}'", m);
+
+            assert!(self.is_existing_user());
+
+            let (_trailing_null, md5_response) = m
+                .split_last()
+                .ok_or_else(|| anyhow::Error::msg("unexpected password message"))?;
+
+            if let Err(e) = self.check_auth_md5(md5_response) {
+                self.pgb
+                    .write_message(&BeMessage::ErrorResponse(format!("{}", e)))?;
+                bail!("auth failed: {}", e);
+            } else {
+                self.pgb
+                    .write_message_noflush(&BeMessage::AuthenticationOk)?;
+                self.pgb
+                    .write_message_noflush(&BeMessage::ParameterStatus)?;
+                self.pgb.write_message(&BeMessage::ReadyForQuery)?;
+            }
+        }
+
+        // ok, we are authorized
+        self.cplane.get_database_uri(&self.user, &self.database)
+    }
+
+    fn handle_new_user(&mut self) -> anyhow::Result<DatabaseInfo> {
+        let mut psql_session_id_buf = [0u8; 8];
+        rand::thread_rng().fill(&mut psql_session_id_buf);
+        self.psql_session_id = hex::encode(psql_session_id_buf);
+
+        let hello_message = format!("☀️  Welcome to Zenith!
+
+To proceed with database creation, open the following link:
+
+    {redirect_uri}{sess_id}
+
+It needs to be done once and we will send you '.pgpass' file, which will allow you to access or create
+databases without opening the browser.
+
+", redirect_uri = self.state.conf.redirect_uri, sess_id = self.psql_session_id);
+
+        self.pgb
+            .write_message_noflush(&BeMessage::AuthenticationOk)?;
+        self.pgb
+            .write_message_noflush(&BeMessage::ParameterStatus)?;
+        self.pgb
+            .write_message(&BeMessage::NoticeResponse(hello_message))?;
+
+        // await for database creation
+        let (tx, rx) = channel::<anyhow::Result<DatabaseInfo>>();
+        let _ = self
+            .state
+            .waiters
+            .lock()
+            .unwrap()
+            .insert(self.psql_session_id.clone(), tx);
+
+        // Wait for web console response
+        // XXX: respond with error to client
+        let dbinfo = rx.recv()??;
+
+        self.pgb.write_message_noflush(&BeMessage::NoticeResponse(
+            "Connecting to database.".to_string(),
+        ))?;
+        self.pgb.write_message(&BeMessage::ReadyForQuery)?;
+
+        Ok(dbinfo)
+    }
+
+    fn check_auth_md5(&self, md5_response: &[u8]) -> anyhow::Result<()> {
+        assert!(self.is_existing_user());
+        self.cplane
+            .check_auth(self.user.as_str(), md5_response, &self.md5_salt)
+    }
+}
+
+/// Create a TCP connection to a postgres database, authenticate with it, and receive the ReadyForQuery message
+async fn connect_to_db(db_info: DatabaseInfo) -> anyhow::Result<tokio::net::TcpStream> {
+    let mut socket = tokio::net::TcpStream::connect(db_info.socket_addr()).await?;
+    let config = db_info.conn_string().parse::<tokio_postgres::Config>()?;
+    let _ = config.connect_raw(&mut socket, NoTls).await?;
+    Ok(socket)
+}
+
+/// Concurrently proxy both directions of the client and server connections
+fn proxy(
+    client_read: ReadStream,
+    client_write: WriteStream,
+    server_read: ReadStream,
+    server_write: WriteStream,
+) -> anyhow::Result<()> {
+    fn do_proxy(mut reader: ReadStream, mut writer: WriteStream) -> io::Result<()> {
+        std::io::copy(&mut reader, &mut writer)?;
+        writer.flush()?;
+        writer.shutdown(std::net::Shutdown::Both)
+    }
+
+    let client_to_server_jh = thread::spawn(move || do_proxy(client_read, server_write));
+
+    let res1 = do_proxy(server_read, client_write);
+    let res2 = client_to_server_jh.join().unwrap();
+    res1?;
+    res2?;
+
+    Ok(())
+}
+
+/// Proxy a client connection to a postgres database
+fn proxy_pass(pgb: PostgresBackend, db_info: DatabaseInfo) -> anyhow::Result<()> {
+    let runtime = tokio::runtime::Builder::new_current_thread().build()?;
+    let db_stream = runtime.block_on(connect_to_db(db_info))?;
+    let db_stream = db_stream.into_std()?;
+    db_stream.set_nonblocking(false)?;
+
+    let db_stream = zenith_utils::sock_split::BidiStream::from_tcp(db_stream);
+    let (db_read, db_write) = db_stream.split();
+
+    let stream = match pgb.into_stream() {
+        Stream::Bidirectional(bidi_stream) => bidi_stream,
+        _ => bail!("invalid stream"),
+    };
+
+    let (client_read, client_write) = stream.split();
+    proxy(client_read, client_write, db_read, db_write)
+}

run_clippy.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# If you save this in your path under the name "cargo-zclippy" (or whatever
+# name you like), then you can run it as "cargo zclippy" from the shell prompt.
+#
+# If your text editor has rust-analyzer integration, you can also use this new
+# command as a replacement for "cargo check" or "cargo clippy" and see clippy
+# warnings and errors right in the editor.
+# In vscode, this setting is Rust-analyzer>Check On Save:Command
+
+cargo clippy "${@:2}" -- -A clippy::new_without_default -A clippy::manual_range_contains -A clippy::comparison_chain

test_runner/Pipfile

@@ -0,0 +1,20 @@
+[[source]]
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+pytest = ">=6.0.0"
+psycopg2 = "*"
+typing-extensions = "*"
+pyjwt = {extras = ["crypto"], version = "*"}
+requests = "*"
+
+[dev-packages]
+yapf = "*"
+flake8 = "*"
+mypy = "*"
+
+[requires]
+# we need at least 3.6, but pipenv doesn't allow to say this directly
+python_version = "3"

test_runner/Pipfile.lock

@@ -0,0 +1,328 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "b666740289d9c82797e5c39b2a7f0074c865c9183ee878ce4fa5cda7928506ea"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.python.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "attrs": {
+            "hashes": [
+                "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1",
+                "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==21.2.0"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee",
+                "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"
+            ],
+            "version": "==2021.5.30"
+        },
+        "cffi": {
+            "hashes": [
+                "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d",
+                "sha256:0c0591bee64e438883b0c92a7bed78f6290d40bf02e54c5bf0978eaf36061771",
+                "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872",
+                "sha256:22b9c3c320171c108e903d61a3723b51e37aaa8c81255b5e7ce102775bd01e2c",
+                "sha256:26bb2549b72708c833f5abe62b756176022a7b9a7f689b571e74c8478ead51dc",
+                "sha256:33791e8a2dc2953f28b8d8d300dde42dd929ac28f974c4b4c6272cb2955cb762",
+                "sha256:3c8d896becff2fa653dc4438b54a5a25a971d1f4110b32bd3068db3722c80202",
+                "sha256:4373612d59c404baeb7cbd788a18b2b2a8331abcc84c3ba40051fcd18b17a4d5",
+                "sha256:487d63e1454627c8e47dd230025780e91869cfba4c753a74fda196a1f6ad6548",
+                "sha256:48916e459c54c4a70e52745639f1db524542140433599e13911b2f329834276a",
+                "sha256:4922cd707b25e623b902c86188aca466d3620892db76c0bdd7b99a3d5e61d35f",
+                "sha256:55af55e32ae468e9946f741a5d51f9896da6b9bf0bbdd326843fec05c730eb20",
+                "sha256:57e555a9feb4a8460415f1aac331a2dc833b1115284f7ded7278b54afc5bd218",
+                "sha256:5d4b68e216fc65e9fe4f524c177b54964af043dde734807586cf5435af84045c",
+                "sha256:64fda793737bc4037521d4899be780534b9aea552eb673b9833b01f945904c2e",
+                "sha256:6d6169cb3c6c2ad50db5b868db6491a790300ade1ed5d1da29289d73bbe40b56",
+                "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224",
+                "sha256:80b06212075346b5546b0417b9f2bf467fea3bfe7352f781ffc05a8ab24ba14a",
+                "sha256:818014c754cd3dba7229c0f5884396264d51ffb87ec86e927ef0be140bfdb0d2",
+                "sha256:8eb687582ed7cd8c4bdbff3df6c0da443eb89c3c72e6e5dcdd9c81729712791a",
+                "sha256:99f27fefe34c37ba9875f224a8f36e31d744d8083e00f520f133cab79ad5e819",
+                "sha256:9f3e33c28cd39d1b655ed1ba7247133b6f7fc16fa16887b120c0c670e35ce346",
+                "sha256:a8661b2ce9694ca01c529bfa204dbb144b275a31685a075ce123f12331be790b",
+                "sha256:a9da7010cec5a12193d1af9872a00888f396aba3dc79186604a09ea3ee7c029e",
+                "sha256:aedb15f0a5a5949ecb129a82b72b19df97bbbca024081ed2ef88bd5c0a610534",
+                "sha256:b315d709717a99f4b27b59b021e6207c64620790ca3e0bde636a6c7f14618abb",
+                "sha256:ba6f2b3f452e150945d58f4badd92310449876c4c954836cfb1803bdd7b422f0",
+                "sha256:c33d18eb6e6bc36f09d793c0dc58b0211fccc6ae5149b808da4a62660678b156",
+                "sha256:c9a875ce9d7fe32887784274dd533c57909b7b1dcadcc128a2ac21331a9765dd",
+                "sha256:c9e005e9bd57bc987764c32a1bee4364c44fdc11a3cc20a40b93b444984f2b87",
+                "sha256:d2ad4d668a5c0645d281dcd17aff2be3212bc109b33814bbb15c4939f44181cc",
+                "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195",
+                "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33",
+                "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f",
+                "sha256:e9dc245e3ac69c92ee4c167fbdd7428ec1956d4e754223124991ef29eb57a09d",
+                "sha256:eb687a11f0a7a1839719edd80f41e459cc5366857ecbed383ff376c4e3cc6afd",
+                "sha256:eb9e2a346c5238a30a746893f23a9535e700f8192a68c07c0258e7ece6ff3728",
+                "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7",
+                "sha256:f0010c6f9d1a4011e429109fda55a225921e3206e7f62a0c22a35344bfd13cca",
+                "sha256:f0c5d1acbfca6ebdd6b1e3eded8d261affb6ddcf2186205518f1428b8569bb99",
+                "sha256:f10afb1004f102c7868ebfe91c28f4a712227fe4cb24974350ace1f90e1febbf",
+                "sha256:f174135f5609428cc6e1b9090f9268f5c8935fddb1b25ccb8255a2d50de6789e",
+                "sha256:f3ebe6e73c319340830a9b2825d32eb6d8475c1dac020b4f0aa774ee3b898d1c",
+                "sha256:f627688813d0a4140153ff532537fbe4afea5a3dffce1f9deb7f91f848a832b5",
+                "sha256:fd4305f86f53dfd8cd3522269ed7fc34856a8ee3709a5e28b2836b2db9d4cd69"
+            ],
+            "version": "==1.14.6"
+        },
+        "charset-normalizer": {
+            "hashes": [
+                "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b",
+                "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==2.0.4"
+        },
+        "cryptography": {
+            "hashes": [
+                "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d",
+                "sha256:1e056c28420c072c5e3cb36e2b23ee55e260cb04eee08f702e0edfec3fb51959",
+                "sha256:240f5c21aef0b73f40bb9f78d2caff73186700bf1bc6b94285699aff98cc16c6",
+                "sha256:26965837447f9c82f1855e0bc8bc4fb910240b6e0d16a664bb722df3b5b06873",
+                "sha256:37340614f8a5d2fb9aeea67fd159bfe4f5f4ed535b1090ce8ec428b2f15a11f2",
+                "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713",
+                "sha256:3d8427734c781ea5f1b41d6589c293089704d4759e34597dce91014ac125aad1",
+                "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177",
+                "sha256:8e56e16617872b0957d1c9742a3f94b43533447fd78321514abbe7db216aa250",
+                "sha256:b01fd6f2737816cb1e08ed4807ae194404790eac7ad030b34f2ce72b332f5586",
+                "sha256:bf40af59ca2465b24e54f671b2de2c59257ddc4f7e5706dbd6930e26823668d3",
+                "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca",
+                "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d",
+                "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9"
+            ],
+            "version": "==3.4.7"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a",
+                "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==3.2"
+        },
+        "iniconfig": {
+            "hashes": [
+                "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
+                "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
+            ],
+            "version": "==1.1.1"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7",
+                "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==21.0"
+        },
+        "pluggy": {
+            "hashes": [
+                "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
+                "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==0.13.1"
+        },
+        "psycopg2": {
+            "hashes": [
+                "sha256:079d97fc22de90da1d370c90583659a9f9a6ee4007355f5825e5f1c70dffc1fa",
+                "sha256:2087013c159a73e09713294a44d0c8008204d06326006b7f652bef5ace66eebb",
+                "sha256:2c992196719fadda59f72d44603ee1a2fdcc67de097eea38d41c7ad9ad246e62",
+                "sha256:7640e1e4d72444ef012e275e7b53204d7fab341fb22bc76057ede22fe6860b25",
+                "sha256:7f91312f065df517187134cce8e395ab37f5b601a42446bdc0f0d51773621854",
+                "sha256:830c8e8dddab6b6716a4bf73a09910c7954a92f40cf1d1e702fb93c8a919cc56",
+                "sha256:89409d369f4882c47f7ea20c42c5046879ce22c1e4ea20ef3b00a4dfc0a7f188",
+                "sha256:bf35a25f1aaa8a3781195595577fcbb59934856ee46b4f252f56ad12b8043bcf",
+                "sha256:de5303a6f1d0a7a34b9d40e4d3bef684ccc44a49bbe3eb85e3c0bffb4a131b7c"
+            ],
+            "index": "pypi",
+            "version": "==2.9.1"
+        },
+        "py": {
+            "hashes": [
+                "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3",
+                "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.10.0"
+        },
+        "pycparser": {
+            "hashes": [
+                "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",
+                "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.20"
+        },
+        "pyjwt": {
+            "extras": [
+                "crypto"
+            ],
+            "hashes": [
+                "sha256:934d73fbba91b0483d3857d1aff50e96b2a892384ee2c17417ed3203f173fca1",
+                "sha256:fba44e7898bbca160a2b2b501f492824fc8382485d3a6f11ba5d0c1937ce6130"
+            ],
+            "index": "pypi",
+            "version": "==2.1.0"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
+                "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
+            ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.4.7"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b",
+                "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890"
+            ],
+            "index": "pypi",
+            "version": "==6.2.4"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24",
+                "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"
+            ],
+            "index": "pypi",
+            "version": "==2.26.0"
+        },
+        "toml": {
+            "hashes": [
+                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
+                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
+            ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==0.10.2"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497",
+                "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342",
+                "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"
+            ],
+            "index": "pypi",
+            "version": "==3.10.0.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4",
+                "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.26.6"
+        }
+    },
+    "develop": {
+        "flake8": {
+            "hashes": [
+                "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b",
+                "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"
+            ],
+            "index": "pypi",
+            "version": "==3.9.2"
+        },
+        "mccabe": {
+            "hashes": [
+                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
+                "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
+            ],
+            "version": "==0.6.1"
+        },
+        "mypy": {
+            "hashes": [
+                "sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9",
+                "sha256:0aadfb2d3935988ec3815952e44058a3100499f5be5b28c34ac9d79f002a4a9a",
+                "sha256:119bed3832d961f3a880787bf621634ba042cb8dc850a7429f643508eeac97b9",
+                "sha256:1a85e280d4d217150ce8cb1a6dddffd14e753a4e0c3cf90baabb32cefa41b59e",
+                "sha256:3c4b8ca36877fc75339253721f69603a9c7fdb5d4d5a95a1a1b899d8b86a4de2",
+                "sha256:3e382b29f8e0ccf19a2df2b29a167591245df90c0b5a2542249873b5c1d78212",
+                "sha256:42c266ced41b65ed40a282c575705325fa7991af370036d3f134518336636f5b",
+                "sha256:53fd2eb27a8ee2892614370896956af2ff61254c275aaee4c230ae771cadd885",
+                "sha256:704098302473cb31a218f1775a873b376b30b4c18229421e9e9dc8916fd16150",
+                "sha256:7df1ead20c81371ccd6091fa3e2878559b5c4d4caadaf1a484cf88d93ca06703",
+                "sha256:866c41f28cee548475f146aa4d39a51cf3b6a84246969f3759cb3e9c742fc072",
+                "sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457",
+                "sha256:adaeee09bfde366d2c13fe6093a7df5df83c9a2ba98638c7d76b010694db760e",
+                "sha256:b6fb13123aeef4a3abbcfd7e71773ff3ff1526a7d3dc538f3929a49b42be03f0",
+                "sha256:b94e4b785e304a04ea0828759172a15add27088520dc7e49ceade7834275bedb",
+                "sha256:c0df2d30ed496a08de5daed2a9ea807d07c21ae0ab23acf541ab88c24b26ab97",
+                "sha256:c6c2602dffb74867498f86e6129fd52a2770c48b7cd3ece77ada4fa38f94eba8",
+                "sha256:ceb6e0a6e27fb364fb3853389607cf7eb3a126ad335790fa1e14ed02fba50811",
+                "sha256:d9dd839eb0dc1bbe866a288ba3c1afc33a202015d2ad83b31e875b5905a079b6",
+                "sha256:e4dab234478e3bd3ce83bac4193b2ecd9cf94e720ddd95ce69840273bf44f6de",
+                "sha256:ec4e0cd079db280b6bdabdc807047ff3e199f334050db5cbb91ba3e959a67504",
+                "sha256:ecd2c3fe726758037234c93df7e98deb257fd15c24c9180dacf1ef829da5f921",
+                "sha256:ef565033fa5a958e62796867b1df10c40263ea9ded87164d67572834e57a174d"
+            ],
+            "index": "pypi",
+            "version": "==0.910"
+        },
+        "mypy-extensions": {
+            "hashes": [
+                "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
+                "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
+            ],
+            "version": "==0.4.3"
+        },
+        "pycodestyle": {
+            "hashes": [
+                "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068",
+                "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.7.0"
+        },
+        "pyflakes": {
+            "hashes": [
+                "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3",
+                "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.3.1"
+        },
+        "toml": {
+            "hashes": [
+                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
+                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
+            ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==0.10.2"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497",
+                "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342",
+                "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"
+            ],
+            "index": "pypi",
+            "version": "==3.10.0.0"
+        },
+        "yapf": {
+            "hashes": [
+                "sha256:408fb9a2b254c302f49db83c59f9aa0b4b0fd0ec25be3a5c51181327922ff63d",
+                "sha256:e3a234ba8455fe201eaa649cdac872d590089a18b661e39bbac7020978dd9c2e"
+            ],
+            "index": "pypi",
+            "version": "==0.31.0"
+        }
+    }
+}

test_runner/README.md

@@ -4,13 +4,11 @@ This directory contains integration tests.
 
 Prerequisites:
 - Python 3.6 or later
-- Python packages: pytest, psycopg2
-    - pytest 6.0 is required.
-    - __NOTE: `apt install` on Debian/Ubuntu won't work.__
-      They ship a much older version of pytest (and sometimes rename it to
-      `pytest-3`.)
-    - Install using something like this:
-        - `pip3 install pytest psycopg2` (Debian or Ubuntu)
+- Dependencies: install them via `pipenv install`. Note that Debian/Ubuntu
+  packages are stale, as it commonly happens, so manual installation is not
+  recommended.
+  Run `pipenv shell` to activate the venv or use `pipenv run` to run a single
+  command in the venv, e.g. `pipenv run pytest`.
 - Zenith and Postgres binaries
     - See the root README.md for build directions
     - Tests can be run from the git tree; or see the environment variables
@@ -72,7 +70,8 @@ The tests make heavy use of pytest fixtures. You can read about how they work he
 Essentially, this means that each time you see a fixture named as an input parameter, the function with that name will be run and passed as a parameter to the function.
 
 So this code:
-```
+
+```python
 def test_something(zenith_cli, pg_bin):
     pass
 ```
@@ -80,9 +79,11 @@ def test_something(zenith_cli, pg_bin):
 ... will run the fixtures called `zenith_cli` and `pg_bin` and deliver those results to the test function.
 
 Fixtures can't be imported using the normal python syntax. Instead, use this:
-```
+
+```python
 pytest_plugins = ("fixtures.something")
 ```
+
 That will make all the fixtures in the `fixtures/something.py` file available.
 
 Anything that's likely to be used in multiple tests should be built into a fixture.
@@ -90,3 +91,15 @@ Anything that's likely to be used in multiple tests should be built into a fixtu
 Note that fixtures can clean up after themselves if they use the `yield` syntax.
 Cleanup will happen even if the test fails (raises an unhandled exception).
 Python destructors, e.g. `__del__()` aren't recommended for cleanup.
+
+
+### Code quality
+
+Before submitting a patch, please consider:
+
+* Writing a couple of docstrings to clarify the reasoning behind a new test.
+* Running `flake8` (or a linter of your choice, e.g. `pycodestyle`) and fixing possible defects, if any.
+* Formatting the code with `yapf -r -i .` (TODO: implement an opt-in pre-commit hook for that).
+* (Optional) Typechecking the code with `mypy .`. Currently this mostly affects `fixtures/zenith_fixtures.py`.
+
+The tools can be installed with `pipenv install --dev`.

test_runner/batch_others/test_auth.py

@@ -0,0 +1,74 @@
+
+from contextlib import closing
+from uuid import uuid4
+import psycopg2
+from fixtures.zenith_fixtures import Postgres, ZenithCli, ZenithPageserver, PgBin
+import pytest
+
+
+def test_pageserver_auth(pageserver_auth_enabled: ZenithPageserver):
+    ps = pageserver_auth_enabled
+
+    tenant_token = ps.auth_keys.generate_tenant_token(ps.initial_tenant)
+    invalid_tenant_token = ps.auth_keys.generate_tenant_token(uuid4().hex)
+    management_token = ps.auth_keys.generate_management_token()
+
+    # this does not invoke auth check and only decodes jwt and checks it for validity
+    # check both tokens
+    ps.safe_psql("status", password=tenant_token)
+    ps.safe_psql("status", password=management_token)
+
+    # tenant can create branches
+    ps.safe_psql(f"branch_create {ps.initial_tenant} new1 main", password=tenant_token)
+    # console can create branches for tenant
+    ps.safe_psql(f"branch_create {ps.initial_tenant} new2 main", password=management_token)
+
+    # fail to create branch using token with different tenantid
+    with pytest.raises(psycopg2.DatabaseError, match='Tenant id mismatch. Permission denied'):
+        ps.safe_psql(f"branch_create {ps.initial_tenant} new2 main", password=invalid_tenant_token)
+
+    # create tenant using management token
+    ps.safe_psql(f"tenant_create {uuid4().hex}", password=management_token)
+
+    # fail to create tenant using tenant token
+    with pytest.raises(psycopg2.DatabaseError, match='Attempt to access management api with tenant scope. Permission denied'):
+        ps.safe_psql(f"tenant_create {uuid4().hex}", password=tenant_token)
+
+
+@pytest.mark.parametrize('with_wal_acceptors', [False, True])
+def test_compute_auth_to_pageserver(
+    zenith_cli: ZenithCli,
+    wa_factory,
+    pageserver_auth_enabled: ZenithPageserver,
+    repo_dir: str,
+    with_wal_acceptors: bool,
+    pg_bin: PgBin
+):
+    ps = pageserver_auth_enabled
+    # since we are in progress of refactoring protocols between compute safekeeper and page server
+    # use hardcoded management token in safekeeper
+    management_token = ps.auth_keys.generate_management_token()
+
+    branch = f"test_compute_auth_to_pageserver{with_wal_acceptors}"
+    zenith_cli.run(["branch", branch, "empty"])
+    if with_wal_acceptors:
+        wa_factory.start_n_new(3, management_token)
+
+    with Postgres(
+        zenith_cli=zenith_cli,
+        repo_dir=repo_dir,
+        pg_bin=pg_bin,
+        tenant_id=ps.initial_tenant,
+        port=55432, # FIXME port distribution is hardcoded in tests and in cli
+    ).create_start(
+        branch,
+        wal_acceptors=wa_factory.get_connstrs() if with_wal_acceptors else None,
+    ) as pg:
+        with closing(pg.connect()) as conn:
+            with conn.cursor() as cur:
+                # we rely upon autocommit after each statement
+                # as waiting for acceptors happens there
+                cur.execute('CREATE TABLE t(key int primary key, value text)')
+                cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
+                cur.execute('SELECT sum(key) FROM t')
+                assert cur.fetchone() == (5000050000, )

test_runner/batch_others/test_branch_behind.py

@@ -1,67 +1,76 @@
-import pytest
-import getpass
-import psycopg2
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
+
 #
 # Create a couple of branches off the main branch, at a historical point in time.
 #
-def test_branch_behind(zenith_cli, pageserver, postgres, pg_bin):
+def test_branch_behind(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
     # Branch at the point where only 100 rows were inserted
-    zenith_cli.run(["branch", "test_branch_behind", "empty"]);
+    zenith_cli.run(["branch", "test_branch_behind", "empty"])
 
     pgmain = postgres.create_start('test_branch_behind')
     print("postgres is running on 'test_branch_behind' branch")
 
-    main_pg_conn = psycopg2.connect(pgmain.connstr());
-    main_pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    main_pg_conn = pgmain.connect()
     main_cur = main_pg_conn.cursor()
 
     # Create table, and insert the first 100 rows
-    main_cur.execute('CREATE TABLE foo (t text)');
-    main_cur.execute("INSERT INTO foo SELECT 'long string to consume some space' || g FROM generate_series(1, 100) g");
-    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
+    main_cur.execute('CREATE TABLE foo (t text)')
+    main_cur.execute('''
+        INSERT INTO foo
+            SELECT 'long string to consume some space' || g
+            FROM generate_series(1, 100) g
+    ''')
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
     lsn_a = main_cur.fetchone()[0]
     print('LSN after 100 rows: ' + lsn_a)
 
     # Insert some more rows. (This generates enough WAL to fill a few segments.)
-    main_cur.execute("INSERT INTO foo SELECT 'long string to consume some space' || g FROM generate_series(1, 100000) g");
-    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
+    main_cur.execute('''
+        INSERT INTO foo
+            SELECT 'long string to consume some space' || g
+            FROM generate_series(1, 100000) g
+    ''')
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
     lsn_b = main_cur.fetchone()[0]
     print('LSN after 100100 rows: ' + lsn_b)
 
     # Branch at the point where only 100 rows were inserted
-    zenith_cli.run(["branch", "test_branch_behind_hundred", "test_branch_behind@"+lsn_a]);
+    zenith_cli.run(["branch", "test_branch_behind_hundred", "test_branch_behind@" + lsn_a])
 
     # Insert many more rows. This generates enough WAL to fill a few segments.
-    main_cur.execute("INSERT INTO foo SELECT 'long string to consume some space' || g FROM generate_series(1, 100000) g");
-    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
+    main_cur.execute('''
+        INSERT INTO foo
+            SELECT 'long string to consume some space' || g
+            FROM generate_series(1, 100000) g
+    ''')
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
 
-    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
     lsn_c = main_cur.fetchone()[0]
     print('LSN after 200100 rows: ' + lsn_c)
 
     # Branch at the point where only 200 rows were inserted
-    zenith_cli.run(["branch", "test_branch_behind_more", "test_branch_behind@"+lsn_b]);
+    zenith_cli.run(["branch", "test_branch_behind_more", "test_branch_behind@" + lsn_b])
 
     pg_hundred = postgres.create_start("test_branch_behind_hundred")
     pg_more = postgres.create_start("test_branch_behind_more")
 
     # On the 'hundred' branch, we should see only 100 rows
-    hundred_pg_conn = psycopg2.connect(pg_hundred.connstr())
-    hundred_pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    hundred_pg_conn = pg_hundred.connect()
     hundred_cur = hundred_pg_conn.cursor()
-    hundred_cur.execute('SELECT count(*) FROM foo');
-    assert(hundred_cur.fetchone()[0] == 100);
+    hundred_cur.execute('SELECT count(*) FROM foo')
+    assert hundred_cur.fetchone() == (100, )
 
     # On the 'more' branch, we should see 100200 rows
-    more_pg_conn = psycopg2.connect(pg_more.connstr())
-    more_pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    more_pg_conn = pg_more.connect()
     more_cur = more_pg_conn.cursor()
-    more_cur.execute('SELECT count(*) FROM foo');
-    assert(more_cur.fetchone()[0] == 100100);
+    more_cur.execute('SELECT count(*) FROM foo')
+    assert more_cur.fetchone() == (100100, )
 
     # All the rows are visible on the main branch
-    main_cur.execute('SELECT count(*) FROM foo');
-    assert(main_cur.fetchone()[0] == 200100);
+    main_cur.execute('SELECT count(*) FROM foo')
+    assert main_cur.fetchone() == (200100, )

test_runner/batch_others/test_clog_truncate.py

@@ -0,0 +1,72 @@
+import time
+import os
+
+from contextlib import closing
+
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+#
+# Test compute node start after clog truncation
+#
+def test_clog_truncate(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_clog_truncate", "empty"])
+
+    # set agressive autovacuum to make sure that truncation will happen
+    config = [
+        'autovacuum_max_workers=10', 'autovacuum_vacuum_threshold=0',
+        'autovacuum_vacuum_insert_threshold=0', 'autovacuum_vacuum_cost_delay=0',
+        'autovacuum_vacuum_cost_limit=10000', 'autovacuum_naptime =1s',
+        'autovacuum_freeze_max_age=100000'
+    ]
+
+    pg = postgres.create_start('test_clog_truncate', config_lines=config)
+    print('postgres is running on test_clog_truncate branch')
+
+    # Install extension containing function needed for test
+    pg.safe_psql('CREATE EXTENSION zenith_test_utils')
+
+    # Consume many xids to advance clog
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('select test_consume_xids(1000*1000*10);')
+            print('xids consumed')
+
+            # call a checkpoint to trigger TruncateSubtrans
+            cur.execute('CHECKPOINT;')
+
+            # ensure WAL flush
+            cur.execute('select txid_current()')
+            print(cur.fetchone())
+
+    # wait for autovacuum to truncate the pg_xact
+    # XXX Is it worth to add a timeout here?
+    pg_xact_0000_path = os.path.join(pg.pg_xact_dir_path(), '0000')
+    print("pg_xact_0000_path = " + pg_xact_0000_path)
+
+    while os.path.isfile(pg_xact_0000_path):
+        print("file exists. wait for truncation. " "pg_xact_0000_path = " + pg_xact_0000_path)
+        time.sleep(5)
+
+    # checkpoint to advance latest lsn
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('CHECKPOINT;')
+            cur.execute('select pg_current_wal_insert_lsn()')
+            lsn_after_truncation = cur.fetchone()[0]
+
+    # create new branch after clog truncation and start a compute node on it
+    print('create branch at lsn_after_truncation ' + lsn_after_truncation)
+    zenith_cli.run(
+        ["branch", "test_clog_truncate_new", "test_clog_truncate@" + lsn_after_truncation])
+
+    pg2 = postgres.create_start('test_clog_truncate_new')
+    print('postgres is running on test_clog_truncate_new branch')
+
+    # check that new node doesn't contain truncated segment
+    pg_xact_0000_path_new = os.path.join(pg2.pg_xact_dir_path(), '0000')
+    print("pg_xact_0000_path_new = " + pg_xact_0000_path_new)
+    assert os.path.isfile(pg_xact_0000_path_new) is False

test_runner/batch_others/test_config.py

@@ -1,7 +1,6 @@
-import pytest
-import os
-import getpass
-import psycopg2
+from contextlib import closing
+
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
@@ -9,22 +8,24 @@ pytest_plugins = ("fixtures.zenith_fixtures")
 #
 # Test starting Postgres with custom options
 #
-def test_config(zenith_cli, pageserver, postgres, pg_bin):
+def test_config(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
     # Create a branch for us
-    zenith_cli.run(["branch", "test_config", "empty"]);
+    zenith_cli.run(["branch", "test_config", "empty"])
 
     # change config
-    pg = postgres.create_start('test_config', ['log_min_messages=debug1'])
+    pg = postgres.create_start('test_config', config_lines=['log_min_messages=debug1'])
     print('postgres is running on test_config branch')
 
-    pg_conn = psycopg2.connect(pg.connstr())
-    pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
-    cur = pg_conn.cursor()
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('''
+                SELECT setting
+                FROM pg_settings
+                WHERE
+                    source != 'default'
+                    AND source != 'override'
+                    AND name = 'log_min_messages'
+            ''')
 
-    #check that config change was applied
-    cur.execute('SELECT name, setting from pg_settings WHERE source!=%s and source!=%s', ("default","override",))
-    for record in cur:
-        if record[0] == 'log_min_messages':
-            assert(record[1] == 'debug1')
-
-    pg_conn.close()
+            # check that config change was applied
+            assert cur.fetchone() == ('debug1', )

test_runner/batch_others/test_createdb.py

@@ -1,37 +0,0 @@
-import pytest
-import getpass
-import psycopg2
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-#
-# Test CREATE DATABASE when there have been relmapper changes
-#
-def test_createdb(zenith_cli, pageserver, postgres, pg_bin):
-    zenith_cli.run(["branch", "test_createdb", "empty"]);
-
-    pg = postgres.create_start('test_createdb')
-    print("postgres is running on 'test_createdb' branch")
-
-    conn = psycopg2.connect(pg.connstr());
-    conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
-    cur = conn.cursor()
-
-    # Cause a 'relmapper' change in the original branch
-    cur.execute('VACUUM FULL pg_class');
-
-    cur.execute('CREATE DATABASE foodb');
-
-    cur.execute('SELECT pg_current_wal_insert_lsn()');
-    lsn = cur.fetchone()[0]
-
-    conn.close();
-
-    # Create a branch
-    zenith_cli.run(["branch", "test_createdb2", "test_createdb@"+lsn]);
-
-    pg2 = postgres.create_start('test_createdb2')
-
-    # Test that you can connect to the new database on both branches
-    conn = psycopg2.connect(pg.connstr('foodb'));
-    conn2 = psycopg2.connect(pg2.connstr('foodb'));

test_runner/batch_others/test_createdropdb.py

@@ -0,0 +1,96 @@
+import os
+import pathlib
+
+from contextlib import closing
+from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory, ZenithCli
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+#
+# Test CREATE DATABASE when there have been relmapper changes
+#
+def test_createdb(
+    zenith_cli: ZenithCli,
+    pageserver: ZenithPageserver,
+    postgres: PostgresFactory,
+    pg_bin,
+):
+    zenith_cli.run(["branch", "test_createdb", "empty"])
+
+    pg = postgres.create_start('test_createdb')
+    print("postgres is running on 'test_createdb' branch")
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            # Cause a 'relmapper' change in the original branch
+            cur.execute('VACUUM FULL pg_class')
+
+            cur.execute('CREATE DATABASE foodb')
+
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn = cur.fetchone()[0]
+
+    # Create a branch
+    zenith_cli.run(["branch", "test_createdb2", "test_createdb@" + lsn])
+
+    pg2 = postgres.create_start('test_createdb2')
+
+    # Test that you can connect to the new database on both branches
+    for db in (pg, pg2):
+        db.connect(dbname='foodb').close()
+
+#
+# Test DROP DATABASE
+#
+def test_dropdb(
+    zenith_cli: ZenithCli,
+    pageserver: ZenithPageserver,
+    postgres: PostgresFactory,
+    pg_bin,
+):
+    zenith_cli.run(["branch", "test_dropdb", "empty"])
+
+    pg = postgres.create_start('test_dropdb')
+    print("postgres is running on 'test_dropdb' branch")
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('CREATE DATABASE foodb')
+
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn_before_drop = cur.fetchone()[0]
+
+            cur.execute("SELECT oid FROM pg_database WHERE datname='foodb';")
+            dboid = cur.fetchone()[0]
+
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('DROP DATABASE foodb')
+
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn_after_drop = cur.fetchone()[0]
+
+
+    # Create two branches before and after database drop.
+    zenith_cli.run(["branch", "test_before_dropdb", "test_dropdb@" + lsn_before_drop])
+    pg_before = postgres.create_start('test_before_dropdb')
+
+    zenith_cli.run(["branch", "test_after_dropdb", "test_dropdb@" + lsn_after_drop])
+    pg_after = postgres.create_start('test_after_dropdb')
+
+    # Test that database exists on the branch before drop
+    pg_before.connect(dbname='foodb').close()
+
+    # Test that database subdir exists on the branch before drop
+    dbpath = pathlib.Path(pg_before.pgdata_dir) / 'base' / str(dboid)
+    print(dbpath)
+
+    assert os.path.isdir(dbpath) == True
+
+    # Test that database subdir doesn't exist on the branch after drop
+    dbpath = pathlib.Path(pg_after.pgdata_dir) / 'base' / str(dboid)
+    print(dbpath)
+
+    assert os.path.isdir(dbpath) == False

test_runner/batch_others/test_createuser.py

@@ -0,0 +1,33 @@
+from contextlib import closing
+
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+#
+# Test CREATE USER to check shared catalog restore
+#
+def test_createuser(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+    zenith_cli.run(["branch", "test_createuser", "empty"])
+
+    pg = postgres.create_start('test_createuser')
+    print("postgres is running on 'test_createuser' branch")
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            # Cause a 'relmapper' change in the original branch
+            cur.execute('CREATE USER testuser with password %s', ('testpwd', ))
+
+            cur.execute('CHECKPOINT')
+
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn = cur.fetchone()[0]
+
+    # Create a branch
+    zenith_cli.run(["branch", "test_createuser2", "test_createuser@" + lsn])
+
+    pg2 = postgres.create_start('test_createuser2')
+
+    # Test that you can connect to new branch as a new user
+    assert pg2.safe_psql('select current_user', username='testuser') == [('testuser', )]

test_runner/batch_others/test_multixact.py

@@ -1,40 +1,39 @@
-import pytest
-import os
-import psycopg2
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
+
 #
 # Test multixact state after branching
 # Now this test is very minimalistic -
 # it only checks next_multixact_id field in restored pg_control,
 # since we don't have functions to check multixact internals.
 #
-def test_multixact(pageserver, postgres, pg_bin, zenith_cli, base_dir):
-
+def test_multixact(pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin, zenith_cli, base_dir):
     # Create a branch for us
     zenith_cli.run(["branch", "test_multixact", "empty"])
     pg = postgres.create_start('test_multixact')
 
     print("postgres is running on 'test_multixact' branch")
-    pg_conn = psycopg2.connect(pg.connstr())
-    pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    pg_conn = pg.connect()
     cur = pg_conn.cursor()
 
-    cur.execute('CREATE TABLE t1(i int primary key);'
-    'INSERT INTO t1 select * from generate_series(1,100);')
+    cur.execute('''
+        CREATE TABLE t1(i int primary key);
+        INSERT INTO t1 select * from generate_series(1, 100);
+    ''')
 
-    cur.execute('SELECT next_multixact_id FROM pg_control_checkpoint();')
+    cur.execute('SELECT next_multixact_id FROM pg_control_checkpoint()')
     next_multixact_id_old = cur.fetchone()[0]
 
     # Lock entries in parallel connections to set multixact
     nclients = 3
     connections = []
     for i in range(nclients):
-        con = psycopg2.connect(pg.connstr())
         # Do not turn on autocommit. We want to hold the key-share locks.
-        con.cursor().execute('select * from t1 for key share;')
-        connections.append(con)
+        conn = pg.connect(autocommit=False)
+        conn.cursor().execute('select * from t1 for key share')
+        connections.append(conn)
 
     # We should have a multixact now. We can close the connections.
     for c in connections:
@@ -43,25 +42,24 @@ def test_multixact(pageserver, postgres, pg_bin, zenith_cli, base_dir):
     # force wal flush
     cur.execute('checkpoint')
 
-    cur.execute('SELECT next_multixact_id, pg_current_wal_flush_lsn() FROM pg_control_checkpoint();')
+    cur.execute('SELECT next_multixact_id, pg_current_wal_flush_lsn() FROM pg_control_checkpoint()')
     res = cur.fetchone()
     next_multixact_id = res[0]
     lsn = res[1]
 
     # Ensure that we did lock some tuples
-    assert(int(next_multixact_id) > int(next_multixact_id_old))
+    assert int(next_multixact_id) > int(next_multixact_id_old)
 
     # Branch at this point
-    zenith_cli.run(["branch", "test_multixact_new", "test_multixact@"+lsn]);
+    zenith_cli.run(["branch", "test_multixact_new", "test_multixact@" + lsn])
     pg_new = postgres.create_start('test_multixact_new')
 
     print("postgres is running on 'test_multixact_new' branch")
-    pg_new_conn = psycopg2.connect(pg_new.connstr())
-    pg_new_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    pg_new_conn = pg_new.connect()
     cur_new = pg_new_conn.cursor()
 
-    cur_new.execute('SELECT next_multixact_id FROM pg_control_checkpoint();')
+    cur_new.execute('SELECT next_multixact_id FROM pg_control_checkpoint()')
     next_multixact_id_new = cur_new.fetchone()[0]
 
     # Check that we restored pg_controlfile correctly
-    assert(next_multixact_id_new == next_multixact_id)
+    assert next_multixact_id_new == next_multixact_id

test_runner/batch_others/test_pageserver_api.py

@@ -1,29 +1,28 @@
+import json
+from uuid import uuid4
 import pytest
 import psycopg2
-import getpass
-import json
+import requests
+from fixtures.zenith_fixtures import ZenithPageserver, ZenithPageserverHttpClient
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
-def test_status(pageserver):
-    pg_conn = psycopg2.connect(pageserver.connstr())
-    pg_conn.autocommit = True
-    cur = pg_conn.cursor()
-    cur.execute('status;')
-    assert cur.fetchone() == ('hello world',)
-    pg_conn.close()
 
-def test_branch_list(pageserver, zenith_cli):
+def test_status_psql(pageserver):
+    assert pageserver.safe_psql('status') == [
+        ('hello world', ),
+    ]
 
+
+def test_branch_list_psql(pageserver: ZenithPageserver, zenith_cli):
     # Create a branch for us
-    zenith_cli.run(["branch", "test_branch_list_main", "empty"]);
+    zenith_cli.run(["branch", "test_branch_list_main", "empty"])
 
-    page_server_conn = psycopg2.connect(pageserver.connstr())
-    page_server_conn.autocommit = True
-    page_server_cur = page_server_conn.cursor()
+    conn = pageserver.connect()
+    cur = conn.cursor()
 
-    page_server_cur.execute('branch_list;')
-    branches = json.loads(page_server_cur.fetchone()[0])
+    cur.execute(f'branch_list {pageserver.initial_tenant}')
+    branches = json.loads(cur.fetchone()[0])
     # Filter out branches created by other tests
     branches = [x for x in branches if x['name'].startswith('test_branch_list')]
 
@@ -38,8 +37,8 @@ def test_branch_list(pageserver, zenith_cli):
     zenith_cli.run(['branch', 'test_branch_list_experimental', 'test_branch_list_main'])
     zenith_cli.run(['pg', 'create', 'test_branch_list_experimental'])
 
-    page_server_cur.execute('branch_list;')
-    new_branches = json.loads(page_server_cur.fetchone()[0])
+    cur.execute(f'branch_list {pageserver.initial_tenant}')
+    new_branches = json.loads(cur.fetchone()[0])
     # Filter out branches created by other tests
     new_branches = [x for x in new_branches if x['name'].startswith('test_branch_list')]
     assert len(new_branches) == 2
@@ -51,4 +50,57 @@ def test_branch_list(pageserver, zenith_cli):
     # TODO: do the LSNs have to match here?
     assert new_branches[1] == branches[0]
 
-    page_server_conn.close()
+    conn.close()
+
+
+def test_tenant_list_psql(pageserver: ZenithPageserver, zenith_cli):
+    res = zenith_cli.run(["tenant", "list"])
+    res.check_returncode()
+    tenants = res.stdout.splitlines()
+    assert tenants == [pageserver.initial_tenant]
+
+    conn = pageserver.connect()
+    cur = conn.cursor()
+
+    # check same tenant cannot be created twice
+    with pytest.raises(psycopg2.DatabaseError, match=f'tenant {pageserver.initial_tenant} already exists'):
+        cur.execute(f'tenant_create {pageserver.initial_tenant}')
+
+    # create one more tenant
+    tenant1 = uuid4().hex
+    cur.execute(f'tenant_create {tenant1}')
+
+    cur.execute('tenant_list')
+
+    # compare tenants list
+    new_tenants = sorted(json.loads(cur.fetchone()[0]))
+    assert sorted([pageserver.initial_tenant, tenant1]) == new_tenants
+
+
+def check_client(client: ZenithPageserverHttpClient, initial_tenant: str):
+    client.check_status()
+
+    # check initial tenant is there
+    assert initial_tenant in set(client.tenant_list())
+
+    # create new tenant and check it is also there
+    tenant_id = uuid4()
+    client.tenant_create(tenant_id)
+    assert tenant_id.hex in set(client.tenant_list())
+
+    # create branch
+    branch_name = uuid4().hex
+    client.branch_create(tenant_id, branch_name, "main")
+
+    # check it is there
+    assert branch_name in {b['name'] for b in client.branch_list(tenant_id)}
+
+
+def test_pageserver_http_api_client(pageserver: ZenithPageserver):
+    client = pageserver.http_client()
+    check_client(client, pageserver.initial_tenant)
+
+
+def test_pageserver_http_api_client_auth_enabled(pageserver_auth_enabled: ZenithPageserver):
+    client = pageserver_auth_enabled.http_client(auth_token=pageserver_auth_enabled.auth_keys.generate_management_token())
+    check_client(client, pageserver_auth_enabled.initial_tenant)