imactive

Adds a warning if a postgres query fails, and some additional context to
errors generated inside `ReceiveWalConn::run`

.circleci/config.yml

@@ -7,7 +7,7 @@ executors:
   zenith-build-executor:
     resource_class: xlarge
     docker:
-      - image: cimg/rust:1.51.0
+      - image: cimg/rust:1.52.1
 
 jobs:
 
@@ -37,7 +37,7 @@ jobs:
           command: |
             if [ ! -e tmp_install/bin/postgres ]; then
               sudo apt update
-              sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libcurl4-openssl-dev
+              sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev
             fi
 
         # Build postgres if the restore_cache didn't find a build.
@@ -119,8 +119,7 @@ jobs:
             - target
 
         # Run rust unit tests
-        # FIXME: remove -p zenith_utils once integration tests are moved to python
-      - run: cargo test -p zenith_utils
+      - run: cargo test
 
         # Install the rust binaries, for use by test jobs
         # `--locked` is required; otherwise, `cargo install` will ignore Cargo.lock.
@@ -192,7 +191,12 @@ jobs:
           condition: << parameters.needs_postgres_source >>
           steps:
             - run: git submodule update --init --depth 1
-      - run: pip install pytest psycopg2
+      - run:
+          name: Install pipenv & deps
+          working_directory: test_runner
+          command: |
+            pip install pipenv
+            pipenv install
       - run:
           name: Run pytest
           working_directory: test_runner
@@ -211,25 +215,21 @@ jobs:
             #
             # The junit.xml file allows CircleCI to display more fine-grained test information
             # in its "Tests" tab in the results page.
-            pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short $TEST_SELECTION $EXTRA_PARAMS
+            # -s prevents pytest from capturing output, which helps to see
+            # what's going on if the test hangs
+            # --verbose prints name of each test (helpful when there are
+            # multiple tests in one file)
+            # -rA prints summary in the end
+            pipenv run pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short -s --verbose -rA $TEST_SELECTION $EXTRA_PARAMS
       - run:
           # CircleCI artifacts are preserved one file at a time, so skipping
           # this step isn't a good idea. If you want to extract the
           # pageserver state, perhaps a tarball would be a better idea.
-          name: Delete pageserver data
+          name: Delete all data but logs
           when: always
           command: |
             du -sh /tmp/test_output/*
-            for DIR in /tmp/test_output/*; do
-              mv $DIR/repo/pageserver.log $DIR/ || true # ignore errors
-              for PGDIR in $DIR/repo/pgdatadirs/pg?; do
-                echo "PGDIR: $PGDIR"
-                NEW_LOG="${PGDIR##*/}_log"
-                mv $PGDIR/log "$DIR/$NEW_LOG" || true # ignore errors
-              done
-              echo "rm $DIR/repo"
-              rm -rf $DIR/repo
-            done
+            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "wal_acceptor.log" ! -name "regression.diffs" -delete
             du -sh /tmp/test_output/*
       - store_artifacts:
           path: /tmp/test_output
@@ -237,6 +237,23 @@ jobs:
       - store_test_results:
           path: /tmp/test_output
 
+  # Build zenithdb/zenith:latest image and push it to Docker hub
+  docker-image:
+    docker:
+      - image: cimg/base:2021.04
+    steps:
+      - checkout
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - run:
+          name: Init postgres submodule
+          command: git submodule update --init --depth 1
+      - run:
+          name: Build and push Docker image
+          command: |
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
+            docker build -t zenithdb/zenith:latest . && docker push zenithdb/zenith:latest
+
 workflows:
   build_and_test:
     jobs:
@@ -265,3 +282,14 @@ workflows:
           test_selection: batch_others
           requires:
             - build-zenith-<< matrix.build_type >>
+      - docker-image:
+          # Context gives an ability to login
+          context: Docker Hub
+          # Build image only for commits to main
+          filters:
+            branches:
+              only:
+                - main
+          requires:
+            - pg_regress tests release
+            - other tests release

.dockerignore

@@ -0,0 +1,13 @@
+**/.git/
+**/__pycache__
+**/.pytest_cache
+
+/target
+/tmp_check
+/tmp_install
+/tmp_check_cli
+/test_output
+/.vscode
+/.zenith
+/integration_tests/.zenith
+/Dockerfile

.github/workflows/testing.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Install postgres dependencies
         run: |
           sudo apt update
-          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libcurl4-openssl-dev
+          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev
 
       - name: Set pg revision for caching
         id: pg_ver

CONTRIBUTING.md

@@ -0,0 +1,31 @@
+# How to contribute
+
+Howdy! Usual good software engineering practices apply. Write
+tests. Write comments. Follow standard Rust coding practices where
+possible. Use 'cargo fmt' and 'clippy' to tidy up formatting.
+
+There are soft spots in the code, which could use cleanup,
+refactoring, additional comments, and so forth. Let's try to raise the
+bar, and clean things up as we go. Try to leave code in a better shape
+than it was before.
+
+## Submitting changes
+
+1. Make a PR for every change.
+
+   Even seemingly trivial patches can break things in surprising ways.
+Use of common sense is OK. If you're only fixing a typo in a comment,
+it's probably fine to just push it. But if in doubt, open a PR.
+
+2. Get at least one +1 on your PR before you push.
+
+   For simple patches, it will only take a minute for someone to review
+it.
+
+3. Always keep the CI green.
+
+   Do not push, if the CI failed on your PR. Even if you think it's not
+your patch's fault. Help to fix the root cause if something else has
+broken the CI, before pushing.
+
+*Happy Hacking!*

COPYRIGHT

@@ -0,0 +1,20 @@
+This software is licensed under the Apache 2.0 License:
+
+----------------------------------------------------------------------------
+Copyright 2021 Zenith Labs, Inc
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+----------------------------------------------------------------------------
+
+The PostgreSQL submodule in vendor/postgres is licensed under the
+PostgreSQL license. See vendor/postgres/COPYRIGHT.

Cargo.toml

@@ -1,11 +1,17 @@
 [workspace]
 members = [
-    "integration_tests",
-    "pageserver",
-    "walkeeper",
-    "zenith",
     "control_plane",
+    "pageserver",
     "postgres_ffi",
-    "zenith_utils",
+    "proxy",
+    "walkeeper",
     "workspace_hack",
+    "zenith",
+    "zenith_metrics",
+    "zenith_utils",
 ]
+
+[profile.release]
+# This is useful for profiling and, to some extent, debug.
+# Besides, debug info should not affect the performance.
+debug = true

Dockerfile

@@ -0,0 +1,78 @@
+#
+# Docker image for console integration testing.
+#
+
+#
+# Build Postgres separately --- this layer will be rebuilt only if one of
+# mentioned paths will get any changes.
+#
+FROM zenithdb/build:buster AS pg-build
+WORKDIR /zenith
+COPY ./vendor/postgres vendor/postgres
+COPY ./Makefile Makefile
+RUN make -j $(getconf _NPROCESSORS_ONLN) -s postgres
+
+#
+# Calculate cargo dependencies.
+# This will always run, but only generate recipe.json with list of dependencies without
+# installing them.
+#
+FROM zenithdb/build:buster AS cargo-deps-inspect
+WORKDIR /zenith
+COPY . .
+RUN cargo chef prepare --recipe-path /zenith/recipe.json
+
+#
+# Build cargo dependencies.
+# This temp cantainner should be rebuilt only if recipe.json was changed.
+#
+FROM zenithdb/build:buster AS deps-build
+WORKDIR /zenith
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=cargo-deps-inspect /usr/local/cargo/bin/cargo-chef /usr/local/cargo/bin/
+COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
+RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json
+
+#
+# Build zenith binaries
+#
+FROM zenithdb/build:buster AS build
+WORKDIR /zenith
+COPY . .
+# Copy cached dependencies
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=deps-build /zenith/target target
+COPY --from=deps-build /usr/local/cargo/ /usr/local/cargo/
+RUN cargo build --release
+
+#
+# Copy binaries to resulting image.
+#
+FROM debian:buster-slim
+WORKDIR /data
+
+RUN apt-get update && apt-get -yq install librocksdb-dev libseccomp-dev openssl && \
+    mkdir zenith_install
+
+COPY --from=build /zenith/target/release/pageserver /usr/local/bin
+COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin
+COPY --from=build /zenith/target/release/proxy /usr/local/bin
+COPY --from=pg-build /zenith/tmp_install postgres_install
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+
+# Remove build artifacts (~ 500 MB)
+RUN rm -rf postgres_install/build && \
+    # 'Install' Postgres binaries locally
+    cp -r postgres_install/* /usr/local/ && \
+    # Prepare an archive of Postgres binaries (should be around 11 MB)
+    # and keep it inside container for an ease of deploy pipeline.
+    cd postgres_install && tar -czf /data/postgres_install.tar.gz . && cd .. && \
+    rm -rf postgres_install
+
+RUN useradd -m -d /data zenith
+
+VOLUME ["/data"]
+USER zenith
+EXPOSE 6400
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["pageserver"]

Dockerfile.alpine

@@ -0,0 +1,95 @@
+#
+# Docker image for console integration testing.
+#
+# We may also reuse it in CI to unify installation process and as a general binaries building
+# tool for production servers.
+#
+# Dynamic linking is used for librocksdb and libstdc++ bacause librocksdb-sys calls
+# bindgen with "dynamic" feature flag. This also prevents usage of dockerhub alpine-rust
+# images which are statically linked and have guards against any dlopen. I would rather
+# prefer all static binaries so we may change the way librocksdb-sys builds or wait until
+# we will have our own storage and drop rockdb dependency.
+#
+# Cargo-chef is used to separate dependencies building from main binaries building. This
+# way `docker build` will download and install dependencies only of there are changes to
+# out Cargo.toml files.
+#
+
+
+#
+# build postgres separately -- this layer will be rebuilt only if one of
+# mentioned paths will get any changes
+#
+FROM alpine:3.13 as pg-build
+RUN apk add --update clang llvm compiler-rt compiler-rt-static lld musl-dev binutils \
+                     make bison flex readline-dev zlib-dev perl linux-headers libseccomp-dev
+WORKDIR zenith
+COPY ./vendor/postgres vendor/postgres
+COPY ./Makefile Makefile
+# Build using clang and lld
+RUN CC='clang' LD='lld' CFLAGS='-fuse-ld=lld --rtlib=compiler-rt' make postgres -j4
+
+#
+# Calculate cargo dependencies.
+# This will always run, but only generate recipe.json with list of dependencies without
+# installing them.
+#
+FROM alpine:20210212 as cargo-deps-inspect
+RUN apk add --update rust cargo
+RUN cargo install cargo-chef
+WORKDIR zenith
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+#
+# Build cargo dependencies.
+# This temp cantainner would be build only if recipe.json was changed.
+#
+FROM alpine:20210212 as deps-build
+RUN apk add --update rust cargo openssl-dev clang build-base
+# rust-rocksdb can be built against system-wide rocksdb -- that saves about
+# 10 minutes during build. Rocksdb apk package is in testing now, but use it
+# anyway. In case of any troubles we can download and build rocksdb here manually
+# (to cache it as a docker layer).
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
+WORKDIR zenith
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=cargo-deps-inspect /root/.cargo/bin/cargo-chef /root/.cargo/bin/
+COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
+RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json
+
+#
+# Build zenith binaries
+#
+FROM alpine:20210212 as build
+RUN apk add --update rust cargo openssl-dev clang build-base
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
+WORKDIR zenith
+COPY . .
+# Copy cached dependencies
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=deps-build /zenith/target target
+COPY --from=deps-build /root/.cargo /root/.cargo
+RUN cargo build --release
+
+#
+# Copy binaries to resulting image.
+# build-base hare to provide libstdc++ (it will also bring gcc, but leave it this way until we figure
+# out how to statically link rocksdb or avoid it at all).
+#
+FROM alpine:3.13
+RUN apk add --update openssl build-base libseccomp-dev
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb
+COPY --from=build /zenith/target/release/pageserver /usr/local/bin
+COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin
+COPY --from=build /zenith/target/release/proxy /usr/local/bin
+COPY --from=pg-build /zenith/tmp_install /usr/local
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+
+RUN addgroup zenith && adduser -h /data -D -G zenith zenith
+VOLUME ["/data"]
+WORKDIR /data
+USER zenith
+EXPOSE 6400
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["pageserver"]

Dockerfile.build

@@ -0,0 +1,15 @@
+#
+# Image with all the required dependencies to build https://github.com/zenithdb/zenith
+# and Postgres from https://github.com/zenithdb/postgres
+# Also includes some rust development and build tools.
+#
+FROM rust:slim-buster
+WORKDIR /zenith
+
+# Install postgres and zenith build dependencies
+# clang is for rocksdb
+RUN apt-get update && apt-get -yq install automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \
+                                          libseccomp-dev pkg-config libssl-dev librocksdb-dev clang
+
+# Install rust tools
+RUN rustup component add clippy && cargo install cargo-chef cargo-audit

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -1,3 +1,11 @@
+# Seccomp BPF is only available for Linux
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Linux)
+	SECCOMP = --with-libseccomp
+else
+	SECCOMP =
+endif
+
 #
 # Top level Makefile to build Zenith and PostgreSQL
 #
@@ -21,8 +29,12 @@ tmp_install/build/config.status:
 	+@echo "Configuring postgres build"
 	mkdir -p tmp_install/build
 	(cd tmp_install/build && \
-	../../vendor/postgres/configure CFLAGS='-O0' --enable-debug --enable-cassert \
-	    --enable-depend --with-libxml --prefix=$(abspath tmp_install) > configure.log)
+	../../vendor/postgres/configure CFLAGS='-O0 -g3 $(CFLAGS)' \
+		--enable-cassert \
+		--enable-debug \
+		--enable-depend \
+		$(SECCOMP) \
+		--prefix=$(abspath tmp_install) > configure.log)
 
 # nicer alias for running 'configure'
 postgres-configure: tmp_install/build/config.status
@@ -38,8 +50,9 @@ postgres: postgres-configure
 	+@echo "Compiling PostgreSQL"
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 install
 	+@echo "Compiling contrib/zenith"
-	(cd vendor/postgres/contrib/zenith && \
-	$(MAKE) PG_CONFIG=$(abspath tmp_install)/bin/pg_config install USE_PGXS=1)
+	$(MAKE) -C tmp_install/build/contrib/zenith install
+	+@echo "Compiling contrib/zenith_test_utils"
+	$(MAKE) -C tmp_install/build/contrib/zenith_test_utils install
 
 postgres-clean:
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 clean

Pipfile

@@ -0,0 +1 @@
+./test_runner/Pipfile

Pipfile.lock

@@ -0,0 +1 @@
+./test_runner/Pipfile.lock

README.md

@@ -4,20 +4,34 @@ Zenith substitutes PostgreSQL storage layer and redistributes data across a clus
 
 ## Running local installation
 
-1. Build zenith and patched postgres
+1. Install build dependencies and other useful packages
+
+On Ubuntu or Debian this set of packages should be sufficient to build the code:
+```text
+apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libseccomp-dev \
+libssl-dev clang
+```
+
+[Rust] 1.52 or later is also required.
+
+To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively.
+
+To run the integration tests (not required to use the code), install
+Python (3.6 or higher), and install python3 packages with `pipenv` using `pipenv install` in the project directory.
+
+2. Build zenith and patched postgres
 ```sh
-git clone --recursive https://github.com/libzenith/zenith.git
+git clone --recursive https://github.com/zenithdb/zenith.git
 cd zenith
 make -j5
 ```
 
-2. Start pageserver and postgres on top of it (should be called from repo root):
+3. Start pageserver and postgres on top of it (should be called from repo root):
 ```sh
 # Create repository in .zenith with proper paths to binaries and data
 # Later that would be responsibility of a package install script
 > ./target/debug/zenith init
-<...>
-new zenith repository was created in .zenith
+pageserver init succeeded
 
 # start pageserver
 > ./target/debug/zenith start
@@ -35,8 +49,8 @@ BRANCH	ADDRESS		LSN		STATUS
 main	127.0.0.1:55432	0/1609610	running
 ```
 
-3. Now it is possible to connect to postgres and run some queries:
-```sh
+4. Now it is possible to connect to postgres and run some queries:
+```text
 > psql -p55432 -h 127.0.0.1 postgres
 postgres=# CREATE TABLE t(key int primary key, value text);
 CREATE TABLE
@@ -49,7 +63,7 @@ postgres=# select * from t;
 (1 row)
 ```
 
-4. And create branches and run postgres on them:
+5. And create branches and run postgres on them:
 ```sh
 # create branch named migration_check
 > ./target/debug/zenith branch migration_check main
@@ -69,7 +83,7 @@ waiting for server to start.... done
 # but all modifications would not affect data in original postgres
 > psql -p55433 -h 127.0.0.1 postgres
 postgres=# select * from t;
- key | value 
+ key | value
 -----+-------
    1 | 1
 (1 row)
@@ -83,33 +97,60 @@ INSERT 0 1
 ```sh
 git clone --recursive https://github.com/libzenith/zenith.git
 make # builds also postgres and installs it to ./tmp_install
+cd test_runner
 pytest
 ```
 
+## Documentation
+
+Now we use README files to cover design ideas and overall architecture for each module and `rustdoc` style documentation comments. See also [/docs/](/docs/) a top-level overview of all available markdown documentation.
+
+To view your `rustdoc` documentation in a browser, try running `cargo doc --no-deps --open`
+
 ## Source tree layout
 
-/walkeeper:
+`/control_plane`:
 
-WAL safekeeper. Written in Rust.
+Local control plane.
+Functions to start, configure and stop pageserver and postgres instances running as a local processes.
+Intended to be used in integration tests and in CLI tools for local installations.
 
-/pageserver:
+`/zenith`
+
+Main entry point for the 'zenith' CLI utility.
+TODO: Doesn't it belong to control_plane?
+
+`/postgres_ffi`:
+
+Utility functions for interacting with PostgreSQL file formats.
+Misc constants, copied from PostgreSQL headers.
+
+`/zenith_utils`:
+
+Helpers that are shared between other crates in this repository.
+
+`/walkeeper`:
+
+WAL safekeeper (also known as WAL acceptor). Written in Rust.
+
+`/pageserver`:
 
 Page Server. Written in Rust.
 
 Depends on the modified 'postgres' binary for WAL redo.
 
-/integration_tests:
-
-Tests with different combinations of a Postgres compute node, WAL safekeeper and Page Server.
-
-/vendor/postgres:
+`/vendor/postgres`:
 
 PostgreSQL source tree, with the modifications needed for Zenith.
 
-/vendor/postgres/src/bin/safekeeper:
+`/vendor/postgres/contrib/zenith`:
 
-Extension (safekeeper_proxy) that runs in the compute node, and connects to the WAL safekeepers
-and streams the WAL
+PostgreSQL extension that implements storage manager API and network communications with remote page server.
 
+`/test_runner`:
 
+Integration tests, written in Python using the `pytest` framework.
 
+`test_runner/zenith_regress`:
+
+Quick way to add new SQL regression test to integration tests set.

control_plane/Cargo.toml

@@ -16,12 +16,10 @@ toml = "0.5"
 lazy_static = "1.4"
 regex = "1"
 anyhow = "1.0"
-# hex = "0.4.3"
 bytes = "1.0.1"
-# fs_extra = "1.2.0"
 nix = "0.20"
-# thiserror = "1"
 url = "2.2.2"
+hex = { version = "0.4.3", features = ["serde"] }
 
 pageserver = { path = "../pageserver" }
 walkeeper = { path = "../walkeeper" }

control_plane/src/compute.rs

@@ -7,17 +7,19 @@ use std::sync::Arc;
 use std::time::Duration;
 use std::{collections::BTreeMap, path::PathBuf};
 use std::{
-    fs::{self, OpenOptions},
+    fs::{self, File, OpenOptions},
     io::Read,
 };
 
 use anyhow::{Context, Result};
 use lazy_static::lazy_static;
 use regex::Regex;
+use zenith_utils::connstring::connection_host_port;
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::ZTenantId;
+use zenith_utils::zid::ZTimelineId;
 
 use crate::local_env::LocalEnv;
-use pageserver::ZTimelineId;
-
 use crate::storage::PageServerNode;
 
 //
@@ -26,27 +28,36 @@ use crate::storage::PageServerNode;
 pub struct ComputeControlPlane {
     base_port: u16,
     pageserver: Arc<PageServerNode>,
-    pub nodes: BTreeMap<String, Arc<PostgresNode>>,
+    pub nodes: BTreeMap<(ZTenantId, String), Arc<PostgresNode>>,
     env: LocalEnv,
 }
 
 impl ComputeControlPlane {
     // Load current nodes with ports from data directories on disk
+    // Directory structure has the following layout:
+    // pgdatadirs
+    // |- tenants
+    // |  |- <tenant_id>
+    // |  |   |- <branch name>
     pub fn load(env: LocalEnv) -> Result<ComputeControlPlane> {
         // TODO: since pageserver do not have config file yet we believe here that
         // it is running on default port. Change that when pageserver will have config.
         let pageserver = Arc::new(PageServerNode::from_env(&env));
 
+        let mut nodes = BTreeMap::default();
         let pgdatadirspath = &env.pg_data_dirs_path();
-        let nodes: Result<BTreeMap<_, _>> = fs::read_dir(&pgdatadirspath)
+
+        for tenant_dir in fs::read_dir(&pgdatadirspath)
             .with_context(|| format!("failed to list {}", pgdatadirspath.display()))?
-            .into_iter()
-            .map(|f| {
-                PostgresNode::from_dir_entry(f?, &env, &pageserver)
-                    .map(|node| (node.name.clone(), Arc::new(node)))
-            })
-            .collect();
-        let nodes = nodes?;
+        {
+            let tenant_dir = tenant_dir?;
+            for timeline_dir in fs::read_dir(tenant_dir.path())
+                .with_context(|| format!("failed to list {}", tenant_dir.path().display()))?
+            {
+                let node = PostgresNode::from_dir_entry(timeline_dir?, &env, &pageserver)?;
+                nodes.insert((node.tenantid, node.name.clone()), Arc::new(node));
+            }
+        }
 
         Ok(ComputeControlPlane {
             base_port: 55431,
@@ -74,86 +85,40 @@ impl ComputeControlPlane {
         }
     }
 
-    /// Connect to a page server, get base backup, and untar it to initialize a
-    /// new data directory
-    pub fn new_from_page_server(
+    pub fn new_node(
         &mut self,
-        is_test: bool,
-        timelineid: ZTimelineId,
-        name: &str,
+        tenantid: ZTenantId,
+        branch_name: &str,
+        config_only: bool,
     ) -> Result<Arc<PostgresNode>> {
+        let timeline_id = self
+            .pageserver
+            .branch_get_by_name(&tenantid, branch_name)?
+            .timeline_id;
+
         let node = Arc::new(PostgresNode {
-            name: name.to_owned(),
+            name: branch_name.to_owned(),
             address: SocketAddr::new("127.0.0.1".parse().unwrap(), self.get_port()),
             env: self.env.clone(),
             pageserver: Arc::clone(&self.pageserver),
-            is_test,
-            timelineid,
+            is_test: false,
+            timelineid: timeline_id,
+            tenantid,
         });
 
-        node.init_from_page_server()?;
-        self.nodes.insert(node.name.clone(), Arc::clone(&node));
-
-        Ok(node)
-    }
-
-    pub fn new_test_node(&mut self, branch_name: &str) -> Arc<PostgresNode> {
-        let timeline_id = self
-            .pageserver
-            .branch_get_by_name(branch_name)
-            .expect("failed to get timeline_id")
-            .timeline_id;
-
-        let node = self.new_from_page_server(true, timeline_id, branch_name);
-        let node = node.unwrap();
+        node.init_from_page_server(self.env.auth_type, config_only)?;
+        self.nodes
+            .insert((tenantid, node.name.clone()), Arc::clone(&node));
 
         // Configure the node to stream WAL directly to the pageserver
         node.append_conf(
             "postgresql.conf",
             format!(
-                "shared_preload_libraries = zenith\n\
-                zenith.callmemaybe_connstring = '{}'\n", // FIXME escaping
-                node.connstr()
-            )
-            .as_str(),
-        )
-        .unwrap();
-
-        node
-    }
-
-    pub fn new_test_master_node(&mut self, branch_name: &str) -> Arc<PostgresNode> {
-        let timeline_id = self
-            .pageserver
-            .branch_get_by_name(branch_name)
-            .expect("failed to get timeline_id")
-            .timeline_id;
-
-        let node = self
-            .new_from_page_server(true, timeline_id, branch_name)
-            .unwrap();
-
-        node.append_conf(
-            "postgresql.conf",
-            "synchronous_standby_names = 'safekeeper_proxy'\n",
-        )
-        .unwrap();
-
-        node
-    }
-
-    pub fn new_node(&mut self, branch_name: &str) -> Result<Arc<PostgresNode>> {
-        let timeline_id = self.pageserver.branch_get_by_name(branch_name)?.timeline_id;
-
-        let node = self.new_from_page_server(false, timeline_id, branch_name)?;
-
-        // Configure the node to stream WAL directly to the pageserver
-        node.append_conf(
-            "postgresql.conf",
-            format!(
-                "shared_preload_libraries = zenith\n\
-                zenith.callmemaybe_connstring = '{}'\n", // FIXME escaping
-                node.connstr()
+                concat!(
+                    "synchronous_standby_names = 'pageserver'\n", // TODO: add a new function arg?
+                    "zenith.callmemaybe_connstring = '{}'\n",     // FIXME escaping
+                ),
+                node.connstr(),
             )
             .as_str(),
         )?;
@@ -164,6 +129,7 @@ impl ComputeControlPlane {
 
 ///////////////////////////////////////////////////////////////////////////////
 
+#[derive(Debug)]
 pub struct PostgresNode {
     pub address: SocketAddr,
     name: String,
@@ -171,6 +137,7 @@ pub struct PostgresNode {
     pageserver: Arc<PageServerNode>,
     is_test: bool,
     pub timelineid: ZTimelineId,
+    pub tenantid: ZTenantId,
 }
 
 impl PostgresNode {
@@ -190,6 +157,8 @@ impl PostgresNode {
             static ref CONF_PORT_RE: Regex = Regex::new(r"(?m)^\s*port\s*=\s*(\d+)\s*$").unwrap();
             static ref CONF_TIMELINE_RE: Regex =
                 Regex::new(r"(?m)^\s*zenith.zenith_timeline\s*=\s*'(\w+)'\s*$").unwrap();
+            static ref CONF_TENANT_RE: Regex =
+                Regex::new(r"(?m)^\s*zenith.zenith_tenant\s*=\s*'(\w+)'\s*$").unwrap();
         }
 
         // parse data directory name
@@ -237,6 +206,22 @@ impl PostgresNode {
             .parse()
             .with_context(|| err_msg)?;
 
+        // parse tenant
+        let err_msg = format!(
+            "failed to find tenant definition in config file {}",
+            cfg_path.to_str().unwrap()
+        );
+        let tenantid = CONF_TENANT_RE
+            .captures(config.as_str())
+            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 1"))?
+            .iter()
+            .last()
+            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 2"))?
+            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 3"))?
+            .as_str()
+            .parse()
+            .with_context(|| err_msg)?;
+
         // ok now
         Ok(PostgresNode {
             address: SocketAddr::new("127.0.0.1".parse().unwrap(), port),
@@ -245,46 +230,19 @@ impl PostgresNode {
             pageserver: Arc::clone(pageserver),
             is_test: false,
             timelineid,
+            tenantid,
         })
     }
 
-    // Connect to a page server, get base backup, and untar it to initialize a
-    // new data directory
-    pub fn init_from_page_server(&self) -> Result<()> {
+    pub fn do_basebackup(&self) -> Result<()> {
         let pgdata = self.pgdata();
-        println!(
-            "Extracting base backup to create postgres instance: path={} port={}",
-            pgdata.display(),
-            self.address.port()
-        );
 
-        // initialize data directory
-        if self.is_test {
-            fs::remove_dir_all(&pgdata).ok();
-        }
-
-        let sql = format!("basebackup {}", self.timelineid);
+        let sql = format!("basebackup {} {}", self.tenantid, self.timelineid);
         let mut client = self
             .pageserver
             .page_server_psql_client()
             .with_context(|| "connecting to page server failed")?;
 
-        fs::create_dir_all(&pgdata)
-            .with_context(|| format!("could not create data directory {}", pgdata.display()))?;
-        fs::set_permissions(pgdata.as_path(), fs::Permissions::from_mode(0o700)).with_context(
-            || {
-                format!(
-                    "could not set permissions in data directory {}",
-                    pgdata.display()
-                )
-            },
-        )?;
-
-        // FIXME: The compute node should be able to stream the WAL it needs from the WAL safekeepers or archive.
-        // But that's not implemented yet. For now, 'pg_wal' is included in the base backup tarball that
-        // we receive from the Page Server, so we don't need to create the empty 'pg_wal' directory here.
-        //fs::create_dir_all(pgdata.join("pg_wal"))?;
-
         let mut copyreader = client
             .copy_out(sql.as_str())
             .with_context(|| "page server 'basebackup' command failed")?;
@@ -300,11 +258,52 @@ impl PostgresNode {
         ar.unpack(&pgdata)
             .with_context(|| "extracting page backup failed")?;
 
-        // listen for selected port
+        Ok(())
+    }
+
+    // Connect to a page server, get base backup, and untar it to initialize a
+    // new data directory
+    pub fn init_from_page_server(&self, auth_type: AuthType, config_only: bool) -> Result<()> {
+        let pgdata = self.pgdata();
+
+        println!(
+            "Extracting base backup to create postgres instance: path={} port={}",
+            pgdata.display(),
+            self.address.port()
+        );
+
+        // initialize data directory
+        if self.is_test {
+            fs::remove_dir_all(&pgdata).ok();
+        }
+
+        fs::create_dir_all(&pgdata)
+            .with_context(|| format!("could not create data directory {}", pgdata.display()))?;
+        fs::set_permissions(pgdata.as_path(), fs::Permissions::from_mode(0o700)).with_context(
+            || {
+                format!(
+                    "could not set permissions in data directory {}",
+                    pgdata.display()
+                )
+            },
+        )?;
+
+        if config_only {
+            //Just create an empty config file
+            File::create(self.pgdata().join("postgresql.conf").to_str().unwrap())?;
+        } else {
+            self.do_basebackup()?;
+            fs::create_dir_all(self.pgdata().join("pg_wal"))?;
+            fs::create_dir_all(self.pgdata().join("pg_wal").join("archive_status"))?;
+        }
+
+        // wal_log_hints is mandatory when running against pageserver (see gh issue#192)
+        // TODO: is it possible to check wal_log_hints at pageserver side via XLOG_PARAMETER_CHANGE?
         self.append_conf(
             "postgresql.conf",
             &format!(
                 "max_wal_senders = 10\n\
+                 wal_log_hints = on\n\
                  max_replication_slots = 10\n\
                  hot_standby = on\n\
                  shared_buffers = 1MB\n\
@@ -321,33 +320,40 @@ impl PostgresNode {
 
         // Never clean up old WAL. TODO: We should use a replication
         // slot or something proper, to prevent the compute node
-        // from removing WAL that hasn't been streamed to the safekeepr or
-        // page server yet. But this will do for now.
+        // from removing WAL that hasn't been streamed to the safekeeper or
+        // page server yet. (gh issue #349)
         self.append_conf("postgresql.conf", "wal_keep_size='10TB'\n")?;
 
-        // Connect it to the page server.
+        // set up authentication
+        let password = if let AuthType::ZenithJWT = auth_type {
+            "$ZENITH_AUTH_TOKEN"
+        } else {
+            ""
+        };
 
         // Configure that node to take pages from pageserver
+        let (host, port) = connection_host_port(&self.pageserver.connection_config);
         self.append_conf(
             "postgresql.conf",
-            &format!(
-                "shared_preload_libraries = zenith \n\
-                 zenith.page_server_connstring = 'host={} port={}'\n\
-                 zenith.zenith_timeline='{}'\n",
-                self.pageserver.address().ip(),
-                self.pageserver.address().port(),
-                self.timelineid
-            ),
+            format!(
+                concat!(
+                    "shared_preload_libraries = zenith\n",
+                    // $ZENITH_AUTH_TOKEN will be replaced with value from environment variable during compute pg startup
+                    // it is done this way because otherwise user will be able to retrieve the value using SHOW command or pg_settings
+                    "zenith.page_server_connstring = 'host={} port={} password={}'\n",
+                    "zenith.zenith_timeline='{}'\n",
+                    "zenith.zenith_tenant='{}'\n",
+                ),
+                host, port, password, self.timelineid, self.tenantid,
+            )
+            .as_str(),
         )?;
 
-        fs::create_dir_all(self.pgdata().join("pg_wal"))?;
-        fs::create_dir_all(self.pgdata().join("pg_wal").join("archive_status"))?;
-        self.pg_resetwal(&["-f"])?;
         Ok(())
     }
 
     pub fn pgdata(&self) -> PathBuf {
-        self.env.pg_data_dir(&self.name)
+        self.env.pg_data_dir(&self.tenantid, &self.name)
     }
 
     pub fn status(&self) -> &str {
@@ -371,57 +377,88 @@ impl PostgresNode {
         Ok(())
     }
 
-    fn pg_ctl(&self, args: &[&str]) -> Result<()> {
+    fn pg_ctl(&self, args: &[&str], auth_token: &Option<String>) -> Result<()> {
         let pg_ctl_path = self.env.pg_bin_dir().join("pg_ctl");
+        let mut cmd = Command::new(pg_ctl_path);
+        cmd.args(
+            [
+                &[
+                    "-D",
+                    self.pgdata().to_str().unwrap(),
+                    "-l",
+                    self.pgdata().join("pg.log").to_str().unwrap(),
+                    "-w", //wait till pg_ctl actually does what was asked
+                ],
+                args,
+            ]
+            .concat(),
+        )
+        .env_clear()
+        .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
+        .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap());
+
+        if let Some(token) = auth_token {
+            cmd.env("ZENITH_AUTH_TOKEN", token);
+        }
+        let pg_ctl = cmd.status().with_context(|| "pg_ctl failed")?;
 
-        let pg_ctl = Command::new(pg_ctl_path)
-            .args(
-                [
-                    &[
-                        "-D",
-                        self.pgdata().to_str().unwrap(),
-                        "-l",
-                        self.pgdata().join("log").to_str().unwrap(),
-                    ],
-                    args,
-                ]
-                .concat(),
-            )
-            .env_clear()
-            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .status()
-            .with_context(|| "pg_ctl failed")?;
         if !pg_ctl.success() {
             anyhow::bail!("pg_ctl failed");
         }
         Ok(())
     }
 
-    fn pg_resetwal(&self, args: &[&str]) -> Result<()> {
-        let pg_resetwal_path = self.env.pg_bin_dir().join("pg_resetwal");
-
-        let pg_ctl = Command::new(pg_resetwal_path)
-            .args([&["-D", self.pgdata().to_str().unwrap()], args].concat())
-            .status()
-            .with_context(|| "pg_resetwal failed")?;
-        if !pg_ctl.success() {
-            anyhow::bail!("pg_resetwal failed");
+    pub fn start(&self, auth_token: &Option<String>) -> Result<()> {
+        // Bail if the node already running.
+        if self.status() == "running" {
+            anyhow::bail!("The node is already running");
         }
-        Ok(())
-    }
 
-    pub fn start(&self) -> Result<()> {
+        // 1. We always start compute node from scratch, so
+        // if old dir exists, preserve config files and drop the directory
+
+        // XXX Now we only use 'postgresql.conf'.
+        // If we will need 'pg_hba.conf', support it here too
+
+        let postgresql_conf_path = self.pgdata().join("postgresql.conf");
+        let postgresql_conf = fs::read(postgresql_conf_path.clone()).with_context(|| {
+            format!(
+                "failed to read config file in {}",
+                postgresql_conf_path.to_str().unwrap()
+            )
+        })?;
+
+        println!(
+            "Destroying postgres data directory '{}'",
+            self.pgdata().to_str().unwrap()
+        );
+        fs::remove_dir_all(&self.pgdata())?;
+
+        // 2. Create new node
+        self.init_from_page_server(self.env.auth_type, false)?;
+
+        // 3. Bring back config files
+
+        if let Ok(mut file) = OpenOptions::new()
+            .append(false)
+            .write(true)
+            .open(&postgresql_conf_path)
+        {
+            file.write_all(&postgresql_conf)?;
+            file.sync_all()?;
+        }
+
+        // 4. Finally start the compute node postgres
         println!("Starting postgres node at '{}'", self.connstr());
-        self.pg_ctl(&["start"])
+        self.pg_ctl(&["start"], auth_token)
     }
 
-    pub fn restart(&self) -> Result<()> {
-        self.pg_ctl(&["restart"])
+    pub fn restart(&self, auth_token: &Option<String>) -> Result<()> {
+        self.pg_ctl(&["restart"], auth_token)
     }
 
     pub fn stop(&self, destroy: bool) -> Result<()> {
-        self.pg_ctl(&["-m", "immediate", "stop"])?;
+        self.pg_ctl(&["-m", "immediate", "stop"], &None)?;
         if destroy {
             println!(
                 "Destroying postgres data directory '{}'",
@@ -434,10 +471,11 @@ impl PostgresNode {
 
     pub fn connstr(&self) -> String {
         format!(
-            "host={} port={} user={}",
+            "host={} port={} user={} dbname={}",
             self.address.ip(),
             self.address.port(),
-            self.whoami()
+            "zenith_admin",
+            "postgres"
         )
     }
 

control_plane/src/lib.rs

@@ -1,7 +1,7 @@
 //
 // Local control plane.
 //
-// Can start, cofigure and stop postgres instances running as a local processes.
+// Can start, configure and stop postgres instances running as a local processes.
 //
 // Intended to be used in integration tests and in CLI tools for
 // local installations.

control_plane/src/local_env.rs

@@ -4,19 +4,24 @@
 // Now it also provides init method which acts like a stub for proper installation
 // script which will use local paths.
 //
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, Context, Result};
+use hex;
 use serde::{Deserialize, Serialize};
 use std::fs;
 use std::path::PathBuf;
+use std::process::{Command, Stdio};
 use std::{collections::BTreeMap, env};
 use url::Url;
+use zenith_utils::auth::{encode_from_key_path, Claims, Scope};
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::ZTenantId;
 
 pub type Remotes = BTreeMap<String, String>;
 
 //
 // This data structures represent deserialized zenith CLI config
 //
-#[derive(Serialize, Deserialize, Clone)]
+#[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct LocalEnv {
     // Pageserver connection strings
     pub pageserver_connstring: String,
@@ -33,6 +38,22 @@ pub struct LocalEnv {
     // Path to pageserver binary. Empty for remote pageserver.
     pub zenith_distrib_dir: Option<PathBuf>,
 
+    // keeping tenant id in config to reduce copy paste when running zenith locally with single tenant
+    #[serde(with = "hex")]
+    pub tenantid: ZTenantId,
+
+    // Repository format, 'rocksdb' or 'layered' or None for default
+    pub repository_format: Option<String>,
+
+    // jwt auth token used for communication with pageserver
+    pub auth_token: String,
+
+    // used to determine which auth type is used
+    pub auth_type: AuthType,
+
+    // used to issue tokens during e.g pg start
+    pub private_key_path: PathBuf,
+
     pub remotes: Remotes,
 }
 
@@ -49,16 +70,18 @@ impl LocalEnv {
         Ok(self
             .zenith_distrib_dir
             .as_ref()
-            .ok_or(anyhow!("Can not manage remote pageserver"))?
+            .ok_or_else(|| anyhow!("Can not manage remote pageserver"))?
             .join("pageserver"))
     }
 
     pub fn pg_data_dirs_path(&self) -> PathBuf {
-        self.base_data_dir.join("pgdatadirs")
+        self.base_data_dir.join("pgdatadirs").join("tenants")
     }
 
-    pub fn pg_data_dir(&self, name: &str) -> PathBuf {
-        self.pg_data_dirs_path().join(name)
+    pub fn pg_data_dir(&self, tenantid: &ZTenantId, branch_name: &str) -> PathBuf {
+        self.pg_data_dirs_path()
+            .join(tenantid.to_string())
+            .join(branch_name)
     }
 
     // TODO: move pageserver files into ./pageserver
@@ -77,7 +100,12 @@ fn base_path() -> PathBuf {
 //
 // Initialize a new Zenith repository
 //
-pub fn init(remote_pageserver: Option<&str>) -> Result<()> {
+pub fn init(
+    remote_pageserver: Option<&str>,
+    tenantid: ZTenantId,
+    auth_type: AuthType,
+    repository_format: Option<&str>,
+) -> Result<()> {
     // check if config already exists
     let base_path = base_path();
     if base_path.exists() {
@@ -86,6 +114,7 @@ pub fn init(remote_pageserver: Option<&str>) -> Result<()> {
             base_path.to_str().unwrap()
         );
     }
+    fs::create_dir(&base_path)?;
 
     // ok, now check that expected binaries are present
 
@@ -102,8 +131,43 @@ pub fn init(remote_pageserver: Option<&str>) -> Result<()> {
         anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
     }
 
-    fs::create_dir(&base_path)?;
-    fs::create_dir(base_path.join("pgdatadirs"))?;
+    // generate keys for jwt
+    // openssl genrsa -out private_key.pem 2048
+    let private_key_path = base_path.join("auth_private_key.pem");
+    let keygen_output = Command::new("openssl")
+        .arg("genrsa")
+        .args(&["-out", private_key_path.to_str().unwrap()])
+        .arg("2048")
+        .stdout(Stdio::null())
+        .output()
+        .with_context(|| "failed to generate auth private key")?;
+    if !keygen_output.status.success() {
+        anyhow::bail!(
+            "openssl failed: '{}'",
+            String::from_utf8_lossy(&keygen_output.stderr)
+        );
+    }
+
+    let public_key_path = base_path.join("auth_public_key.pem");
+    // openssl rsa -in private_key.pem -pubout -outform PEM -out public_key.pem
+    let keygen_output = Command::new("openssl")
+        .arg("rsa")
+        .args(&["-in", private_key_path.to_str().unwrap()])
+        .arg("-pubout")
+        .args(&["-outform", "PEM"])
+        .args(&["-out", public_key_path.to_str().unwrap()])
+        .stdout(Stdio::null())
+        .output()
+        .with_context(|| "failed to generate auth private key")?;
+    if !keygen_output.status.success() {
+        anyhow::bail!(
+            "openssl failed: '{}'",
+            String::from_utf8_lossy(&keygen_output.stderr)
+        );
+    }
+
+    let auth_token =
+        encode_from_key_path(&Claims::new(None, Scope::PageServerApi), &private_key_path)?;
 
     let conf = if let Some(addr) = remote_pageserver {
         // check that addr is parsable
@@ -115,6 +179,11 @@ pub fn init(remote_pageserver: Option<&str>) -> Result<()> {
             zenith_distrib_dir: None,
             base_data_dir: base_path,
             remotes: BTreeMap::default(),
+            tenantid,
+            repository_format: repository_format.map(|x| x.into()),
+            auth_token,
+            auth_type,
+            private_key_path,
         }
     } else {
         // Find zenith binaries.
@@ -129,9 +198,16 @@ pub fn init(remote_pageserver: Option<&str>) -> Result<()> {
             zenith_distrib_dir: Some(zenith_distrib_dir),
             base_data_dir: base_path,
             remotes: BTreeMap::default(),
+            tenantid,
+            repository_format: repository_format.map(|x| x.into()),
+            auth_token,
+            auth_type,
+            private_key_path,
         }
     };
 
+    fs::create_dir_all(conf.pg_data_dirs_path())?;
+
     let toml = toml::to_string_pretty(&conf)?;
     fs::write(conf.base_data_dir.join("config"), toml)?;
 

control_plane/src/storage.rs

@@ -1,5 +1,5 @@
 use std::collections::HashMap;
-use std::net::{SocketAddr, TcpStream};
+use std::net::TcpStream;
 use std::path::PathBuf;
 use std::process::Command;
 use std::thread;
@@ -8,53 +8,77 @@ use std::time::Duration;
 use anyhow::{anyhow, bail, Result};
 use nix::sys::signal::{kill, Signal};
 use nix::unistd::Pid;
-use postgres::{Client, NoTls};
+use postgres::{Config, NoTls};
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::ZTenantId;
 
 use crate::local_env::LocalEnv;
 use crate::read_pidfile;
 use pageserver::branches::BranchInfo;
+use zenith_utils::connstring::connection_address;
 
 //
 // Control routines for pageserver.
 //
 // Used in CLI and tests.
 //
+#[derive(Debug)]
 pub struct PageServerNode {
     pub kill_on_exit: bool,
-    pub listen_address: Option<SocketAddr>,
+    pub connection_config: Config,
     pub env: LocalEnv,
 }
 
 impl PageServerNode {
     pub fn from_env(env: &LocalEnv) -> PageServerNode {
+        let password = if matches!(env.auth_type, AuthType::ZenithJWT) {
+            &env.auth_token
+        } else {
+            ""
+        };
+
         PageServerNode {
             kill_on_exit: false,
-            listen_address: None, // default
+            connection_config: Self::default_config(password), // default
             env: env.clone(),
         }
     }
 
-    pub fn address(&self) -> SocketAddr {
-        match self.listen_address {
-            Some(addr) => addr,
-            None => "127.0.0.1:64000".parse().unwrap(),
-        }
+    fn default_config(password: &str) -> Config {
+        format!("postgresql://no_user:{}@localhost:64000/no_db", password)
+            .parse()
+            .unwrap()
     }
 
-    pub fn init(&self) -> Result<()> {
+    pub fn init(
+        &self,
+        create_tenant: Option<&str>,
+        enable_auth: bool,
+        repository_format: Option<&str>,
+    ) -> Result<()> {
         let mut cmd = Command::new(self.env.pageserver_bin()?);
+        let mut args = vec![
+            "--init",
+            "-D",
+            self.env.base_data_dir.to_str().unwrap(),
+            "--postgres-distrib",
+            self.env.pg_distrib_dir.to_str().unwrap(),
+        ];
+
+        if enable_auth {
+            args.extend(&["--auth-validation-public-key-path", "auth_public_key.pem"]);
+            args.extend(&["--auth-type", "ZenithJWT"]);
+        }
+
+        if let Some(repo_format) = repository_format {
+            args.extend(&["--repository-format", repo_format]);
+        }
+
+        create_tenant.map(|tenantid| args.extend(&["--create-tenant", tenantid]));
         let status = cmd
-            .args(&["--init", "-D", self.env.base_data_dir.to_str().unwrap()])
+            .args(args)
             .env_clear()
             .env("RUST_BACKTRACE", "1")
-            .env(
-                "POSTGRES_DISTRIB_DIR",
-                self.env.pg_distrib_dir.to_str().unwrap(),
-            )
-            .env("ZENITH_REPO_DIR", self.repo_path())
-            .env("PATH", self.env.pg_bin_dir().to_str().unwrap()) // needs postres-wal-redo binary
-            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
             .status()
             .expect("pageserver init failed");
 
@@ -76,28 +100,15 @@ impl PageServerNode {
     pub fn start(&self) -> Result<()> {
         println!(
             "Starting pageserver at '{}' in {}",
-            self.address(),
+            connection_address(&self.connection_config),
             self.repo_path().display()
         );
 
         let mut cmd = Command::new(self.env.pageserver_bin()?);
-        cmd.args(&[
-            "-l",
-            self.address().to_string().as_str(),
-            "-D",
-            self.repo_path().to_str().unwrap(),
-        ])
-        .arg("-d")
-        .env_clear()
-        .env("RUST_BACKTRACE", "1")
-        .env(
-            "POSTGRES_DISTRIB_DIR",
-            self.env.pg_distrib_dir.to_str().unwrap(),
-        )
-        .env("ZENITH_REPO_DIR", self.repo_path())
-        .env("PATH", self.env.pg_bin_dir().to_str().unwrap()) // needs postres-wal-redo binary
-        .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-        .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap());
+        cmd.args(&["-D", self.repo_path().to_str().unwrap()])
+            .arg("-d")
+            .env_clear()
+            .env("RUST_BACKTRACE", "1");
 
         if !cmd.status()?.success() {
             bail!(
@@ -109,18 +120,21 @@ impl PageServerNode {
         // It takes a while for the page server to start up. Wait until it is
         // open for business.
         for retries in 1..15 {
-            let client = self.page_server_psql_client();
-            if client.is_ok() {
-                break;
-            } else {
-                println!("Pageserver not responding yet, retrying ({})...", retries);
-                thread::sleep(Duration::from_secs(1));
+            match self.page_server_psql_client() {
+                Ok(_) => {
+                    println!("Pageserver started");
+                    return Ok(());
+                }
+                Err(err) => {
+                    println!(
+                        "Pageserver not responding yet, err {} retrying ({})...",
+                        err, retries
+                    );
+                    thread::sleep(Duration::from_secs(1));
+                }
             }
         }
-
-        println!("Pageserver started");
-
-        Ok(())
+        bail!("pageserver failed to start");
     }
 
     pub fn stop(&self) -> Result<()> {
@@ -131,47 +145,55 @@ impl PageServerNode {
         }
 
         // wait for pageserver stop
+        let address = connection_address(&self.connection_config);
         for _ in 0..5 {
-            let stream = TcpStream::connect(self.address());
+            let stream = TcpStream::connect(&address);
             thread::sleep(Duration::from_secs(1));
             if let Err(_e) = stream {
                 println!("Pageserver stopped");
                 return Ok(());
             }
-            println!("Stopping pageserver on {}", self.address());
+            println!("Stopping pageserver on {}", address);
         }
 
         bail!("Failed to stop pageserver with pid {}", pid);
     }
 
     pub fn page_server_psql(&self, sql: &str) -> Vec<postgres::SimpleQueryMessage> {
-        let connstring = format!(
-            "host={} port={} dbname={} user={}",
-            self.address().ip(),
-            self.address().port(),
-            "no_db",
-            "no_user",
-        );
-        let mut client = Client::connect(connstring.as_str(), NoTls).unwrap();
+        let mut client = self.connection_config.connect(NoTls).unwrap();
 
         println!("Pageserver query: '{}'", sql);
         client.simple_query(sql).unwrap()
     }
 
     pub fn page_server_psql_client(&self) -> Result<postgres::Client, postgres::Error> {
-        let connstring = format!(
-            "host={} port={} dbname={} user={}",
-            self.address().ip(),
-            self.address().port(),
-            "no_db",
-            "no_user",
-        );
-        Client::connect(connstring.as_str(), NoTls)
+        self.connection_config.connect(NoTls)
     }
 
-    pub fn branches_list(&self) -> Result<Vec<BranchInfo>> {
+    pub fn tenants_list(&self) -> Result<Vec<String>> {
         let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query("branch_list")?;
+        let query_result = client.simple_query("tenant_list")?;
+        let tenants_json = query_result
+            .first()
+            .map(|msg| match msg {
+                postgres::SimpleQueryMessage::Row(row) => row.get(0),
+                _ => None,
+            })
+            .flatten()
+            .ok_or_else(|| anyhow!("missing tenants"))?;
+
+        Ok(serde_json::from_str(tenants_json)?)
+    }
+
+    pub fn tenant_create(&self, tenantid: &ZTenantId) -> Result<()> {
+        let mut client = self.page_server_psql_client()?;
+        client.simple_query(format!("tenant_create {}", tenantid).as_str())?;
+        Ok(())
+    }
+
+    pub fn branches_list(&self, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
+        let mut client = self.page_server_psql_client()?;
+        let query_result = client.simple_query(&format!("branch_list {}", tenantid))?;
         let branches_json = query_result
             .first()
             .map(|msg| match msg {
@@ -181,14 +203,19 @@ impl PageServerNode {
             .flatten()
             .ok_or_else(|| anyhow!("missing branches"))?;
 
-        let res: Vec<BranchInfo> = serde_json::from_str(branches_json)?;
-        Ok(res)
+        Ok(serde_json::from_str(branches_json)?)
     }
 
-    pub fn branch_create(&self, name: &str, startpoint: &str) -> Result<BranchInfo> {
+    pub fn branch_create(
+        &self,
+        branch_name: &str,
+        startpoint: &str,
+        tenantid: &ZTenantId,
+    ) -> Result<BranchInfo> {
         let mut client = self.page_server_psql_client()?;
-        let query_result =
-            client.simple_query(format!("branch_create {} {}", name, startpoint).as_str())?;
+        let query_result = client.simple_query(
+            format!("branch_create {} {} {}", tenantid, branch_name, startpoint).as_str(),
+        )?;
 
         let branch_json = query_result
             .first()
@@ -211,8 +238,12 @@ impl PageServerNode {
     }
 
     // TODO: make this a separate request type and avoid loading all the branches
-    pub fn branch_get_by_name(&self, name: &str) -> Result<BranchInfo> {
-        let branch_infos = self.branches_list()?;
+    pub fn branch_get_by_name(
+        &self,
+        tenantid: &ZTenantId,
+        branch_name: &str,
+    ) -> Result<BranchInfo> {
+        let branch_infos = self.branches_list(tenantid)?;
         let branche_by_name: Result<HashMap<String, BranchInfo>> = branch_infos
             .into_iter()
             .map(|branch_info| Ok((branch_info.name.clone(), branch_info)))
@@ -220,27 +251,11 @@ impl PageServerNode {
         let branche_by_name = branche_by_name?;
 
         let branch = branche_by_name
-            .get(name)
-            .ok_or_else(|| anyhow!("Branch {} not found", name))?;
+            .get(branch_name)
+            .ok_or_else(|| anyhow!("Branch {} not found", branch_name))?;
 
         Ok(branch.clone())
     }
-
-    pub fn system_id_get(&self) -> Result<u64> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result = client
-            .simple_query("identify_system")?
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
-            })
-            .flatten()
-            .ok_or_else(|| anyhow!("failed to get system_id"))?
-            .parse::<u64>()?;
-
-        Ok(query_result)
-    }
 }
 
 impl Drop for PageServerNode {

docker-entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+if [ "$1" = 'pageserver' ]; then
+    if [ ! -d "/data/tenants" ]; then
+        echo "Initializing pageserver data directory"
+        pageserver --init -D /data --postgres-distrib /usr/local
+    fi
+    echo "Staring pageserver at 0.0.0.0:6400"
+    pageserver -l 0.0.0.0:6400 -D /data
+else
+    "$@"
+fi

docs/README.md

@@ -0,0 +1,11 @@
+# Zenith documentation
+
+## Table of contents
+
+- [authentication.md](authentication.md) — pageserver JWT authentication.
+- [docker.md](docker.md) — Docker images and building pipeline.
+- [multitenancy.md](multitenancy.md) — how multitenancy is organized in the pageserver and Zenith CLI.
+- [pageserver/README](/pageserver/README) — pageserver overview.
+- [postgres_ffi/README](/postgres_ffi/README) — Postgres FFI overview.
+- [test_runner/README.md](/test_runner/README.md) — tests infrastructure overview.
+- [walkeeper/README](/walkeeper/README.md) — WAL service overview.

docs/authentication.md

@@ -0,0 +1,30 @@
+## Authentication
+
+### Overview
+
+Current state of authentication includes usage of JWT tokens in communication between compute and pageserver and between CLI and pageserver. JWT token is signed using RSA keys. CLI generates a key pair during call to `zenith init`. Using following openssl commands:
+
+```bash
+openssl genrsa -out private_key.pem 2048
+openssl rsa -in private_key.pem -pubout -outform PEM -out public_key.pem
+```
+
+CLI also generates signed token and saves it in the config for later access to pageserver. Now authentication is optional. Pageserver has two variables in config: `auth_validation_public_key_path` and `auth_type`, so when auth type present and set to `ZenithJWT` pageserver will require authentication for connections. Actual JWT is passed in password field of connection string. There is a caveat for psql, it silently truncates passwords to 100 symbols, so to correctly pass JWT via psql you have to either use PGPASSWORD environment variable, or store password in psql config file.
+
+Currently there is no authentication between compute and safekeepers, because this communication layer is under heavy refactoring. After this refactoring support for authentication will be added there too. Now safekeeper supports "hardcoded" token passed via environment variable to be able to use callmemaybe command in pageserver.
+
+Compute uses token passed via environment variable to communicate to pageserver and in the future to the safekeeper too.
+
+JWT authentication now supports two scopes: tenant and pageserverapi. Tenant scope is intended for use in tenant related api calls, e.g. create_branch. Compute launched for particular tenant also uses this scope. Scope pageserver api is intended to be used by console to manage pageserver. For now we have only one management operation - create tenant.
+
+Examples for token generation in python:
+
+```python
+# generate pageserverapi token
+management_token = jwt.encode({"scope": "pageserverapi"}, auth_keys.priv, algorithm="RS256")
+
+# generate tenant token
+tenant_token = jwt.encode({"scope": "tenant", "tenant_id": ps.initial_tenant}, auth_keys.priv, algorithm="RS256")
+```
+
+Utility functions to work with jwts in rust are located in zenith_utils/src/auth.rs

docs/docker.md

@@ -0,0 +1,38 @@
+# Docker images of Zenith
+
+## Images
+
+Currently we build two main images:
+
+- [zenithdb/zenith](https://hub.docker.com/repository/docker/zenithdb/zenith) — image with pre-built `pageserver`, `wal_acceptor` and `proxy` binaries and all the required runtime dependencies. Built from [/Dockerfile](/Dockerfile).
+- [zenithdb/compute-node](https://hub.docker.com/repository/docker/zenithdb/compute-node) — compute node image with pre-built Postgres binaries from [zenithdb/postgres](https://github.com/zenithdb/postgres).
+
+And two intermediate images used either to reduce build time or to deliver some additional binary tools from other repos:
+
+- [zenithdb/build](https://hub.docker.com/repository/docker/zenithdb/build) — image with all the dependencies required to build Zenith and compute node images. This image is based on `rust:slim-buster`, so it also has a proper `rust` environment. Built from [/Dockerfile.build](/Dockerfile.build).
+- [zenithdb/compute-tools](https://hub.docker.com/repository/docker/zenithdb/compute-tools) — compute node configuration management tools.
+
+## Building pipeline
+
+1. Image `zenithdb/compute-tools` is re-built automatically.
+
+2. Image `zenithdb/build` is built manually. If you want to introduce any new compile time dependencies to Zenith or compute node you have to update this image as well, build it and push to Docker Hub.
+
+Build:
+```sh
+docker build -t zenithdb/build:buster -f Dockerfile.build .
+```
+
+Login:
+```sh
+docker login
+```
+
+Push to Docker Hub:
+```sh
+docker push zenithdb/build:buster
+```
+
+3. Image `zenithdb/compute-node` is built independently in the [zenithdb/postgres](https://github.com/zenithdb/postgres) repo.
+
+4. Image `zenithdb/zenith` is built in this repo after a successful `release` tests run and pushed to Docker Hub automatically.

docs/multitenancy.md

@@ -0,0 +1,59 @@
+## Multitenancy
+
+### Overview
+
+Zenith supports multitenancy. One pageserver can serve multiple tenants at once. Tenants can be managed via zenith CLI. During page server setup tenant can be created using ```zenith init --create-tenant``` Also tenants can be added into the system on the fly without pageserver restart. This can be done using the following cli command: ```zenith tenant create``` Tenants use random identifiers which can be represented as a 32 symbols hexadecimal string. So zenith tenant create accepts desired tenant id as an optional argument. The concept of timelines/branches is working independently per tenant.
+
+### Tenants in other commands
+
+By default during `zenith init` new tenant is created on the pageserver. Newly created tenant's id is saved to cli config, so other commands can use it automatically if no direct arugment `--tenantid=<tenantid>` is provided. So generally tenantid more frequently appears in internal pageserver interface. Its commands take tenantid argument to distinguish to which tenant operation should be applied. CLI support creation of new tenants.
+
+Examples for cli:
+
+```sh
+zenith tenant list
+
+zenith tenant create // generates new id
+
+zenith tenant create ee6016ec31116c1b7c33dfdfca38892f
+
+zenith pg create main // default tenant from zenith init
+
+zenith pg create main --tenantid=ee6016ec31116c1b7c33dfdfca38892f
+
+zenith branch --tenantid=ee6016ec31116c1b7c33dfdfca38892f
+```
+
+### Data layout
+
+On the page server tenants introduce one level of indirection, so data directory structured the following way:
+```
+<pageserver working directory>
+├── pageserver.log
+├── pageserver.pid
+├── pageserver.toml
+└── tenants
+   ├── 537cffa58a4fa557e49e19951b5a9d6b
+   ├── de182bc61fb11a5a6b390a8aed3a804a
+   └── ee6016ec31116c1b7c33dfdfca38891f
+```
+Wal redo activity, timelines, snapshots are managed for each tenant independently.
+
+For local environment used for example in tests there also new level of indirection for tenants. It touches `pgdatadirs` directory. Now it contains `tenants` subdirectory so the structure looks the following way:
+
+```
+pgdatadirs
+└── tenants
+   ├── de182bc61fb11a5a6b390a8aed3a804a
+   │  └── main
+   └── ee6016ec31116c1b7c33dfdfca38892f
+      └── main
+```
+
+### Changes to postgres
+
+Tenant id is passed to postgres via GUC the same way as the timeline. Tenant id is added to commands issued to pageserver, namely: pagestream, callmemaybe. Tenant id is also exists in ServerInfo structure, this is needed to pass the value to wal receiver to be able to forward it to the pageserver.
+
+### Safety
+
+For now particular tenant can only appear on a particular pageserver. Set of WAL acceptors are also pinned to particular (tenantid, timeline) pair so there can only be one writer for particular (tenantid, timeline).

integration_tests/.gitignore

@@ -1 +0,0 @@
-tmp_check/

integration_tests/Cargo.toml

@@ -1,18 +0,0 @@
-[package]
-name = "integration_tests"
-version = "0.1.0"
-authors = ["Stas Kelvich <stas@zenith.tech>"]
-edition = "2018"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-lazy_static = "1.4.0"
-rand = "0.8.3"
-anyhow = "1.0"
-nix = "0.20"
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-
-pageserver = { path = "../pageserver" }
-walkeeper = { path = "../walkeeper" }
-control_plane = { path = "../control_plane" }

integration_tests/src/lib.rs

@@ -1,416 +0,0 @@
-use std::collections::BTreeMap;
-use std::convert::TryInto;
-use std::fs::{self, File, OpenOptions};
-use std::io::Read;
-use std::net::SocketAddr;
-use std::path::{Path, PathBuf};
-use std::process::{Command, ExitStatus};
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
-
-use anyhow::{bail, Result};
-use nix::sys::signal::{kill, Signal};
-use nix::unistd::Pid;
-use postgres;
-
-use control_plane::compute::PostgresNode;
-use control_plane::read_pidfile;
-use control_plane::{local_env::LocalEnv, storage::PageServerNode};
-
-// Find the directory where the binaries were put (i.e. target/debug/)
-fn cargo_bin_dir() -> PathBuf {
-    let mut pathbuf = std::env::current_exe().unwrap();
-
-    pathbuf.pop();
-    if pathbuf.ends_with("deps") {
-        pathbuf.pop();
-    }
-
-    pathbuf
-}
-
-// local compute env for tests
-pub fn create_test_env(testname: &str) -> LocalEnv {
-    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
-        .join("../tmp_check/")
-        .join(testname);
-
-    let base_path_str = base_path.to_str().unwrap();
-
-    // Remove remnants of old test repo
-    let _ = fs::remove_dir_all(&base_path);
-
-    fs::create_dir_all(&base_path)
-        .expect(format!("could not create directory for {}", base_path_str).as_str());
-
-    let pgdatadirs_path = base_path.join("pgdatadirs");
-    fs::create_dir(&pgdatadirs_path)
-        .expect(format!("could not create directory {:?}", pgdatadirs_path).as_str());
-
-    LocalEnv {
-        pageserver_connstring: "postgresql://127.0.0.1:64000".to_string(),
-        pg_distrib_dir: Path::new(env!("CARGO_MANIFEST_DIR")).join("../tmp_install"),
-        zenith_distrib_dir: Some(cargo_bin_dir()),
-        base_data_dir: base_path,
-        remotes: BTreeMap::default(),
-    }
-}
-
-//
-// Collection of several example deployments useful for tests.
-//
-// I'm intendedly modelling storage and compute control planes as a separate entities
-// as it is closer to the actual setup.
-//
-pub struct TestStorageControlPlane {
-    pub wal_acceptors: Vec<WalAcceptorNode>,
-    pub pageserver: Arc<PageServerNode>,
-    pub test_done: AtomicBool,
-}
-
-impl TestStorageControlPlane {
-    // postgres <-> page_server
-    //
-    // Initialize a new repository and configure a page server to run in it
-    //
-    pub fn one_page_server(local_env: &LocalEnv) -> TestStorageControlPlane {
-        let pserver = Arc::new(PageServerNode {
-            env: local_env.clone(),
-            kill_on_exit: true,
-            listen_address: None,
-        });
-        pserver.init().unwrap();
-        pserver.start().unwrap();
-
-        TestStorageControlPlane {
-            wal_acceptors: Vec::new(),
-            pageserver: pserver,
-            test_done: AtomicBool::new(false),
-        }
-    }
-
-    // postgres <-> {wal_acceptor1, wal_acceptor2, ...}
-    pub fn fault_tolerant(local_env: &LocalEnv, redundancy: usize) -> TestStorageControlPlane {
-        let mut cplane = TestStorageControlPlane {
-            wal_acceptors: Vec::new(),
-            pageserver: Arc::new(PageServerNode {
-                env: local_env.clone(),
-                kill_on_exit: true,
-                listen_address: None,
-            }),
-            test_done: AtomicBool::new(false),
-            // repopath,
-        };
-        cplane.pageserver.init().unwrap();
-        cplane.pageserver.start().unwrap();
-
-        let systemid = cplane.pageserver.system_id_get().unwrap();
-
-        const WAL_ACCEPTOR_PORT: usize = 54321;
-
-        let datadir_base = local_env.base_data_dir.join("safekeepers");
-        fs::create_dir_all(&datadir_base).unwrap();
-
-        for i in 0..redundancy {
-            let wal_acceptor = WalAcceptorNode {
-                listen: format!("127.0.0.1:{}", WAL_ACCEPTOR_PORT + i)
-                    .parse()
-                    .unwrap(),
-                data_dir: datadir_base.join(format!("wal_acceptor_{}", i)),
-                systemid,
-                env: local_env.clone(),
-                pass_to_pageserver: true,
-            };
-            wal_acceptor.init();
-            wal_acceptor.start();
-            cplane.wal_acceptors.push(wal_acceptor);
-        }
-        cplane
-    }
-
-    pub fn stop(&self) {
-        for wa in self.wal_acceptors.iter() {
-            let _ = wa.stop();
-        }
-        self.test_done.store(true, Ordering::Relaxed);
-    }
-
-    pub fn get_wal_acceptor_conn_info(&self) -> String {
-        self.wal_acceptors
-            .iter()
-            .map(|wa| wa.listen.to_string())
-            .collect::<Vec<String>>()
-            .join(",")
-    }
-
-    pub fn is_running(&self) -> bool {
-        self.test_done.load(Ordering::Relaxed)
-    }
-}
-
-impl Drop for TestStorageControlPlane {
-    fn drop(&mut self) {
-        self.stop();
-    }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-//
-// PostgresNodeExt
-//
-///////////////////////////////////////////////////////////////////////////////
-
-///
-/// Testing utilities for PostgresNode type
-///
-pub trait PostgresNodeExt {
-    fn pg_regress(&self) -> ExitStatus;
-    fn pg_bench(&self, clients: u32, seconds: u32) -> ExitStatus;
-    fn start_proxy(&self, wal_acceptors: &str) -> WalProposerNode;
-    fn open_psql(&self, db: &str) -> postgres::Client;
-    fn dump_log_file(&self);
-    fn safe_psql(&self, db: &str, sql: &str) -> Vec<postgres::Row>;
-}
-
-impl PostgresNodeExt for PostgresNode {
-    fn pg_regress(&self) -> ExitStatus {
-        self.safe_psql("postgres", "CREATE DATABASE regression");
-
-        let regress_run_path = self.env.base_data_dir.join("regress");
-        fs::create_dir_all(&regress_run_path).unwrap();
-        fs::create_dir_all(regress_run_path.join("testtablespace")).unwrap();
-        std::env::set_current_dir(regress_run_path).unwrap();
-
-        let regress_build_path =
-            Path::new(env!("CARGO_MANIFEST_DIR")).join("../tmp_install/build/src/test/regress");
-        let regress_src_path =
-            Path::new(env!("CARGO_MANIFEST_DIR")).join("../vendor/postgres/src/test/regress");
-
-        let regress_check = Command::new(regress_build_path.join("pg_regress"))
-            .args(&[
-                "--bindir=''",
-                "--use-existing",
-                format!("--bindir={}", self.env.pg_bin_dir().to_str().unwrap()).as_str(),
-                format!("--dlpath={}", regress_build_path.to_str().unwrap()).as_str(),
-                format!(
-                    "--schedule={}",
-                    regress_src_path.join("parallel_schedule").to_str().unwrap()
-                )
-                .as_str(),
-                format!("--inputdir={}", regress_src_path.to_str().unwrap()).as_str(),
-            ])
-            .env_clear()
-            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .env("PGPORT", self.address.port().to_string())
-            .env("PGUSER", self.whoami())
-            .env("PGHOST", self.address.ip().to_string())
-            .status()
-            .expect("pg_regress failed");
-        if !regress_check.success() {
-            if let Ok(mut file) = File::open("regression.diffs") {
-                let mut buffer = String::new();
-                file.read_to_string(&mut buffer).unwrap();
-                println!("--------------- regression.diffs:\n{}", buffer);
-            }
-            self.dump_log_file();
-        }
-        regress_check
-    }
-
-    fn pg_bench(&self, clients: u32, seconds: u32) -> ExitStatus {
-        let port = self.address.port().to_string();
-        let clients = clients.to_string();
-        let seconds = seconds.to_string();
-        let _pg_bench_init = Command::new(self.env.pg_bin_dir().join("pgbench"))
-            .args(&["-i", "-p", port.as_str(), "postgres"])
-            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .status()
-            .expect("pgbench -i");
-        let pg_bench_run = Command::new(self.env.pg_bin_dir().join("pgbench"))
-            .args(&[
-                "-p",
-                port.as_str(),
-                "-T",
-                seconds.as_str(),
-                "-P",
-                "1",
-                "-c",
-                clients.as_str(),
-                "-M",
-                "prepared",
-                "postgres",
-            ])
-            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .status()
-            .expect("pgbench run");
-        pg_bench_run
-    }
-
-    fn start_proxy(&self, wal_acceptors: &str) -> WalProposerNode {
-        let proxy_path = self.env.pg_bin_dir().join("safekeeper_proxy");
-        match Command::new(proxy_path.as_path())
-            .args(&["--ztimelineid", &self.timelineid.to_string()])
-            .args(&["-s", wal_acceptors])
-            .args(&["-h", &self.address.ip().to_string()])
-            .args(&["-p", &self.address.port().to_string()])
-            .arg("-v")
-            .stderr(
-                OpenOptions::new()
-                    .create(true)
-                    .append(true)
-                    .open(self.pgdata().join("safekeeper_proxy.log"))
-                    .unwrap(),
-            )
-            .spawn()
-        {
-            Ok(child) => WalProposerNode { pid: child.id() },
-            Err(e) => panic!("Failed to launch {:?}: {}", proxy_path, e),
-        }
-    }
-
-    fn dump_log_file(&self) {
-        if let Ok(mut file) = File::open(self.env.pageserver_data_dir().join("pageserver.log")) {
-            let mut buffer = String::new();
-            file.read_to_string(&mut buffer).unwrap();
-            println!("--------------- pageserver.log:\n{}", buffer);
-        }
-    }
-
-    fn safe_psql(&self, db: &str, sql: &str) -> Vec<postgres::Row> {
-        let connstring = format!(
-            "host={} port={} dbname={} user={}",
-            self.address.ip(),
-            self.address.port(),
-            db,
-            self.whoami()
-        );
-        let mut client = postgres::Client::connect(connstring.as_str(), postgres::NoTls).unwrap();
-
-        println!("Running {}", sql);
-        let result = client.query(sql, &[]);
-        if result.is_err() {
-            self.dump_log_file();
-        }
-        result.unwrap()
-    }
-
-    fn open_psql(&self, db: &str) -> postgres::Client {
-        let connstring = format!(
-            "host={} port={} dbname={} user={}",
-            self.address.ip(),
-            self.address.port(),
-            db,
-            self.whoami()
-        );
-        postgres::Client::connect(connstring.as_str(), postgres::NoTls).unwrap()
-    }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-//
-// WalAcceptorNode
-//
-///////////////////////////////////////////////////////////////////////////////
-
-//
-// Control routines for WalAcceptor.
-//
-// Now used only in test setups.
-//
-pub struct WalAcceptorNode {
-    listen: SocketAddr,
-    data_dir: PathBuf,
-    systemid: u64,
-    env: LocalEnv,
-    pass_to_pageserver: bool,
-}
-
-impl WalAcceptorNode {
-    pub fn init(&self) {
-        if self.data_dir.exists() {
-            fs::remove_dir_all(self.data_dir.clone()).unwrap();
-        }
-        fs::create_dir_all(self.data_dir.clone()).unwrap();
-    }
-
-    pub fn start(&self) {
-        println!(
-            "Starting wal_acceptor in {} listening '{}'",
-            self.data_dir.to_str().unwrap(),
-            self.listen
-        );
-
-        let ps_arg = if self.pass_to_pageserver {
-            // Tell page server it can receive WAL from this WAL safekeeper
-            ["--pageserver", "127.0.0.1:64000"].to_vec()
-        } else {
-            [].to_vec()
-        };
-
-        let status = Command::new(
-            self.env
-                .zenith_distrib_dir
-                .as_ref()
-                .unwrap()
-                .join("wal_acceptor"),
-        )
-        .args(&["-D", self.data_dir.to_str().unwrap()])
-        .args(&["-l", self.listen.to_string().as_str()])
-        .args(&["--systemid", self.systemid.to_string().as_str()])
-        .args(&ps_arg)
-        .arg("-d")
-        .arg("-n")
-        .status()
-        .expect("failed to start wal_acceptor");
-
-        if !status.success() {
-            panic!("wal_acceptor start failed");
-        }
-    }
-
-    pub fn stop(&self) -> Result<()> {
-        println!("Stopping wal acceptor on {}", self.listen);
-        let pidfile = self.data_dir.join("wal_acceptor.pid");
-        let pid = read_pidfile(&pidfile)?;
-        let pid = Pid::from_raw(pid);
-        if kill(pid, Signal::SIGTERM).is_err() {
-            bail!("Failed to kill wal_acceptor with pid {}", pid);
-        }
-        Ok(())
-    }
-}
-
-impl Drop for WalAcceptorNode {
-    fn drop(&mut self) {
-        // Ignore errors.
-        let _ = self.stop();
-    }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-//
-// WalProposerNode
-//
-///////////////////////////////////////////////////////////////////////////////
-
-pub struct WalProposerNode {
-    pub pid: u32,
-}
-
-impl WalProposerNode {
-    pub fn stop(&self) {
-        // std::process::Child::id() returns u32, we need i32.
-        let pid: i32 = self.pid.try_into().unwrap();
-        let pid = Pid::from_raw(pid);
-        kill(pid, Signal::SIGTERM).expect("failed to execute kill");
-    }
-}
-
-impl Drop for WalProposerNode {
-    fn drop(&mut self) {
-        self.stop();
-    }
-}

integration_tests/tests/test_wal_acceptor.rs

@@ -1,332 +0,0 @@
-use rand::Rng;
-use std::sync::Arc;
-use std::time::SystemTime;
-use std::{thread, time};
-
-use control_plane::compute::{ComputeControlPlane, PostgresNode};
-
-use integration_tests;
-use integration_tests::PostgresNodeExt;
-use integration_tests::TestStorageControlPlane;
-
-const DOWNTIME: u64 = 2;
-
-fn start_node_with_wal_proposer(
-    timeline: &str,
-    compute_cplane: &mut ComputeControlPlane,
-    wal_acceptors: &String,
-) -> Arc<PostgresNode> {
-    let node = compute_cplane.new_test_master_node(timeline);
-    let _node = node.append_conf(
-        "postgresql.conf",
-        &format!("wal_acceptors='{}'\n", wal_acceptors),
-    );
-    node.start().unwrap();
-    node
-}
-
-#[test]
-fn test_embedded_wal_proposer() {
-    let local_env = integration_tests::create_test_env("test_embedded_wal_proposer");
-
-    const REDUNDANCY: usize = 3;
-    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
-    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
-    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
-
-    // start postgres
-    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
-
-    // check basic work with table
-    node.safe_psql(
-        "postgres",
-        "CREATE TABLE t(key int primary key, value text)",
-    );
-    node.safe_psql(
-        "postgres",
-        "INSERT INTO t SELECT generate_series(1,100000), 'payload'",
-    );
-    let count: i64 = node
-        .safe_psql("postgres", "SELECT sum(key) FROM t")
-        .first()
-        .unwrap()
-        .get(0);
-    println!("sum = {}", count);
-    assert_eq!(count, 5000050000);
-    // check wal files equality
-}
-
-#[test]
-fn test_acceptors_normal_work() {
-    let local_env = integration_tests::create_test_env("test_acceptors_normal_work");
-
-    const REDUNDANCY: usize = 3;
-    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
-    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
-    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
-
-    // start postgres
-    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
-
-    // check basic work with table
-    node.safe_psql(
-        "postgres",
-        "CREATE TABLE t(key int primary key, value text)",
-    );
-    node.safe_psql(
-        "postgres",
-        "INSERT INTO t SELECT generate_series(1,100000), 'payload'",
-    );
-    let count: i64 = node
-        .safe_psql("postgres", "SELECT sum(key) FROM t")
-        .first()
-        .unwrap()
-        .get(0);
-    println!("sum = {}", count);
-    assert_eq!(count, 5000050000);
-    // check wal files equality
-}
-
-// Run page server and multiple safekeepers, and multiple compute nodes running
-// against different timelines.
-#[test]
-fn test_many_timelines() {
-    // Initialize a new repository, and set up WAL safekeepers and page server.
-    const REDUNDANCY: usize = 3;
-    const N_TIMELINES: usize = 5;
-    let local_env = integration_tests::create_test_env("test_many_timelines");
-    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
-    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
-    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
-
-    // Create branches
-    let mut timelines: Vec<String> = Vec::new();
-    timelines.push("main".to_string());
-
-    for i in 1..N_TIMELINES {
-        let branchname = format!("experimental{}", i);
-        storage_cplane
-            .pageserver
-            .branch_create(&branchname, "main")
-            .unwrap();
-        timelines.push(branchname);
-    }
-
-    // start postgres on each timeline
-    let mut nodes = Vec::new();
-    for tli_name in timelines {
-        let node = start_node_with_wal_proposer(&tli_name, &mut compute_cplane, &wal_acceptors);
-        nodes.push(node.clone());
-    }
-
-    // create schema
-    for node in &nodes {
-        node.safe_psql(
-            "postgres",
-            "CREATE TABLE t(key int primary key, value text)",
-        );
-    }
-
-    // Populate data
-    for node in &nodes {
-        node.safe_psql(
-            "postgres",
-            "INSERT INTO t SELECT generate_series(1,100000), 'payload'",
-        );
-    }
-
-    // Check data
-    for node in &nodes {
-        let count: i64 = node
-            .safe_psql("postgres", "SELECT sum(key) FROM t")
-            .first()
-            .unwrap()
-            .get(0);
-        println!("sum = {}", count);
-        assert_eq!(count, 5000050000);
-    }
-}
-
-// Majority is always alive
-#[test]
-fn test_acceptors_restarts() {
-    let local_env = integration_tests::create_test_env("test_acceptors_restarts");
-
-    // Start pageserver that reads WAL directly from that postgres
-    const REDUNDANCY: usize = 3;
-    const FAULT_PROBABILITY: f32 = 0.01;
-
-    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
-    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
-    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
-    let mut rng = rand::thread_rng();
-
-    // start postgres
-    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
-
-    let mut failed_node: Option<usize> = None;
-
-    // check basic work with table
-    node.safe_psql(
-        "postgres",
-        "CREATE TABLE t(key int primary key, value text)",
-    );
-    let mut psql = node.open_psql("postgres");
-    for i in 1..=1000 {
-        psql.execute("INSERT INTO t values ($1, 'payload')", &[&i])
-            .unwrap();
-        let prob: f32 = rng.gen();
-        if prob <= FAULT_PROBABILITY {
-            if let Some(node) = failed_node {
-                storage_cplane.wal_acceptors[node].start();
-                failed_node = None;
-            } else {
-                let node: usize = rng.gen_range(0..REDUNDANCY);
-                failed_node = Some(node);
-                storage_cplane.wal_acceptors[node].stop().unwrap();
-            }
-        }
-    }
-    let count: i64 = node
-        .safe_psql("postgres", "SELECT sum(key) FROM t")
-        .first()
-        .unwrap()
-        .get(0);
-    println!("sum = {}", count);
-    assert_eq!(count, 500500);
-}
-
-fn start_acceptor(cplane: &Arc<TestStorageControlPlane>, no: usize) {
-    let cp = cplane.clone();
-    thread::spawn(move || {
-        thread::sleep(time::Duration::from_secs(DOWNTIME));
-        cp.wal_acceptors[no].start();
-    });
-}
-
-// Stop majority of acceptors while compute is under the load. Boot
-// them again and check that nothing was losed. Repeat.
-// N_CRASHES env var
-#[test]
-fn test_acceptors_unavailability() {
-    let local_env = integration_tests::create_test_env("test_acceptors_unavailability");
-
-    // Start pageserver that reads WAL directly from that postgres
-    const REDUNDANCY: usize = 2;
-
-    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
-    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
-    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
-
-    // start postgres
-    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
-
-    // check basic work with table
-    node.safe_psql(
-        "postgres",
-        "CREATE TABLE t(key int primary key, value text)",
-    );
-    let mut psql = node.open_psql("postgres");
-    psql.execute("INSERT INTO t values (1, 'payload')", &[])
-        .unwrap();
-
-    // Shut down all wal acceptors
-    storage_cplane.wal_acceptors[0].stop().unwrap();
-    let cp = Arc::new(storage_cplane);
-    start_acceptor(&cp, 0);
-    let now = SystemTime::now();
-    psql.execute("INSERT INTO t values (2, 'payload')", &[])
-        .unwrap();
-    // Here we check that the query above was hanging
-    // while wal_acceptor was unavailiable
-    assert!(now.elapsed().unwrap().as_secs() >= DOWNTIME);
-    psql.execute("INSERT INTO t values (3, 'payload')", &[])
-        .unwrap();
-
-    cp.wal_acceptors[1].stop().unwrap();
-    start_acceptor(&cp, 1);
-    psql.execute("INSERT INTO t values (4, 'payload')", &[])
-        .unwrap();
-    // Here we check that the query above was hanging
-    // while wal_acceptor was unavailiable
-    assert!(now.elapsed().unwrap().as_secs() >= 2 * DOWNTIME);
-
-    psql.execute("INSERT INTO t values (5, 'payload')", &[])
-        .unwrap();
-
-    let count: i64 = node
-        .safe_psql("postgres", "SELECT sum(key) FROM t")
-        .first()
-        .unwrap()
-        .get(0);
-    println!("sum = {}", count);
-    // Ensure that all inserts succeeded.
-    // Including ones that were waiting for wal acceptor restart.
-    assert_eq!(count, 15);
-}
-
-fn simulate_failures(cplane: Arc<TestStorageControlPlane>) {
-    let mut rng = rand::thread_rng();
-    let n_acceptors = cplane.wal_acceptors.len();
-    let failure_period = time::Duration::from_secs(1);
-    while cplane.is_running() {
-        thread::sleep(failure_period);
-        let mask: u32 = rng.gen_range(0..(1 << n_acceptors));
-        for i in 0..n_acceptors {
-            if (mask & (1 << i)) != 0 {
-                cplane.wal_acceptors[i].stop().unwrap();
-            }
-        }
-        thread::sleep(failure_period);
-        for i in 0..n_acceptors {
-            if (mask & (1 << i)) != 0 {
-                cplane.wal_acceptors[i].start();
-            }
-        }
-    }
-}
-
-// Race condition test
-#[test]
-fn test_race_conditions() {
-    let local_env = integration_tests::create_test_env("test_race_conditions");
-
-    // Start pageserver that reads WAL directly from that postgres
-    const REDUNDANCY: usize = 3;
-
-    let storage_cplane = Arc::new(TestStorageControlPlane::fault_tolerant(
-        &local_env, REDUNDANCY,
-    ));
-    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
-    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
-
-    // start postgres
-    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
-
-    // check basic work with table
-    node.safe_psql(
-        "postgres",
-        "CREATE TABLE t(key int primary key, value text)",
-    );
-
-    let cp = storage_cplane.clone();
-    let failures_thread = thread::spawn(move || {
-        simulate_failures(cp);
-    });
-
-    let mut psql = node.open_psql("postgres");
-    for i in 1..=1000 {
-        psql.execute("INSERT INTO t values ($1, 'payload')", &[&i])
-            .unwrap();
-    }
-    let count: i64 = node
-        .safe_psql("postgres", "SELECT sum(key) FROM t")
-        .first()
-        .unwrap()
-        .get(0);
-    println!("sum = {}", count);
-    assert_eq!(count, 500500);
-
-    storage_cplane.stop();
-    failures_thread.join().unwrap();
-}

monitoring/docker-compose.yml

@@ -0,0 +1,25 @@
+version: "3"
+services:
+
+  prometheus:
+    container_name: prometheus
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
+    # ports:
+    #   - "9090:9090"
+    # TODO: find a proper portable solution
+    network_mode: "host"
+
+  grafana:
+    image: grafana/grafana:latest
+    volumes:
+      - ./grafana.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+    # ports:
+    #   - "3000:3000"
+    # TODO: find a proper portable solution
+    network_mode: "host"

monitoring/grafana.yaml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+datasources:
+- name: Prometheus
+  type: prometheus
+  access: proxy
+  orgId: 1
+  url: http://localhost:9090
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: false

monitoring/prometheus.yaml

@@ -0,0 +1,5 @@
+scrape_configs:
+  - job_name: 'default'
+    scrape_interval: 10s
+    static_configs:
+      - targets: ['localhost:9898']

pageserver/Cargo.toml

@@ -7,10 +7,11 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+bookfile = "^0.3"
 chrono = "0.4.19"
 rand = "0.8.3"
 regex = "1.4.5"
-bytes = "1.0.1"
+bytes = { version = "1.0.1", features = ['serde'] }
 byteorder = "1.4.3"
 futures = "0.3.13"
 lazy_static = "1.4.0"
@@ -21,27 +22,29 @@ slog-term = "2.8.0"
 slog = "2.7.0"
 log = "0.4.14"
 clap = "2.33.0"
-termion = "1.5.6"
-tui = "0.14.0"
 daemonize = "0.4.1"
 rust-s3 = { version = "0.27.0-rc4", features = ["no-verify-ssl"] }
-tokio = { version = "1.3.0", features = ["full"] }
-tokio-stream = { version = "0.1.4" }
+tokio = { version = "1.5.0", features = ["full"] }
+tokio-stream = { version = "0.1.5" }
 postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-rocksdb = "0.16.0"
+# by default rust-rocksdb tries to build a lot of compression algos. Use lz4 only for now as it is simplest dependency.
+rocksdb = { version = "0.16.0", features = ["lz4"], default-features = false }
 anyhow = "1.0"
 crc32c = "0.6.0"
 walkdir = "2"
 thiserror = "1.0"
 hex = "0.4.3"
 tar = "0.4.33"
-parse_duration = "2.1.1"
+humantime = "2.1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 fs_extra = "1.2.0"
+toml = "0.5"
+scopeguard = "1.1.0"
 
 postgres_ffi = { path = "../postgres_ffi" }
+zenith_metrics = { path = "../zenith_metrics" }
 zenith_utils = { path = "../zenith_utils" }
 workspace_hack = { path = "../workspace_hack" }

pageserver/README

@@ -1,82 +1,4 @@
-Page Server
-===========
-
-
-How to test
------------
-
-
-1. Compile and install Postgres from this repository (there are
-   modifications, so vanilla Postgres won't do)
-
-    ./configure --prefix=/home/heikki/zenith-install
-
-2. Compile the page server
-
-    cd pageserver
-    cargo build
-
-3. Create another "dummy" cluster that will be used by the page server when it applies
-   the WAL records. (shouldn't really need this, getting rid of it is a TODO):
-
-    /home/heikki/zenith-install/bin/initdb -D /data/zenith-dummy
-
-
-4. Initialize and start a new postgres cluster
-
-    /home/heikki/zenith-install/bin/initdb -D /data/zenith-test-db --username=postgres
-    /home/heikki/zenith-install/bin/postgres -D /data/zenith-test-db
-
-5. In another terminal, start the page server.
-
-    PGDATA=/data/zenith-dummy PATH=/home/heikki/zenith-install/bin:$PATH ./target/debug/pageserver
-
-   It should connect to the postgres instance using streaming replication, and print something
-   like this:
-
-    $ PGDATA=/data/zenith-dummy PATH=/home/heikki/zenith-install/bin:$PATH ./target/debug/pageserver
-    Starting WAL receiver
-    connecting...
-    Starting page server on 127.0.0.1:5430
-    connected!
-    page cache is empty
-
-6. You can now open another terminal and issue DDL commands. Generated WAL records will
-   be streamed to the page servers, and attached to blocks that they apply to in its
-   page cache
-
-    $ psql postgres -U postgres
-    psql (14devel)
-    Type "help" for help.
-    
-    postgres=# create table mydata (i int4);
-    CREATE TABLE
-    postgres=# insert into mydata select g from generate_series(1,100) g;
-    INSERT 0 100
-    postgres=# 
-
-7. The GetPage@LSN interface to the compute nodes isn't working yet, but to simulate
-   that, the page server generates a test GetPage@LSN call every 5 seconds on a random
-   block that's in the page cache. In a few seconds, you should see output from that:
-
-    testing GetPage@LSN for block 0
-    WAL record at LSN 23584576 initializes the page
-    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167DF40
-    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167DF80
-    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167DFC0
-    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E018
-    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E058
-    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E098
-    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E0D8
-    2021-03-19 11:03:13.792 EET [11439] LOG:  applied WAL record at 0/167E118
-    2021-03-19 11:03:13.792 EET [11439] LOG:  applied WAL record at 0/167E158
-    2021-03-19 11:03:13.792 EET [11439] LOG:  applied WAL record at 0/167E198
-    applied 10 WAL records to produce page image at LSN 18446744073709547246
-
-
-
-Architecture
-============
+## Page server architecture
 
 The Page Server is responsible for all operations on a number of
 "chunks" of relation data. A chunk corresponds to a PostgreSQL
@@ -84,8 +6,10 @@ relation segment (i.e. one max. 1 GB file in the data directory), but
 it holds all the different versions of every page in the segment that
 are still needed by the system.
 
-Determining which chunk each Page Server holds is handled elsewhere. (TODO:
-currently, there is only one Page Server which holds all chunks)
+Currently we do not specifically organize data in chunks.
+All page images and corresponding WAL records are stored as entries in a key-value storage,
+where StorageKey is a zenith_timeline_id + BufferTag + LSN.
+
 
 The Page Server has a few different duties:
 
@@ -154,11 +78,33 @@ and stores them to the page cache.
 Page Cache
 ----------
 
-The Page Cache is a data structure, to hold all the different page versions.
-It is accessed by all the other threads, to perform their duties.
+The Page Cache is a switchboard to access different Repositories.
 
-Currently, the page cache is implemented fully in-memory. TODO: Store it
-on disk. Define a file format.
+#### Repository
+Repository corresponds to one .zenith directory.
+Repository is needed to manage Timelines.
+
+#### Timeline
+Timeline is a page cache workhorse that accepts page changes
+and serves get_page_at_lsn() and get_rel_size() requests.
+Note: this has nothing to do with PostgreSQL WAL timeline.
+
+#### Branch
+We can create branch at certain LSN.
+Each Branch lives in a corresponding timeline and has an ancestor.
+
+To get full snapshot of data at certain moment we need to traverse timeline and its ancestors.
+
+#### ObjectRepository
+ObjectRepository implements Repository and has associated ObjectStore and WAL redo service.
+
+#### ObjectStore
+ObjectStore is an interface for key-value store for page images and wal records.
+Currently it has one implementation - RocksDB.
+
+#### WAL redo service
+WAL redo service - service that runs PostgreSQL in a special wal_redo mode
+to apply given WAL records over an old page image and return new page image.
 
 
 TODO: Garbage Collection / Compaction
@@ -177,3 +123,7 @@ The backup service is responsible for periodically pushing the chunks to S3.
 TODO: How/when do restore from S3? Whenever we get a GetPage@LSN request for
 a chunk we don't currently have? Or when an external Control Plane tells us?
 
+TODO: Sharding
+--------------------
+
+We should be able to run multiple Page Servers that handle sharded data.

pageserver/launch.sh

@@ -1,62 +0,0 @@
-#!/bin/sh
-#
-# Set up a simple Compute Node + Page Server combination locally.
-#
-# NOTE: This doesn't clean up between invocations. You'll need to manually:
-#
-# - Kill any previous 'postgres' and 'pageserver' processes
-# - Clear the S3 bucket
-# - Remove the 'zenith-pgdata' directory
-
-
-set -e
-
-# Set up some config.
-#
-# CHANGE THESE ACCORDING TO YOUR S3 INSTALLATION
-export S3_REGION=auto
-export S3_ENDPOINT=https://localhost:9000
-export S3_ACCESSKEY=minioadmin
-export S3_SECRET=pikkunen
-export S3_BUCKET=zenith-testbucket
-
-
-COMPUTE_NODE_PGDATA=zenith-pgdata
-
-
-# 1. Initialize a cluster.
-initdb -D $COMPUTE_NODE_PGDATA -U zenith
-
-echo "port=65432" >> $COMPUTE_NODE_PGDATA/postgresql.conf
-echo "log_connections=on" >> $COMPUTE_NODE_PGDATA/postgresql.conf
-
-# Use a small shared_buffers, so that we hit the Page Server more
-# easily.
-echo "shared_buffers = 1MB" >> $COMPUTE_NODE_PGDATA/postgresql.conf
-
-# TODO: page server should use a replication slot, or some other mechanism
-# to make sure that the primary doesn't lose data that the page server still
-# needs. (The WAL safekeepers should ensure that)
-echo "wal_keep_size=10GB" >> $COMPUTE_NODE_PGDATA/postgresql.conf
-
-# Tell the Postgres server how to connect to the Page Server
-echo "page_server_connstring='host=localhost port=5430'" >> $COMPUTE_NODE_PGDATA/postgresql.conf
-
-
-# 2. Run zenith_push to push a base backup fo the database to an S3 bucket. The
-# Page Server will read it from there
-zenith_push -D $COMPUTE_NODE_PGDATA
-
-
-# 3. Launch page server
-rm -rf /tmp/pgdata-dummy
-initdb -N -D /tmp/pgdata-dummy
-PGDATA=/tmp/pgdata-dummy ./target/debug/pageserver  &
-
-# 4. Start up the Postgres server
-postgres -D $COMPUTE_NODE_PGDATA &
-
-
-echo "ALL SET! You can now connect to Postgres with something like:"
-echo ""
-echo 'psql "dbname=postgres host=localhost user=zenith port=65432"'

pageserver/src/basebackup.rs

@@ -1,23 +1,268 @@
-use crate::ZTimelineId;
+//!
+//! Generate a tarball with files needed to bootstrap ComputeNode.
+//!
+//! TODO: this module has nothing to do with PostgreSQL pg_basebackup.
+//! It could use a better name.
+//!
+//! Stateless Postgres compute node is launched by sending a tarball
+//! which contains non-relational data (multixacts, clog, filenodemaps, twophase files),
+//! generated pg_control and dummy segment of WAL.
+//! This module is responsible for creation of such tarball
+//! from data stored in object storage.
+//!
+use bytes::{BufMut, BytesMut};
 use log::*;
+use std::io;
 use std::io::Write;
 use std::sync::Arc;
 use std::time::SystemTime;
-use tar::{Builder, Header};
-use walkdir::WalkDir;
-use bytes::{BufMut, BytesMut};
+use tar::{Builder, EntryType, Header};
 
-use crate::repository::{BufferTag, RelTag, Timeline};
-use postgres_ffi::relfile_utils::*;
+use crate::relish::*;
+use crate::repository::Timeline;
+use postgres_ffi::xlog_utils::*;
 use postgres_ffi::*;
 use zenith_utils::lsn::Lsn;
 
+/// This is short-living object only for the time of tarball creation,
+/// created mostly to avoid passing a lot of parameters between various functions
+/// used for constructing tarball.
+pub struct Basebackup<'a> {
+    ar: Builder<&'a mut dyn Write>,
+    timeline: &'a Arc<dyn Timeline>,
+    lsn: Lsn,
+    prev_record_lsn: Lsn,
+}
+
+impl<'a> Basebackup<'a> {
+    pub fn new(
+        write: &'a mut dyn Write,
+        timeline: &'a Arc<dyn Timeline>,
+        lsn: Lsn,
+        prev_record_lsn: Lsn,
+    ) -> Basebackup<'a> {
+        Basebackup {
+            ar: Builder::new(write),
+            timeline,
+            lsn,
+            prev_record_lsn,
+        }
+    }
+
+    pub fn send_tarball(&mut self) -> anyhow::Result<()> {
+        // Create pgdata subdirs structure
+        for dir in pg_constants::PGDATA_SUBDIRS.iter() {
+            info!("send subdir {:?}", *dir);
+            let header = new_tar_header_dir(*dir)?;
+            self.ar.append(&header, &mut io::empty())?;
+        }
+
+        // Send empty config files.
+        for filepath in pg_constants::PGDATA_SPECIAL_FILES.iter() {
+            if *filepath == "pg_hba.conf" {
+                let data = pg_constants::PG_HBA.as_bytes();
+                let header = new_tar_header(&filepath, data.len() as u64)?;
+                self.ar.append(&header, &data[..])?;
+            } else {
+                let header = new_tar_header(&filepath, 0)?;
+                self.ar.append(&header, &mut io::empty())?;
+            }
+        }
+
+        // Gather non-relational files from object storage pages.
+        for obj in self.timeline.list_nonrels(self.lsn)? {
+            match obj {
+                RelishTag::Slru { slru, segno } => {
+                    self.add_slru_segment(slru, segno)?;
+                }
+                RelishTag::FileNodeMap { spcnode, dbnode } => {
+                    self.add_relmap_file(spcnode, dbnode)?;
+                }
+                RelishTag::TwoPhase { xid } => {
+                    self.add_twophase_file(xid)?;
+                }
+                _ => {}
+            }
+        }
+
+        // Generate pg_control and bootstrap WAL segment.
+        self.add_pgcontrol_file()?;
+        self.ar.finish()?;
+        debug!("all tarred up!");
+        Ok(())
+    }
+
+    //
+    // Generate SLRU segment files from repository.
+    //
+    fn add_slru_segment(&mut self, slru: SlruKind, segno: u32) -> anyhow::Result<()> {
+        let seg_size = self
+            .timeline
+            .get_relish_size(RelishTag::Slru { slru, segno }, self.lsn)?;
+
+        if seg_size == None {
+            trace!(
+                "SLRU segment {}/{:>04X} was truncated",
+                slru.to_str(),
+                segno
+            );
+            return Ok(());
+        }
+
+        let nblocks = seg_size.unwrap();
+
+        let mut slru_buf: Vec<u8> =
+            Vec::with_capacity(nblocks as usize * pg_constants::BLCKSZ as usize);
+        for blknum in 0..nblocks {
+            let img = self.timeline.get_page_at_lsn_nowait(
+                RelishTag::Slru { slru, segno },
+                blknum,
+                self.lsn,
+            )?;
+            assert!(img.len() == pg_constants::BLCKSZ as usize);
+
+            slru_buf.extend_from_slice(&img);
+        }
+
+        let segname = format!("{}/{:>04X}", slru.to_str(), segno);
+        let header = new_tar_header(&segname, slru_buf.len() as u64)?;
+        self.ar.append(&header, slru_buf.as_slice())?;
+
+        trace!("Added to basebackup slru {} relsize {}", segname, nblocks);
+        Ok(())
+    }
+
+    //
+    // Extract pg_filenode.map files from repository
+    // Along with them also send PG_VERSION for each database.
+    //
+    fn add_relmap_file(&mut self, spcnode: u32, dbnode: u32) -> anyhow::Result<()> {
+        let img = self.timeline.get_page_at_lsn_nowait(
+            RelishTag::FileNodeMap { spcnode, dbnode },
+            0,
+            self.lsn,
+        )?;
+        let path = if spcnode == pg_constants::GLOBALTABLESPACE_OID {
+            let dst_path = "PG_VERSION";
+            let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
+            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
+            self.ar.append(&header, &version_bytes[..])?;
+
+            let dst_path = format!("global/PG_VERSION");
+            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
+            self.ar.append(&header, &version_bytes[..])?;
+
+            String::from("global/pg_filenode.map") // filenode map for global tablespace
+        } else {
+            // User defined tablespaces are not supported
+            assert!(spcnode == pg_constants::DEFAULTTABLESPACE_OID);
+
+            // Append dir path for each database
+            let path = format!("base/{}", dbnode);
+            let header = new_tar_header_dir(&path)?;
+            self.ar.append(&header, &mut io::empty())?;
+
+            let dst_path = format!("base/{}/PG_VERSION", dbnode);
+            let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
+            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
+            self.ar.append(&header, &version_bytes[..])?;
+
+            format!("base/{}/pg_filenode.map", dbnode)
+        };
+        assert!(img.len() == 512);
+        let header = new_tar_header(&path, img.len() as u64)?;
+        self.ar.append(&header, &img[..])?;
+        Ok(())
+    }
+
+    //
+    // Extract twophase state files
+    //
+    fn add_twophase_file(&mut self, xid: TransactionId) -> anyhow::Result<()> {
+        if let Ok(img) =
+            self.timeline
+                .get_page_at_lsn_nowait(RelishTag::TwoPhase { xid }, 0, self.lsn)
+        {
+            let mut buf = BytesMut::new();
+            buf.extend_from_slice(&img[..]);
+            let crc = crc32c::crc32c(&img[..]);
+            buf.put_u32_le(crc);
+            let path = format!("pg_twophase/{:>08X}", xid);
+            let header = new_tar_header(&path, buf.len() as u64)?;
+            self.ar.append(&header, &buf[..])?;
+        }
+        Ok(())
+    }
+
+    //
+    // Add generated pg_control file and bootstrap WAL segment.
+    // Also send zenith.signal file with extra bootstrap data.
+    //
+    fn add_pgcontrol_file(&mut self) -> anyhow::Result<()> {
+        let checkpoint_bytes =
+            self.timeline
+                .get_page_at_lsn_nowait(RelishTag::Checkpoint, 0, self.lsn)?;
+        let pg_control_bytes =
+            self.timeline
+                .get_page_at_lsn_nowait(RelishTag::ControlFile, 0, self.lsn)?;
+        let mut pg_control = ControlFileData::decode(&pg_control_bytes)?;
+        let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
+
+        // Generate new pg_control and WAL needed for bootstrap
+        let checkpoint_segno = self.lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE);
+        let checkpoint_lsn = XLogSegNoOffsetToRecPtr(
+            checkpoint_segno,
+            XLOG_SIZE_OF_XLOG_LONG_PHD as u32,
+            pg_constants::WAL_SEGMENT_SIZE,
+        );
+        checkpoint.redo = self.lsn.0 + self.lsn.calc_padding(8u32);
+
+        //reset some fields we don't want to preserve
+        //TODO Check this.
+        //We may need to determine the value from twophase data.
+        checkpoint.oldestActiveXid = 0;
+
+        //save new values in pg_control
+        pg_control.checkPoint = checkpoint_lsn;
+        pg_control.checkPointCopy = checkpoint;
+        pg_control.state = pg_constants::DB_SHUTDOWNED;
+
+        // add zenith.signal file
+        self.ar.append(
+            &new_tar_header("zenith.signal", 8)?,
+            &self.prev_record_lsn.0.to_le_bytes()[..],
+        )?;
+
+        //send pg_control
+        let pg_control_bytes = pg_control.encode();
+        let header = new_tar_header("global/pg_control", pg_control_bytes.len() as u64)?;
+        self.ar.append(&header, &pg_control_bytes[..])?;
+
+        //send wal segment
+        let wal_file_name = XLogFileName(
+            1, // FIXME: always use Postgres timeline 1
+            checkpoint_segno,
+            pg_constants::WAL_SEGMENT_SIZE,
+        );
+        let wal_file_path = format!("pg_wal/{}", wal_file_name);
+        let header = new_tar_header(&wal_file_path, pg_constants::WAL_SEGMENT_SIZE as u64)?;
+        let wal_seg = generate_wal_segment(&pg_control);
+        assert!(wal_seg.len() == pg_constants::WAL_SEGMENT_SIZE);
+        self.ar.append(&header, &wal_seg[..])?;
+        Ok(())
+    }
+}
+
+//
+// Create new tarball entry header
+//
 fn new_tar_header(path: &str, size: u64) -> anyhow::Result<Header> {
     let mut header = Header::new_gnu();
     header.set_size(size);
     header.set_path(path)?;
-    header.set_mode(0b110000000);
+    header.set_mode(0b110000000); // -rw-------
     header.set_mtime(
+        // use currenttime as last modified time
         SystemTime::now()
             .duration_since(SystemTime::UNIX_EPOCH)
             .unwrap()
@@ -27,373 +272,19 @@ fn new_tar_header(path: &str, size: u64) -> anyhow::Result<Header> {
     Ok(header)
 }
 
-//
-// Generate SRLU segment files from repository
-//
-fn add_slru_segments(
-    ar: &mut Builder<&mut dyn Write>,
-    timeline: &Arc<dyn Timeline>,
-    path: &str,
-    forknum: u8,
-    lsn: Lsn,
-) -> anyhow::Result<()> {
-    let rel = RelTag {
-        spcnode: 0,
-        dbnode: 0,
-        relnode: 0,
-        forknum,
-    };
-    let (first, last) = timeline.get_range(rel, lsn)?;
-    const SEG_SIZE: usize =
-        pg_constants::BLCKSZ as usize * pg_constants::SLRU_PAGES_PER_SEGMENT as usize;
-    let mut seg_buf = [0u8; SEG_SIZE];
-    let mut curr_segno: Option<u32> = None;
-    for page in first..last {
-        let tag = BufferTag { rel, blknum: page };
-        let img = timeline.get_page_at_lsn(tag, lsn)?;
-        // Zero length image indicates truncated segment: just skip it
-        if img.len() != 0 {
-            assert!(img.len() == pg_constants::BLCKSZ as usize);
-
-            let segno = page / pg_constants::SLRU_PAGES_PER_SEGMENT;
-            if curr_segno.is_some() && curr_segno.unwrap() != segno {
-                let segname = format!("{}/{:>04X}", path, curr_segno.unwrap());
-                let header = new_tar_header(&segname, SEG_SIZE as u64)?;
-                ar.append(&header, &seg_buf[..])?;
-                seg_buf = [0u8; SEG_SIZE];
-            }
-            curr_segno = Some(segno);
-            let offs_start = (page % pg_constants::SLRU_PAGES_PER_SEGMENT) as usize
-                * pg_constants::BLCKSZ as usize;
-            let offs_end = offs_start + pg_constants::BLCKSZ as usize;
-            seg_buf[offs_start..offs_end].copy_from_slice(&img);
-        }
-    }
-    if curr_segno.is_some() {
-        let segname = format!("{}/{:>04X}", path, curr_segno.unwrap());
-        let header = new_tar_header(&segname, SEG_SIZE as u64)?;
-        ar.append(&header, &seg_buf[..])?;
-    }
-    Ok(())
-}
-
-//
-// Extract pg_filenode.map files from repository
-//
-fn add_relmap_files(
-    ar: &mut Builder<&mut dyn Write>,
-    timeline: &Arc<dyn Timeline>,
-    lsn: Lsn,
-    snappath: &str,
-) -> anyhow::Result<()> {
-    for db in timeline.get_databases(lsn)?.iter() {
-        let tag = BufferTag {
-            rel: *db,
-            blknum: 0,
-        };
-        let img = timeline.get_page_at_lsn(tag, lsn)?;
-        let path = if db.spcnode == pg_constants::GLOBALTABLESPACE_OID {
-            String::from("global/pg_filenode.map")
-        } else {
-            // User defined tablespaces are not supported
-            assert!(db.spcnode == pg_constants::DEFAULTTABLESPACE_OID);
-            let src_path = format!("{}/base/1/PG_VERSION", snappath);
-            let dst_path = format!("base/{}/PG_VERSION", db.dbnode);
-            ar.append_path_with_name(&src_path, &dst_path)?;
-            format!("base/{}/pg_filenode.map", db.dbnode)
-        };
-        assert!(img.len() == 512);
-        let header = new_tar_header(&path, img.len() as u64)?;
-        ar.append(&header, &img[..])?;
-    }
-    Ok(())
-}
-
-//
-// Extract twophase state files
-//
-fn add_twophase_files(
-    ar: &mut Builder<&mut dyn Write>,
-    timeline: &Arc<dyn Timeline>,
-    lsn: Lsn,
-) -> anyhow::Result<()> {
-    for xid in timeline.get_twophase(lsn)?.iter() {
-        let tag = BufferTag {
-            rel: RelTag {
-                spcnode: 0,
-                dbnode: 0,
-                relnode: 0,
-                forknum: pg_constants::PG_TWOPHASE_FORKNUM,
-            },
-            blknum: *xid,
-        };
-        let img = timeline.get_page_at_lsn(tag, lsn)?;
-		let mut buf = BytesMut::new();
-		buf.extend_from_slice(&img[..]);
-		let crc = crc32c::crc32c(&img[..]);
-		buf.put_u32_le(crc);
-        let path = format!("pg_twophase/{:>08X}", xid);
-        let header = new_tar_header(&path, buf.len() as u64)?;
-        ar.append(&header, &buf[..])?;
-    }
-    Ok(())
-}
-
-//
-// Add generated pg_control file
-//
-fn add_pgcontrol_file(
-    ar: &mut Builder<&mut dyn Write>,
-    timeline: &Arc<dyn Timeline>,
-    lsn: Lsn,
-) -> anyhow::Result<()> {
-    if let Some(checkpoint_bytes) =
-        timeline.get_page_image(BufferTag::fork(pg_constants::PG_CHECKPOINT_FORKNUM), Lsn(0))?
-    {
-        if let Some(pg_control_bytes) = timeline.get_page_image(
-            BufferTag::fork(pg_constants::PG_CONTROLFILE_FORKNUM),
-            Lsn(0),
-        )? {
-            let mut pg_control = postgres_ffi::decode_pg_control(pg_control_bytes)?;
-            let mut checkpoint = postgres_ffi::decode_checkpoint(checkpoint_bytes)?;
-
-            checkpoint.redo = lsn.0;
-            checkpoint.nextXid.value += 1;
-            // TODO: When we restart master there are no active transaction and oldestXid is
-            // equal to nextXid if there are no prepared transactions.
-            // Let's ignore them for a while...
-            checkpoint.oldestXid = checkpoint.nextXid.value as u32;
-            pg_control.checkPointCopy = checkpoint;
-            let pg_control_bytes = postgres_ffi::encode_pg_control(pg_control);
-            let header = new_tar_header("global/pg_control", pg_control_bytes.len() as u64)?;
-            ar.append(&header, &pg_control_bytes[..])?;
-        }
-    }
-    Ok(())
-}
-
-///
-/// Generate tarball with non-relational files from repository
-///
-pub fn send_tarball_at_lsn(
-    write: &mut dyn Write,
-    timelineid: ZTimelineId,
-    timeline: &Arc<dyn Timeline>,
-    lsn: Lsn,
-    snapshot_lsn: Lsn,
-) -> anyhow::Result<()> {
-    let mut ar = Builder::new(write);
-
-    let snappath = format!("timelines/{}/snapshots/{:016X}", timelineid, snapshot_lsn.0);
-
-    debug!("sending tarball of snapshot in {}", snappath);
-    for entry in WalkDir::new(&snappath) {
-        let entry = entry?;
-        let fullpath = entry.path();
-        let relpath = entry.path().strip_prefix(&snappath).unwrap();
-
-        if relpath.to_str().unwrap() == "" {
-            continue;
-        }
-
-        if entry.file_type().is_dir() {
-            trace!(
-                "sending dir {} as {}",
-                fullpath.display(),
-                relpath.display()
-            );
-            ar.append_dir(relpath, fullpath)?;
-        } else if entry.file_type().is_symlink() {
-            error!("ignoring symlink in snapshot dir");
-        } else if entry.file_type().is_file() {
-            // Shared catalogs are exempt
-            if relpath.starts_with("global/") {
-                trace!("sending shared catalog {}", relpath.display());
-                ar.append_path_with_name(fullpath, relpath)?;
-            } else if !is_rel_file_path(relpath.to_str().unwrap()) {
-                if entry.file_name() != "pg_filenode.map"
-                    && entry.file_name() != "pg_control"
-                    && !relpath.starts_with("pg_xact/")
-                    && !relpath.starts_with("pg_multixact/")
-                {
-                    trace!("sending {}", relpath.display());
-                    ar.append_path_with_name(fullpath, relpath)?;
-                }
-            } else {
-                trace!("not sending {}", relpath.display());
-            }
-        } else {
-            error!("unknown file type: {}", fullpath.display());
-        }
-    }
-
-    add_slru_segments(
-        &mut ar,
-        timeline,
-        "pg_xact",
-        pg_constants::PG_XACT_FORKNUM,
-        lsn,
-    )?;
-    add_slru_segments(
-        &mut ar,
-        timeline,
-        "pg_multixact/members",
-        pg_constants::PG_MXACT_MEMBERS_FORKNUM,
-        lsn,
-    )?;
-    add_slru_segments(
-        &mut ar,
-        timeline,
-        "pg_multixact/offsets",
-        pg_constants::PG_MXACT_OFFSETS_FORKNUM,
-        lsn,
-    )?;
-    add_relmap_files(&mut ar, timeline, lsn, &snappath)?;
-    add_twophase_files(&mut ar, timeline, lsn)?;
-    add_pgcontrol_file(&mut ar, timeline, lsn)?;
-
-    ar.finish()?;
-    debug!("all tarred up!");
-    Ok(())
-}
-
-///
-/// Send a tarball containing a snapshot of all non-relation files in the
-/// PostgreSQL data directory, at given LSN
-///
-/// There must be a snapshot at the given LSN in the snapshots directory, we cannot
-/// reconstruct the state at an arbitrary LSN at the moment.
-///
-pub fn send_snapshot_tarball(
-    write: &mut dyn Write,
-    timelineid: ZTimelineId,
-    snapshotlsn: Lsn,
-) -> Result<(), std::io::Error> {
-    let mut ar = Builder::new(write);
-
-    let snappath = format!("timelines/{}/snapshots/{:016X}", timelineid, snapshotlsn.0);
-    let walpath = format!("timelines/{}/wal", timelineid);
-
-    debug!("sending tarball of snapshot in {}", snappath);
-    //ar.append_dir_all("", &snappath)?;
-
-    for entry in WalkDir::new(&snappath) {
-        let entry = entry?;
-        let fullpath = entry.path();
-        let relpath = entry.path().strip_prefix(&snappath).unwrap();
-
-        if relpath.to_str().unwrap() == "" {
-            continue;
-        }
-
-        if entry.file_type().is_dir() {
-            trace!(
-                "sending dir {} as {}",
-                fullpath.display(),
-                relpath.display()
-            );
-            ar.append_dir(relpath, fullpath)?;
-        } else if entry.file_type().is_symlink() {
-            error!("ignoring symlink in snapshot dir");
-        } else if entry.file_type().is_file() {
-            // Shared catalogs are exempt
-            if relpath.starts_with("global/") {
-                trace!("sending shared catalog {}", relpath.display());
-                ar.append_path_with_name(fullpath, relpath)?;
-            } else if !is_rel_file_path(relpath.to_str().unwrap()) {
-                trace!("sending {}", relpath.display());
-                ar.append_path_with_name(fullpath, relpath)?;
-            } else {
-                trace!("not sending {}", relpath.display());
-
-                // FIXME: For now, also send all the relation files.
-                // This really shouldn't be necessary, and kind of
-                // defeats the point of having a page server in the
-                // first place. But it is useful at least when
-                // debugging with the DEBUG_COMPARE_LOCAL option (see
-                // vendor/postgres/src/backend/storage/smgr/pagestore_smgr.c)
-
-                ar.append_path_with_name(fullpath, relpath)?;
-            }
-        } else {
-            error!("unknown file type: {}", fullpath.display());
-        }
-    }
-
-    // FIXME: Also send all the WAL. The compute node would only need
-    // the WAL that applies to non-relation files, because the page
-    // server handles all the relation files. But we don't have a
-    // mechanism for separating relation and non-relation WAL at the
-    // moment.
-    for entry in std::fs::read_dir(&walpath)? {
-        let entry = entry?;
-        let fullpath = &entry.path();
-        let relpath = fullpath.strip_prefix(&walpath).unwrap();
-
-        if !entry.path().is_file() {
-            continue;
-        }
-
-        let archive_fname = relpath.to_str().unwrap();
-        let archive_fname = archive_fname
-            .strip_suffix(".partial")
-            .unwrap_or(&archive_fname);
-        let archive_path = "pg_wal/".to_owned() + archive_fname;
-        ar.append_path_with_name(fullpath, archive_path)?;
-    }
-
-    ar.finish()?;
-    debug!("all tarred up!");
-    Ok(())
-}
-
-///
-/// Parse a path, relative to the root of PostgreSQL data directory, as
-/// a PostgreSQL relation data file.
-///
-fn parse_rel_file_path(path: &str) -> Result<(), FilePathError> {
-    /*
-     * Relation data files can be in one of the following directories:
-     *
-     * global/
-     *		shared relations
-     *
-     * base/<db oid>/
-     *		regular relations, default tablespace
-     *
-     * pg_tblspc/<tblspc oid>/<tblspc version>/
-     *		within a non-default tablespace (the name of the directory
-     *		depends on version)
-     *
-     * And the relation data files themselves have a filename like:
-     *
-     * <oid>.<segment number>
-     */
-    if let Some(fname) = path.strip_prefix("global/") {
-        let (_relnode, _forknum, _segno) = parse_relfilename(fname)?;
-
-        Ok(())
-    } else if let Some(dbpath) = path.strip_prefix("base/") {
-        let mut s = dbpath.split('/');
-        let dbnode_str = s.next().ok_or(FilePathError::InvalidFileName)?;
-        let _dbnode = dbnode_str.parse::<u32>()?;
-        let fname = s.next().ok_or(FilePathError::InvalidFileName)?;
-        if s.next().is_some() {
-            return Err(FilePathError::InvalidFileName);
-        };
-
-        let (_relnode, _forknum, _segno) = parse_relfilename(fname)?;
-
-        Ok(())
-    } else if let Some(_) = path.strip_prefix("pg_tblspc/") {
-        // TODO
-        error!("tablespaces not implemented yet");
-        Err(FilePathError::InvalidFileName)
-    } else {
-        Err(FilePathError::InvalidFileName)
-    }
-}
-
-fn is_rel_file_path(path: &str) -> bool {
-    parse_rel_file_path(path).is_ok()
+fn new_tar_header_dir(path: &str) -> anyhow::Result<Header> {
+    let mut header = Header::new_gnu();
+    header.set_size(0);
+    header.set_path(path)?;
+    header.set_mode(0o755); // -rw-------
+    header.set_entry_type(EntryType::dir());
+    header.set_mtime(
+        // use currenttime as last modified time
+        SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap()
+            .as_secs(),
+    );
+    header.set_cksum();
+    Ok(header)
 }

pageserver/src/bin/pageserver.rs

@@ -3,29 +3,170 @@
 //
 
 use log::*;
-use parse_duration::parse;
-use std::io;
-use std::process::exit;
-use std::thread;
-use std::time::Duration;
-use std::{env, path::PathBuf};
+use serde::{Deserialize, Serialize};
 use std::{
-    fs::{File, OpenOptions},
+    env,
     net::TcpListener,
+    path::{Path, PathBuf},
+    process::exit,
+    str::FromStr,
+    sync::Arc,
+    thread,
+    time::Duration,
 };
+use zenith_utils::{auth::JwtAuth, postgres_backend::AuthType};
 
-use anyhow::{Context, Result};
-use clap::{App, Arg};
+use anyhow::{ensure, Result};
+use clap::{App, Arg, ArgMatches};
 use daemonize::Daemonize;
 
-use slog::{Drain, FnValue};
+use pageserver::{branches, logger, page_cache, page_service, PageServerConf, RepositoryFormat};
+use zenith_utils::http_endpoint;
 
-use pageserver::{branches, page_cache, page_service, tui, PageServerConf};
+const DEFAULT_LISTEN_ADDR: &str = "127.0.0.1:64000";
+const DEFAULT_HTTP_ENDPOINT_ADDR: &str = "127.0.0.1:9898";
 
 const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
-const DEFAULT_GC_PERIOD_SEC: u64 = 10;
-//const DEFAULT_GC_HORIZON: u64 = 1024 * 1024 * 1024;
-//const DEFAULT_GC_PERIOD_SEC: u64 = 600;
+const DEFAULT_GC_PERIOD: Duration = Duration::from_secs(10);
+
+const DEFAULT_SUPERUSER: &str = "zenith_admin";
+
+/// String arguments that can be declared via CLI or config file
+#[derive(Serialize, Deserialize)]
+struct CfgFileParams {
+    listen_addr: Option<String>,
+    http_endpoint_addr: Option<String>,
+    gc_horizon: Option<String>,
+    gc_period: Option<String>,
+    pg_distrib_dir: Option<String>,
+    auth_validation_public_key_path: Option<String>,
+    auth_type: Option<String>,
+    repository_format: Option<String>,
+}
+
+impl CfgFileParams {
+    /// Extract string arguments from CLI
+    fn from_args(arg_matches: &ArgMatches) -> Self {
+        let get_arg = |arg_name: &str| -> Option<String> {
+            arg_matches.value_of(arg_name).map(str::to_owned)
+        };
+
+        Self {
+            listen_addr: get_arg("listen"),
+            http_endpoint_addr: get_arg("http_endpoint"),
+            gc_horizon: get_arg("gc_horizon"),
+            gc_period: get_arg("gc_period"),
+            pg_distrib_dir: get_arg("postgres-distrib"),
+            auth_validation_public_key_path: get_arg("auth-validation-public-key-path"),
+            auth_type: get_arg("auth-type"),
+            repository_format: get_arg("repository-format"),
+        }
+    }
+
+    /// Fill missing values in `self` with `other`
+    fn or(self, other: CfgFileParams) -> Self {
+        // TODO cleaner way to do this
+        Self {
+            listen_addr: self.listen_addr.or(other.listen_addr),
+            http_endpoint_addr: self.http_endpoint_addr.or(other.http_endpoint_addr),
+            gc_horizon: self.gc_horizon.or(other.gc_horizon),
+            gc_period: self.gc_period.or(other.gc_period),
+            pg_distrib_dir: self.pg_distrib_dir.or(other.pg_distrib_dir),
+            auth_validation_public_key_path: self
+                .auth_validation_public_key_path
+                .or(other.auth_validation_public_key_path),
+            auth_type: self.auth_type.or(other.auth_type),
+            repository_format: self.repository_format.or(other.repository_format),
+        }
+    }
+
+    /// Create a PageServerConf from these string parameters
+    fn try_into_config(&self) -> Result<PageServerConf> {
+        let workdir = PathBuf::from(".");
+
+        let listen_addr = match self.listen_addr.as_ref() {
+            Some(addr) => addr.clone(),
+            None => DEFAULT_LISTEN_ADDR.to_owned(),
+        };
+
+        let http_endpoint_addr = match self.http_endpoint_addr.as_ref() {
+            Some(addr) => addr.clone(),
+            None => DEFAULT_HTTP_ENDPOINT_ADDR.to_owned(),
+        };
+
+        let gc_horizon: u64 = match self.gc_horizon.as_ref() {
+            Some(horizon_str) => horizon_str.parse()?,
+            None => DEFAULT_GC_HORIZON,
+        };
+        let gc_period = match self.gc_period.as_ref() {
+            Some(period_str) => humantime::parse_duration(period_str)?,
+            None => DEFAULT_GC_PERIOD,
+        };
+
+        let pg_distrib_dir = match self.pg_distrib_dir.as_ref() {
+            Some(pg_distrib_dir_str) => PathBuf::from(pg_distrib_dir_str),
+            None => env::current_dir()?.join("tmp_install"),
+        };
+
+        let auth_validation_public_key_path = self
+            .auth_validation_public_key_path
+            .as_ref()
+            .map(PathBuf::from);
+
+        let auth_type = self
+            .auth_type
+            .as_ref()
+            .map_or(Ok(AuthType::Trust), |auth_type| {
+                AuthType::from_str(&auth_type)
+            })?;
+
+        if !pg_distrib_dir.join("bin/postgres").exists() {
+            anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
+        }
+
+        if auth_type == AuthType::ZenithJWT {
+            ensure!(
+                auth_validation_public_key_path.is_some(),
+                "Missing auth_validation_public_key_path when auth_type is ZenithJWT"
+            );
+            let path_ref = auth_validation_public_key_path.as_ref().unwrap();
+            ensure!(
+                path_ref.exists(),
+                format!("Can't find auth_validation_public_key at {:?}", path_ref)
+            );
+        }
+
+        let repository_format = match self.repository_format.as_ref() {
+            Some(repo_format_str) if repo_format_str == "rocksdb" => RepositoryFormat::RocksDb,
+            Some(repo_format_str) if repo_format_str == "layered" => RepositoryFormat::Layered,
+            Some(repo_format_str) => anyhow::bail!(
+                "invalid --repository-format '{}', must be 'rocksdb' or 'layered'",
+                repo_format_str
+            ),
+            None => RepositoryFormat::Layered, // default
+        };
+
+        Ok(PageServerConf {
+            daemonize: false,
+
+            listen_addr,
+            http_endpoint_addr,
+            gc_horizon,
+            gc_period,
+
+            superuser: String::from(DEFAULT_SUPERUSER),
+
+            workdir,
+
+            pg_distrib_dir,
+
+            auth_validation_public_key_path,
+            auth_type,
+
+            repository_format,
+        })
+    }
+}
 
 fn main() -> Result<()> {
     let arg_matches = App::new("Zenith page server")
@@ -37,13 +178,6 @@ fn main() -> Result<()> {
                 .takes_value(true)
                 .help("listen for incoming page requests on ip:port (default: 127.0.0.1:5430)"),
         )
-        .arg(
-            Arg::with_name("interactive")
-                .short("i")
-                .long("interactive")
-                .takes_value(false)
-                .help("Interactive mode"),
-        )
         .arg(
             Arg::with_name("daemonize")
                 .short("d")
@@ -76,115 +210,97 @@ fn main() -> Result<()> {
                 .takes_value(true)
                 .help("Working directory for the pageserver"),
         )
+        .arg(
+            Arg::with_name("postgres-distrib")
+                .long("postgres-distrib")
+                .takes_value(true)
+                .help("Postgres distribution directory"),
+        )
+        .arg(
+            Arg::with_name("create-tenant")
+                .long("create-tenant")
+                .takes_value(true)
+                .help("Create tenant during init")
+                .requires("init"),
+        )
+        .arg(
+            Arg::with_name("auth-validation-public-key-path")
+                .long("auth-validation-public-key-path")
+                .takes_value(true)
+                .help("Path to public key used to validate jwt signature"),
+        )
+        .arg(
+            Arg::with_name("auth-type")
+                .long("auth-type")
+                .takes_value(true)
+                .help("Authentication scheme type. One of: Trust, MD5, ZenithJWT"),
+        )
+        .arg(
+            Arg::with_name("repository-format")
+                .long("repository-format")
+                .takes_value(true)
+                .help("Which repository implementation to use, 'rocksdb' or 'layered'"),
+        )
         .get_matches();
 
-    let workdir = if let Some(workdir_arg) = arg_matches.value_of("workdir") {
-        PathBuf::from(workdir_arg)
-    } else if let Some(workdir_arg) = std::env::var_os("ZENITH_REPO_DIR") {
-        PathBuf::from(workdir_arg.to_str().unwrap())
+    let workdir = Path::new(arg_matches.value_of("workdir").unwrap_or(".zenith"));
+    let cfg_file_path = workdir.canonicalize()?.join("pageserver.toml");
+
+    let args_params = CfgFileParams::from_args(&arg_matches);
+
+    let init = arg_matches.is_present("init");
+    let create_tenant = arg_matches.value_of("create-tenant");
+
+    let params = if init {
+        // We're initializing the repo, so there's no config file yet
+        args_params
     } else {
-        PathBuf::from(".zenith")
+        // Supplement the CLI arguments with the config file
+        let cfg_file_contents = std::fs::read_to_string(&cfg_file_path)?;
+        let file_params: CfgFileParams = toml::from_str(&cfg_file_contents)?;
+        args_params.or(file_params)
     };
 
-    let pg_distrib_dir: PathBuf = {
-        if let Some(postgres_bin) = env::var_os("POSTGRES_DISTRIB_DIR") {
-            postgres_bin.into()
-        } else {
-            let cwd = env::current_dir()?;
-            cwd.join("tmp_install")
-        }
-    };
+    // Set CWD to workdir for non-daemon modes
+    env::set_current_dir(&workdir)?;
 
-    if !pg_distrib_dir.join("bin/postgres").exists() {
-        anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
-    }
+    // Ensure the config is valid, even if just init-ing
+    let mut conf = params.try_into_config()?;
 
-    let mut conf = PageServerConf {
-        daemonize: false,
-        interactive: false,
-        gc_horizon: DEFAULT_GC_HORIZON,
-        gc_period: Duration::from_secs(DEFAULT_GC_PERIOD_SEC),
-        listen_addr: "127.0.0.1:64000".parse().unwrap(),
-        // we will change the current working directory to the repository below,
-        // so always set 'workdir' to '.'
-        workdir: PathBuf::from("."),
-        pg_distrib_dir,
-    };
+    conf.daemonize = arg_matches.is_present("daemonize");
 
-    if arg_matches.is_present("daemonize") {
-        conf.daemonize = true;
-    }
-
-    if arg_matches.is_present("interactive") {
-        conf.interactive = true;
-    }
-
-    if conf.daemonize && conf.interactive {
-        eprintln!("--daemonize is not allowed with --interactive: choose one");
+    if init && conf.daemonize {
+        eprintln!("--daemonize cannot be used with --init");
         exit(1);
     }
 
-    if let Some(addr) = arg_matches.value_of("listen") {
-        conf.listen_addr = addr.parse()?;
-    }
-
-    if let Some(horizon) = arg_matches.value_of("gc_horizon") {
-        conf.gc_horizon = horizon.parse()?;
-    }
-
-    if let Some(period) = arg_matches.value_of("gc_period") {
-        conf.gc_period = parse(period)?;
-    }
-
     // The configuration is all set up now. Turn it into a 'static
     // that can be freely stored in structs and passed across threads
     // as a ref.
     let conf: &'static PageServerConf = Box::leak(Box::new(conf));
 
     // Create repo and exit if init was requested
-    if arg_matches.is_present("init") {
-        branches::init_repo(conf, &workdir)?;
+    if init {
+        branches::init_pageserver(conf, create_tenant)?;
+        // write the config file
+        let cfg_file_contents = toml::to_string_pretty(&params)?;
+        // TODO support enable-auth flag
+        std::fs::write(&cfg_file_path, cfg_file_contents)?;
+
         return Ok(());
     }
 
-    // Set CWD to workdir for non-daemon modes
-    env::set_current_dir(&workdir)?;
-
     start_pageserver(conf)
 }
 
 fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
-    let log_filename = "pageserver.log";
-    // Don't open the same file for output multiple times;
-    // the different fds could overwrite each other's output.
-    let log_file = OpenOptions::new()
-        .create(true)
-        .append(true)
-        .open(&log_filename)
-        .with_context(|| format!("failed to open {:?}", &log_filename))?;
-
     // Initialize logger
-    let logger_file = log_file.try_clone().unwrap();
-    let _scope_guard = init_logging(&conf, logger_file)?;
+    let (_scope_guard, log_file) = logger::init_logging(&conf, "pageserver.log")?;
     let _log_guard = slog_stdlog::init()?;
 
     // Note: this `info!(...)` macro comes from `log` crate
     info!("standard logging redirected to slog");
 
-    let tui_thread = if conf.interactive {
-        // Initialize the UI
-        Some(
-            thread::Builder::new()
-                .name("UI thread".into())
-                .spawn(|| {
-                    let _ = tui::ui_main();
-                })
-                .unwrap(),
-        )
-    } else {
-        None
-    };
-
     // TODO: Check that it looks like a valid repository before going further
 
     if conf.daemonize {
@@ -207,79 +323,40 @@ fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
         }
     }
 
-    // Check that we can bind to address before further initialization
+    // Spawn a new thread for the http endpoint
+    thread::Builder::new()
+        .name("Metrics thread".into())
+        .spawn(move || http_endpoint::thread_main(conf.http_endpoint_addr.clone()))?;
+
+    // Check that we can bind to address before starting threads to simplify shutdown
+    // sequence if port is occupied.
     info!("Starting pageserver on {}", conf.listen_addr);
-    let pageserver_listener = TcpListener::bind(conf.listen_addr)?;
+    let pageserver_listener = TcpListener::bind(conf.listen_addr.clone())?;
 
     // Initialize page cache, this will spawn walredo_thread
     page_cache::init(conf);
 
+    // initialize authentication for incoming connections
+    let auth = match &conf.auth_type {
+        AuthType::Trust | AuthType::MD5 => Arc::new(None),
+        AuthType::ZenithJWT => {
+            // unwrap is ok because check is performed when creating config, so path is set and file exists
+            let key_path = conf.auth_validation_public_key_path.as_ref().unwrap();
+            Arc::new(Some(JwtAuth::from_key_path(key_path)?))
+        }
+    };
+    info!("Using auth: {:#?}", conf.auth_type);
     // Spawn a thread to listen for connections. It will spawn further threads
     // for each connection.
     let page_service_thread = thread::Builder::new()
         .name("Page Service thread".into())
-        .spawn(move || page_service::thread_main(conf, pageserver_listener))?;
+        .spawn(move || {
+            page_service::thread_main(conf, auth, pageserver_listener, conf.auth_type)
+        })?;
 
-    if let Some(tui_thread) = tui_thread {
-        // The TUI thread exits when the user asks to Quit.
-        tui_thread.join().unwrap();
-    } else {
-        page_service_thread
-            .join()
-            .expect("Page service thread has panicked")?
-    }
+    page_service_thread
+        .join()
+        .expect("Page service thread has panicked")?;
 
     Ok(())
 }
-
-fn init_logging(
-    conf: &PageServerConf,
-    log_file: File,
-) -> Result<slog_scope::GlobalLoggerGuard, io::Error> {
-    if conf.interactive {
-        Ok(tui::init_logging())
-    } else if conf.daemonize {
-        let decorator = slog_term::PlainSyncDecorator::new(log_file);
-        let drain = slog_term::FullFormat::new(decorator).build();
-        let drain = slog::Filter::new(drain, |record: &slog::Record| {
-            if record.level().is_at_least(slog::Level::Info) {
-                return true;
-            }
-            false
-        });
-        let drain = std::sync::Mutex::new(drain).fuse();
-        let logger = slog::Logger::root(
-            drain,
-            slog::o!(
-                "location" =>
-                FnValue(move |record| {
-                    format!("{}, {}:{}",
-                            record.module(),
-                            record.file(),
-                            record.line()
-                            )
-                    }
-                )
-            ),
-        );
-        Ok(slog_scope::set_global_logger(logger))
-    } else {
-        let decorator = slog_term::TermDecorator::new().build();
-        let drain = slog_term::FullFormat::new(decorator).build().fuse();
-        let drain = slog_async::Async::new(drain).chan_size(1000).build().fuse();
-        let drain = slog::Filter::new(drain, |record: &slog::Record| {
-            if record.level().is_at_least(slog::Level::Info) {
-                return true;
-            }
-            if record.level().is_at_least(slog::Level::Debug)
-                && record.module().starts_with("pageserver")
-            {
-                return true;
-            }
-            false
-        })
-        .fuse();
-        let logger = slog::Logger::root(drain, slog::o!());
-        Ok(slog_scope::set_global_logger(logger))
-    }
-}

pageserver/src/branches.rs

@@ -1,28 +1,30 @@
-//
-// Branch management code
-//
+//!
+//! Branch management code
+//!
 // TODO: move all paths construction to conf impl
 //
 
-use anyhow::{anyhow, bail, Context, Result};
-use bytes::Bytes;
-use fs::File;
-use fs_extra;
-use postgres_ffi::{pg_constants, xlog_utils};
-use rand::Rng;
+use anyhow::{bail, ensure, Context, Result};
+use postgres_ffi::ControlFileData;
 use serde::{Deserialize, Serialize};
-use std::env;
-use std::io::{Read, Write};
 use std::{
-    collections::HashMap,
-    fs, io,
-    path::{Path, PathBuf},
+    fs,
+    path::Path,
     process::{Command, Stdio},
     str::FromStr,
+    sync::Arc,
 };
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
+
+use log::*;
 use zenith_utils::lsn::Lsn;
 
-use crate::{repository::Repository, PageServerConf, ZTimelineId};
+use crate::logger;
+use crate::object_repository::ObjectRepository;
+use crate::page_cache;
+use crate::restore_local_repo;
+use crate::walredo::WalRedoManager;
+use crate::{repository::Repository, PageServerConf, RepositoryFormat};
 
 #[derive(Serialize, Deserialize, Clone)]
 pub struct BranchInfo {
@@ -39,38 +41,106 @@ pub struct PointInTime {
     pub lsn: Lsn,
 }
 
-pub fn init_repo(conf: &PageServerConf, repo_dir: &Path) -> Result<()> {
+pub fn init_pageserver(conf: &'static PageServerConf, create_tenant: Option<&str>) -> Result<()> {
+    // Initialize logger
+    let (_scope_guard, _log_file) = logger::init_logging(&conf, "pageserver.log")?;
+    let _log_guard = slog_stdlog::init()?;
+
+    if let Some(tenantid) = create_tenant {
+        let tenantid = ZTenantId::from_str(tenantid)?;
+        println!("initializing tenantid {}", tenantid);
+        create_repo(
+            conf,
+            tenantid,
+            Arc::new(crate::walredo::DummyRedoManager {}),
+        )
+        .with_context(|| "failed to create repo")?;
+    }
+    fs::create_dir_all(conf.tenants_path())?;
+
+    println!("pageserver init succeeded");
+    Ok(())
+}
+
+pub fn create_repo(
+    conf: &'static PageServerConf,
+    tenantid: ZTenantId,
+    wal_redo_manager: Arc<dyn WalRedoManager + Send + Sync>,
+) -> Result<Arc<dyn Repository>> {
+    let repo_dir = conf.tenant_path(&tenantid);
+    if repo_dir.exists() {
+        bail!("repo for {} already exists", tenantid)
+    }
+
     // top-level dir may exist if we are creating it through CLI
-    fs::create_dir_all(repo_dir)
+    fs::create_dir_all(&repo_dir)
         .with_context(|| format!("could not create directory {}", repo_dir.display()))?;
 
-    env::set_current_dir(repo_dir)?;
+    // Note: this `info!(...)` macro comes from `log` crate
+    info!("standard logging redirected to slog");
 
-    fs::create_dir(std::path::Path::new("timelines"))?;
-    fs::create_dir(std::path::Path::new("refs"))?;
-    fs::create_dir(std::path::Path::new("refs").join("branches"))?;
-    fs::create_dir(std::path::Path::new("refs").join("tags"))?;
-    fs::create_dir(std::path::Path::new("wal-redo"))?;
+    fs::create_dir(conf.timelines_path(&tenantid))?;
+    fs::create_dir_all(conf.branches_path(&tenantid))?;
+    fs::create_dir_all(conf.tags_path(&tenantid))?;
 
-    println!("created directory structure in {}", repo_dir.display());
+    info!("created directory structure in {}", repo_dir.display());
 
-    // Create initial timeline
-    let tli = create_timeline(conf, None)?;
-    let timelinedir = conf.timeline_path(tli);
-    println!("created initial timeline {}", tli);
+    let tli = create_timeline(conf, None, &tenantid)?;
 
-    // Run initdb
+    // We don't use page_cache here, because we don't want to spawn the WAL redo thread during
+    // repository initialization.
     //
-    // We create the cluster temporarily in a "tmp" directory inside the repository,
-    // and move it to the right location from there.
-    let tmppath = std::path::Path::new("tmp");
+    // FIXME: That caused trouble, because the WAL redo thread launched initdb in the background,
+    // and it kept running even after the "zenith init" had exited. In tests, we started the
+    // page server immediately after that, so that initdb was still running in the background,
+    // and we failed to run initdb again in the same directory. This has been solved for the
+    // rapid init+start case now, but the general race condition remains if you restart the
+    // server quickly.
+    let repo: Arc<dyn Repository + Sync + Send> =
+        match conf.repository_format {
+            RepositoryFormat::Layered => Arc::new(
+                crate::layered_repository::LayeredRepository::new(conf, wal_redo_manager, tenantid),
+            ),
+            RepositoryFormat::RocksDb => {
+                let obj_store = crate::rocksdb_storage::RocksObjectStore::create(conf, &tenantid)?;
 
-    print!("running initdb... ");
-    io::stdout().flush()?;
+                Arc::new(ObjectRepository::new(
+                    conf,
+                    Arc::new(obj_store),
+                    wal_redo_manager,
+                    tenantid,
+                ))
+            }
+        };
+
+    // Load data into pageserver
+    // TODO To implement zenith import we need to
+    //      move data loading out of create_repo()
+    bootstrap_timeline(conf, tenantid, tli, &*repo)?;
+
+    Ok(repo)
+}
+
+// Returns checkpoint LSN from controlfile
+fn get_lsn_from_controlfile(path: &Path) -> Result<Lsn> {
+    // Read control file to extract the LSN
+    let controlfile_path = path.join("global").join("pg_control");
+    let controlfile = ControlFileData::decode(&fs::read(controlfile_path)?)?;
+    let lsn = controlfile.checkPoint;
+
+    Ok(Lsn(lsn))
+}
+
+// Create the cluster temporarily in a initdbpath directory inside the repository
+// to get bootstrap data for timeline initialization.
+//
+fn run_initdb(conf: &'static PageServerConf, initdbpath: &Path) -> Result<()> {
+    info!("running initdb... ");
 
     let initdb_path = conf.pg_bin_dir().join("initdb");
-    let initdb_otput = Command::new(initdb_path)
-        .args(&["-D", tmppath.to_str().unwrap()])
+    let initdb_output = Command::new(initdb_path)
+        .args(&["-D", initdbpath.to_str().unwrap()])
+        .args(&["-U", &conf.superuser])
         .arg("--no-instructions")
         .env_clear()
         .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
@@ -78,53 +148,77 @@ pub fn init_repo(conf: &PageServerConf, repo_dir: &Path) -> Result<()> {
         .stdout(Stdio::null())
         .output()
         .with_context(|| "failed to execute initdb")?;
-    if !initdb_otput.status.success() {
-        anyhow::bail!("initdb failed");
+    if !initdb_output.status.success() {
+        anyhow::bail!(
+            "initdb failed: '{}'",
+            String::from_utf8_lossy(&initdb_output.stderr)
+        );
     }
-    println!("initdb succeeded");
-
-    // Read control file to extract the LSN and system id
-    let controlfile_path = tmppath.join("global").join("pg_control");
-    let controlfile = postgres_ffi::decode_pg_control(Bytes::from(fs::read(controlfile_path)?))?;
-    // let systemid = controlfile.system_identifier;
-    let lsn = controlfile.checkPoint;
-    let lsnstr = format!("{:016X}", lsn);
-
-    // Move the initial WAL file
-    fs::rename(
-        tmppath.join("pg_wal").join("000000010000000000000001"),
-        timelinedir
-            .join("wal")
-            .join("000000010000000000000001.partial"),
-    )?;
-    println!("moved initial WAL file");
-
-    // Remove pg_wal
-    fs::remove_dir_all(tmppath.join("pg_wal"))?;
-
-    let target = timelinedir.join("snapshots").join(&lsnstr);
-    fs::rename(tmppath, &target)?;
-
-    // Create 'main' branch to refer to the initial timeline
-    let data = tli.to_string();
-    fs::write(conf.branch_path("main"), data)?;
-    println!("created main branch");
-
-    println!(
-        "new zenith repository was created in {}",
-        repo_dir.display()
-    );
+    info!("initdb succeeded");
 
     Ok(())
 }
 
-pub(crate) fn get_branches(
-    conf: &PageServerConf,
-    repository: &dyn Repository,
-) -> Result<Vec<BranchInfo>> {
+//
+// - run initdb to init temporary instance and get bootstrap data
+// - after initialization complete, remove the temp dir.
+//
+fn bootstrap_timeline(
+    conf: &'static PageServerConf,
+    tenantid: ZTenantId,
+    tli: ZTimelineId,
+    repo: &dyn Repository,
+) -> Result<()> {
+    let initdb_path = conf.tenant_path(&tenantid).join("tmp");
+
+    // Init temporarily repo to get bootstrap data
+    run_initdb(conf, &initdb_path)?;
+    let pgdata_path = initdb_path;
+
+    let lsn = get_lsn_from_controlfile(&pgdata_path)?;
+
+    info!("bootstrap_timeline {:?} at lsn {}", pgdata_path, lsn);
+
+    let timeline = repo.create_empty_timeline(tli, lsn)?;
+    restore_local_repo::import_timeline_from_postgres_datadir(&pgdata_path, &*timeline, lsn)?;
+
+    let wal_dir = pgdata_path.join("pg_wal");
+    restore_local_repo::import_timeline_wal(&wal_dir, &*timeline, timeline.get_last_record_lsn())?;
+
+    println!(
+        "created initial timeline {} timeline.lsn {}",
+        tli,
+        timeline.get_last_record_lsn()
+    );
+
+    let data = tli.to_string();
+    fs::write(conf.branch_path("main", &tenantid), data)?;
+    println!("created main branch");
+
+    // Remove temp dir. We don't need it anymore
+    fs::remove_dir_all(pgdata_path)?;
+
+    Ok(())
+}
+
+pub(crate) fn get_tenants(conf: &PageServerConf) -> Result<Vec<String>> {
+    let tenants_dir = conf.tenants_path();
+
+    std::fs::read_dir(&tenants_dir)?
+        .map(|dir_entry_res| {
+            let dir_entry = dir_entry_res?;
+            ensure!(dir_entry.file_type()?.is_dir());
+            Ok(dir_entry.file_name().to_str().unwrap().to_owned())
+        })
+        .collect()
+}
+
+pub(crate) fn get_branches(conf: &PageServerConf, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
+    let repo = page_cache::get_repository_for_tenant(tenantid)?;
+
     // Each branch has a corresponding record (text file) in the refs/branches
     // with timeline_id.
-    let branches_dir = std::path::Path::new("refs").join("branches");
+    let branches_dir = conf.branches_path(tenantid);
 
     std::fs::read_dir(&branches_dir)?
         .map(|dir_entry_res| {
@@ -132,12 +226,12 @@ pub(crate) fn get_branches(
             let name = dir_entry.file_name().to_str().unwrap().to_string();
             let timeline_id = std::fs::read_to_string(dir_entry.path())?.parse::<ZTimelineId>()?;
 
-            let latest_valid_lsn = repository
+            let latest_valid_lsn = repo
                 .get_timeline(timeline_id)
                 .map(|timeline| timeline.get_last_valid_lsn())
                 .ok();
 
-            let ancestor_path = conf.ancestor_path(timeline_id);
+            let ancestor_path = conf.ancestor_path(&timeline_id, tenantid);
             let mut ancestor_id: Option<String> = None;
             let mut ancestor_lsn: Option<String> = None;
 
@@ -170,67 +264,40 @@ pub(crate) fn get_branches(
         .collect()
 }
 
-pub(crate) fn get_system_id(conf: &PageServerConf) -> Result<u64> {
-    // let branches = get_branches();
-
-    let branches_dir = std::path::Path::new("refs").join("branches");
-    let branches = std::fs::read_dir(&branches_dir)?
-        .map(|dir_entry_res| {
-            let dir_entry = dir_entry_res?;
-            let name = dir_entry.file_name().to_str().unwrap().to_string();
-            let timeline_id = std::fs::read_to_string(dir_entry.path())?.parse::<ZTimelineId>()?;
-            Ok((name, timeline_id))
-        })
-        .collect::<Result<HashMap<String, ZTimelineId>>>()?;
-
-    let main_tli = branches
-        .get("main")
-        .ok_or_else(|| anyhow!("Branch main not found"))?;
-
-    let (_, main_snap_dir) = find_latest_snapshot(conf, *main_tli)?;
-    let controlfile_path = main_snap_dir.join("global").join("pg_control");
-    let controlfile = postgres_ffi::decode_pg_control(Bytes::from(fs::read(controlfile_path)?))?;
-    Ok(controlfile.system_identifier)
-}
-
 pub(crate) fn create_branch(
     conf: &PageServerConf,
     branchname: &str,
     startpoint_str: &str,
+    tenantid: &ZTenantId,
 ) -> Result<BranchInfo> {
-    if conf.branch_path(&branchname).exists() {
+    let repo = page_cache::get_repository_for_tenant(tenantid)?;
+
+    if conf.branch_path(branchname, tenantid).exists() {
         anyhow::bail!("branch {} already exists", branchname);
     }
 
-    let mut startpoint = parse_point_in_time(conf, startpoint_str)?;
+    let mut startpoint = parse_point_in_time(conf, startpoint_str, tenantid)?;
 
     if startpoint.lsn == Lsn(0) {
         // Find end of WAL on the old timeline
-        let end_of_wal = find_end_of_wal(conf, startpoint.timelineid)?;
+        let end_of_wal = repo
+            .get_timeline(startpoint.timelineid)?
+            .get_last_record_lsn();
         println!("branching at end of WAL: {}", end_of_wal);
         startpoint.lsn = end_of_wal;
     }
 
-    // create a new timeline for it
-    let newtli = create_timeline(conf, Some(startpoint))?;
-    let newtimelinedir = conf.timeline_path(newtli);
+    // create a new timeline directory for it
+    let newtli = create_timeline(conf, Some(startpoint), tenantid)?;
 
+    // Let the Repository backend do its initialization
+    repo.branch_timeline(startpoint.timelineid, newtli, startpoint.lsn)?;
+
+    // Remember the human-readable branch name for the new timeline.
+    // FIXME: there's a race condition, if you create a branch with the same
+    // name concurrently.
     let data = newtli.to_string();
-    fs::write(conf.branch_path(&branchname), data)?;
-
-    // Copy the latest snapshot (TODO: before the startpoint) and all WAL
-    // TODO: be smarter and avoid the copying...
-    let (_maxsnapshot, oldsnapshotdir) = find_latest_snapshot(conf, startpoint.timelineid)?;
-    let copy_opts = fs_extra::dir::CopyOptions::new();
-    fs_extra::dir::copy(oldsnapshotdir, newtimelinedir.join("snapshots"), &copy_opts)?;
-
-    let oldtimelinedir = conf.timeline_path(startpoint.timelineid);
-    copy_wal(
-        &oldtimelinedir.join("wal"),
-        &newtimelinedir.join("wal"),
-        startpoint.lsn,
-        pg_constants::WAL_SEGMENT_SIZE,
-    )?;
+    fs::write(conf.branch_path(&branchname, tenantid), data)?;
 
     Ok(BranchInfo {
         name: branchname.to_string(),
@@ -260,7 +327,11 @@ pub(crate) fn create_branch(
 //    mytag
 //
 //
-fn parse_point_in_time(conf: &PageServerConf, s: &str) -> Result<PointInTime> {
+fn parse_point_in_time(
+    conf: &PageServerConf,
+    s: &str,
+    tenantid: &ZTenantId,
+) -> Result<PointInTime> {
     let mut strings = s.split('@');
     let name = strings.next().unwrap();
 
@@ -275,21 +346,21 @@ fn parse_point_in_time(conf: &PageServerConf, s: &str) -> Result<PointInTime> {
 
     // Check if it's a tag
     if lsn.is_none() {
-        let tagpath = conf.tag_path(name);
+        let tagpath = conf.tag_path(name, &tenantid);
         if tagpath.exists() {
             let pointstr = fs::read_to_string(tagpath)?;
 
-            return parse_point_in_time(conf, &pointstr);
+            return parse_point_in_time(conf, &pointstr, &tenantid);
         }
     }
 
     // Check if it's a branch
     // Check if it's branch @ LSN
-    let branchpath = conf.branch_path(name);
+    let branchpath = conf.branch_path(name, &tenantid);
     if branchpath.exists() {
         let pointstr = fs::read_to_string(branchpath)?;
 
-        let mut result = parse_point_in_time(conf, &pointstr)?;
+        let mut result = parse_point_in_time(conf, &pointstr, &tenantid)?;
 
         result.lsn = lsn.unwrap_or(Lsn(0));
         return Ok(result);
@@ -298,7 +369,7 @@ fn parse_point_in_time(conf: &PageServerConf, s: &str) -> Result<PointInTime> {
     // Check if it's a timelineid
     // Check if it's timelineid @ LSN
     if let Ok(timelineid) = ZTimelineId::from_str(name) {
-        let tlipath = conf.timeline_path(timelineid);
+        let tlipath = conf.timeline_path(&timelineid, &tenantid);
         if tlipath.exists() {
             return Ok(PointInTime {
                 timelineid,
@@ -310,16 +381,18 @@ fn parse_point_in_time(conf: &PageServerConf, s: &str) -> Result<PointInTime> {
     bail!("could not parse point-in-time {}", s);
 }
 
-fn create_timeline(conf: &PageServerConf, ancestor: Option<PointInTime>) -> Result<ZTimelineId> {
+fn create_timeline(
+    conf: &PageServerConf,
+    ancestor: Option<PointInTime>,
+    tenantid: &ZTenantId,
+) -> Result<ZTimelineId> {
     // Create initial timeline
-    let mut tli_buf = [0u8; 16];
-    rand::thread_rng().fill(&mut tli_buf);
-    let timelineid = ZTimelineId::from(tli_buf);
 
-    let timelinedir = conf.timeline_path(timelineid);
+    let timelineid = ZTimelineId::generate();
+
+    let timelinedir = conf.timeline_path(&timelineid, tenantid);
 
     fs::create_dir(&timelinedir)?;
-    fs::create_dir(&timelinedir.join("snapshots"))?;
     fs::create_dir(&timelinedir.join("wal"))?;
 
     if let Some(ancestor) = ancestor {
@@ -329,81 +402,3 @@ fn create_timeline(conf: &PageServerConf, ancestor: Option<PointInTime>) -> Resu
 
     Ok(timelineid)
 }
-
-///
-/// Copy all WAL segments from one directory to another, up to given LSN.
-///
-/// If the given LSN is in the middle of a segment, the last segment containing it
-/// is written out as .partial, and padded with zeros.
-///
-fn copy_wal(src_dir: &Path, dst_dir: &Path, upto: Lsn, wal_seg_size: usize) -> Result<()> {
-    let last_segno = upto.segment_number(wal_seg_size);
-    let last_segoff = upto.segment_offset(wal_seg_size);
-
-    for entry in fs::read_dir(src_dir).unwrap() {
-        if let Ok(entry) = entry {
-            let entry_name = entry.file_name();
-            let fname = entry_name.to_str().unwrap();
-
-            // Check if the filename looks like an xlog file, or a .partial file.
-            if !xlog_utils::IsXLogFileName(fname) && !xlog_utils::IsPartialXLogFileName(fname) {
-                continue;
-            }
-            let (segno, _tli) = xlog_utils::XLogFromFileName(fname, wal_seg_size as usize);
-
-            let copylen;
-            let mut dst_fname = PathBuf::from(fname);
-            if segno > last_segno {
-                // future segment, skip
-                continue;
-            } else if segno < last_segno {
-                copylen = wal_seg_size;
-                dst_fname.set_extension("");
-            } else {
-                copylen = last_segoff;
-                dst_fname.set_extension("partial");
-            }
-
-            let src_file = File::open(entry.path())?;
-            let mut dst_file = File::create(dst_dir.join(&dst_fname))?;
-            std::io::copy(&mut src_file.take(copylen as u64), &mut dst_file)?;
-
-            if copylen < wal_seg_size {
-                std::io::copy(
-                    &mut std::io::repeat(0).take((wal_seg_size - copylen) as u64),
-                    &mut dst_file,
-                )?;
-            }
-        }
-    }
-    Ok(())
-}
-
-// Find the end of valid WAL in a wal directory
-pub fn find_end_of_wal(conf: &PageServerConf, timeline: ZTimelineId) -> Result<Lsn> {
-    let waldir = conf.timeline_path(timeline).join("wal");
-    let (lsn, _tli) = xlog_utils::find_end_of_wal(&waldir, pg_constants::WAL_SEGMENT_SIZE, true);
-    Ok(Lsn(lsn))
-}
-
-// Find the latest snapshot for a timeline
-fn find_latest_snapshot(conf: &PageServerConf, timeline: ZTimelineId) -> Result<(Lsn, PathBuf)> {
-    let snapshotsdir = conf.snapshots_path(timeline);
-    let paths = fs::read_dir(&snapshotsdir)?;
-    let mut maxsnapshot = Lsn(0);
-    let mut snapshotdir: Option<PathBuf> = None;
-    for path in paths {
-        let path = path?;
-        let filename = path.file_name().to_str().unwrap().to_owned();
-        if let Ok(lsn) = Lsn::from_hex(&filename) {
-            maxsnapshot = std::cmp::max(lsn, maxsnapshot);
-            snapshotdir = Some(path.path());
-        }
-    }
-    if maxsnapshot == Lsn(0) {
-        // TODO: check ancestor timeline
-        anyhow::bail!("no snapshot found in {}", snapshotsdir.display());
-    }
-
-    Ok((maxsnapshot, snapshotdir.unwrap()))
-}

pageserver/src/layered_repository/README.md

@@ -0,0 +1,298 @@
+# Overview
+
+The on-disk format is based on immutable files. The page server
+receives a stream of incoming WAL, parses the WAL records to determine
+which pages they apply to, and accumulates the incoming changes in
+memory. Every now and then, the accumulated changes are written out to
+new files.
+
+The files are called "snapshot files". Each snapshot file corresponds
+to one 10 MB slice of a PostgreSQL relation fork. The snapshot files
+for each timeline are stored in the timeline's subdirectory under
+.zenith/tenants/<tenantid>/timelines.
+
+The files are named like this:
+
+    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>_<end LSN>
+
+For example:
+
+    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000
+
+Some non-relation files are also stored in repository. For example,
+a CLOG segment would be named like this:
+
+    pg_xact_0000_0_00000000198B06B0_00000000198C2550
+
+There is no difference in how the relation and non-relation files are
+managed, except that the first part of file names is different.
+Internally, the relations and non-relation files that are managed in
+the versioned store are together called "relishes".
+
+Each snapshot file contains a full snapshot, that is, full copy of all
+pages in the relation, as of the "start LSN". It also contains all WAL
+records applicable to the relation between the start and end
+LSNs. With this information, the page server can reconstruct any page
+version of the relation in the LSN range.
+
+If a file has been dropped, the last snapshot file for it is created
+with the _DROPPED suffix, e.g.
+
+    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000_DROPPED
+
+In addition to the relations, with "rel_*" prefix, we use the same
+format for storing various smaller files from the PostgreSQL data
+directory. They will use different suffixes and the naming scheme
+up to the LSN range varies. The Zenith source code uses the term
+"relish" to mean "a relation, or other file that's treated like a
+relation in the storage"
+
+## Notation used in this document
+
+The full path of a snapshot file looks like this:
+
+    .zenith/tenants/941ddc8604413b88b3d208bddf90396c/timelines/4af489b06af8eed9e27a841775616962/rel_1663_13990_2609_0_10_000000000169C348_0000000001702000
+
+For simplicity, the examples below use a simplified notation for the
+paths.  The tenant ID is left out, the timeline ID is replaced with
+the human-readable branch name, and spcnode+dbnode+relnode+forkum+segno
+with a human-readable table name. The LSNs are also shorter. For
+example, a snapshot file for 'orders' table on 'main' branch, with LSN
+range 100-200 would be:
+
+    main/orders_100_200
+
+
+# Creating snapshot files
+
+Let's start with a simple example with a system that contains one
+branch called 'main' and two tables, 'orders' and 'customers'. The end
+of WAL is currently at LSN 250. In this starting situation, you would
+have two files on disk:
+
+	main/orders_100_200
+	main/customers_100_200
+
+In addition to those files, the recent changes between LSN 200 and the
+end of WAL at 250 are kept in memory. If the page server crashes, the
+latest records between 200-250 need to be re-read from the WAL.
+
+Whenever enough WAL has been accumulated in memory, the page server
+writes out the changes in memory into new snapshot files. This process
+is called "checkpointing" (not to be confused with the PostgreSQL
+checkpoints, that's a different thing). The page server only creates
+snapshot files for relations that have been modified since the last
+checkpoint. For example, if the current end of WAL is at LSN 450, and
+the last checkpoint happened at LSN 400 but there hasn't been any
+recent changes to 'customers' table, you would have these files on
+disk:
+
+	main/orders_100_200
+	main/orders_200_300
+	main/orders_300_400
+	main/customers_100_200
+
+If the customers table is modified later, a new file is created for it
+at the next checkpoint. The new file will cover the "gap" from the
+last snapshot file, so the LSN ranges are always contiguous:
+
+	main/orders_100_200
+	main/orders_200_300
+	main/orders_300_400
+	main/customers_100_200
+	main/customers_200_500
+
+## Reading page versions
+
+Whenever a GetPage@LSN request comes in from the compute node, the
+page server needs to reconstruct the requested page, as it was at the
+requested LSN. To do that, the page server first checks the recent
+in-memory layer; if the requested page version is found there, it can
+be returned immediatedly without looking at the files on
+disk. Otherwise the page server needs to locate the snapshot file that
+contains the requested page version.
+
+For example, if a request comes in for table 'orders' at LSN 250, the
+page server would load the 'main/orders_200_300' file into memory, and
+reconstruct and return the requested page from it, as it was at
+LSN 250. Because the snapshot file consists of a full image of the
+relation at the start LSN and the WAL, reconstructing the page
+involves replaying any WAL records applicable to the page between LSNs
+200-250, starting from the base image at LSN 200.
+
+A request at a file boundary can be satisfied using either file. For
+example, if there are two files on disk:
+
+	main/orders_100_200
+	main/orders_200_300
+
+And a request comes with LSN 200, either file can be used for it. It
+is better to use the later file, however, because it contains an
+already materialized version of all the pages at LSN 200. Using the
+first file, you would need to apply any WAL records between 100 and
+200 to reconstruct the requested page.
+
+# Multiple branches
+
+Imagine that a child branch is created at LSN 250:
+
+            @250
+    ----main--+-------------------------->
+               \
+                +---child-------------->
+
+
+Then, the 'orders' table is updated differently on the 'main' and
+'child' branches. You now have this situation on disk:
+
+    main/orders_100_200
+    main/orders_200_300
+    main/orders_300_400
+    main/customers_100_200
+    child/orders_250_300
+    child/orders_300_400
+
+Because the 'customers' table hasn't been modified on the child
+branch, there is no file for it there. If you request a page for it on
+the 'child' branch, the page server will not find any snapshot file
+for it in the 'child' directory, so it will recurse to look into the
+parent 'main' branch instead.
+
+From the 'child' branch's point of view, the history for each relation
+is linear, and the request's LSN identifies unambiguously which file
+you need to look at. For example, the history for the 'orders' table
+on the 'main' branch consists of these files:
+
+    main/orders_100_200
+    main/orders_200_300
+    main/orders_300_400
+
+And from the 'child' branch's point of view, it consists of these
+files:
+
+    main/orders_100_200
+    main/orders_200_300
+    child/orders_250_300
+    child/orders_300_400
+
+The branch metadata includes the point where the child branch was
+created, LSN 250. If a page request comes with LSN 275, we read the
+page version from the 'child/orders_250_300' file. If the request LSN
+is 225, we read it from the 'main/orders_200_300' file instead.  The
+page versions between 250-300 in the 'main/orders_200_300' file are
+ignored when operating on the child branch.
+
+Note: It doesn't make any difference if the child branch is created
+when the end of the main branch was at LSN 250, or later when the tip of
+the main branch had already moved on. The latter case, creating a
+branch at a historic LSN, is how we support PITR in Zenith.
+
+
+# Garbage collection
+
+In this scheme, we keep creating new snapshot files over time. We also
+need a mechanism to remove old files that are no longer needed,
+because disk space isn't infinite.
+
+What files are still needed? Currently, the page server supports PITR
+and branching from any branch at any LSN that is "recent enough" from
+the tip of the branch.  "Recent enough" is defined as an LSN horizon,
+which by default is 64 MB.  (See DEFAULT_GC_HORIZON). For this
+example, let's assume that the LSN horizon is 150 units.
+
+Let's look at the single branch scenario again. Imagine that the end
+of the branch is LSN 525, so that the GC horizon is currently at
+525-150 = 375
+
+	main/orders_100_200
+	main/orders_200_300
+	main/orders_300_400
+	main/orders_400_500
+	main/customers_100_200
+
+We can remove files 'main/orders_100_200' and 'main/orders_200_300',
+because the end LSNs of those files are older than GC horizon 375, and
+there are more recent snapshot files for the table. 'main/orders_300_400'
+and 'main/orders_400_500' are still within the horizon, so they must be
+retained. 'main/customers_100_200' is old enough, but it cannot be
+removed because there is no newer snapshot file for the table.
+
+Things get slightly more complicated with multiple branches. All of
+the above still holds, but in addition to recent files we must also
+retain older shapshot files that are still needed by child branches.
+For example, if child branch is created at LSN 150, and the 'customers'
+table is updated on the branch, you would have these files:
+
+	main/orders_100_200
+	main/orders_200_300
+	main/orders_300_400
+	main/orders_400_500
+	main/customers_100_200
+	child/customers_150_300
+
+In this situation, the 'main/orders_100_200' file cannot be removed,
+even though it is older than the GC horizon, because it is still
+needed by the child branch.  'main/orders_200_300' can still be
+removed. So after garbage collection, these files would remain:
+
+	main/orders_100_200
+
+	main/orders_300_400
+	main/orders_400_500
+	main/customers_100_200
+	child/customers_150_300
+
+If 'orders' is modified later on the 'child' branch, we will create a
+snapshot file for it on the child:
+
+	main/orders_100_200
+
+	main/orders_300_400
+	main/orders_400_500
+	main/customers_100_200
+	child/customers_150_300
+	child/orders_150_400
+
+After this, the 'main/orders_100_200' file can be removed. It is no
+longer needed by the child branch, because there is a newer snapshot
+file there. TODO: This optimization hasn't been implemented! The GC
+algorithm will currently keep the file on the 'main' branch anyway, for
+as long as the child branch exists.
+
+
+# TODO: On LSN ranges
+
+In principle, each relation can be checkpointed separately, i.e. the
+LSN ranges of the files don't need to line up. So this would be legal:
+
+	main/orders_100_200
+	main/orders_200_300
+	main/orders_300_400
+	main/customers_150_250
+	main/customers_250_500
+
+However, the code currently always checkpoints all relations together.
+So that situation doesn't arise in practice.
+
+It would also be OK to have overlapping LSN ranges for the same relation:
+
+	main/orders_100_200
+	main/orders_200_300
+	main/orders_250_350
+	main/orders_300_400
+
+The code that reads the snapshot files should cope with this, but this
+situation doesn't arise either, because the checkpointing code never
+does that.  It could be useful, however, as a transient state when
+garbage collecting around branch points, or explicit recovery
+points. For example, if we start with this:
+
+	main/orders_100_200
+	main/orders_200_300
+	main/orders_300_400
+
+And there is a branch or explicit recovery point at LSN 150, we could
+replace 'main/orders_100_200' with 'main/orders_150_150' to keep a
+snapshot only at that exact point that's still needed, removing the
+other page versions around it. But such compaction has not been
+implemented yet.

pageserver/src/layered_repository/inmemory_layer.rs

@@ -0,0 +1,491 @@
+//!
+//! An in-memory layer stores recently received page versions in memory. The page versions
+//! are held in a BTreeMap, and there's another BTreeMap to track the size of the relation.
+//!
+use crate::layered_repository::storage_layer::{
+    Layer, PageReconstructData, PageVersion, SegmentTag, RELISH_SEG_SIZE,
+};
+use crate::layered_repository::LayeredTimeline;
+use crate::layered_repository::SnapshotLayer;
+use crate::repository::WALRecord;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
+use anyhow::{bail, Result};
+use bytes::Bytes;
+use log::*;
+use std::collections::BTreeMap;
+use std::ops::Bound::Included;
+use std::sync::{Arc, Mutex};
+
+use zenith_utils::lsn::Lsn;
+
+pub struct InMemoryLayer {
+    conf: &'static PageServerConf,
+    tenantid: ZTenantId,
+    timelineid: ZTimelineId,
+    seg: SegmentTag,
+
+    ///
+    /// This layer contains all the changes from 'start_lsn'. The
+    /// start is inclusive. There is no end LSN; we only use in-memory
+    /// layer at the end of a timeline.
+    ///
+    start_lsn: Lsn,
+
+    /// The above fields never change. The parts that do change are in 'inner',
+    /// and protected by mutex.
+    inner: Mutex<InMemoryLayerInner>,
+}
+
+pub struct InMemoryLayerInner {
+    /// If this relation was dropped, remember when that happened.
+    drop_lsn: Option<Lsn>,
+
+    ///
+    /// All versions of all pages in the layer are are kept here.
+    /// Indexed by block number and LSN.
+    ///
+    page_versions: BTreeMap<(u32, Lsn), PageVersion>,
+
+    ///
+    /// `segsizes` tracks the size of the segment at different points in time.
+    ///
+    segsizes: BTreeMap<Lsn, u32>,
+}
+
+impl Layer for InMemoryLayer {
+    fn get_timeline_id(&self) -> ZTimelineId {
+        return self.timelineid;
+    }
+
+    fn get_seg_tag(&self) -> SegmentTag {
+        return self.seg;
+    }
+
+    fn get_start_lsn(&self) -> Lsn {
+        return self.start_lsn;
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
+        let inner = self.inner.lock().unwrap();
+
+        if let Some(drop_lsn) = inner.drop_lsn {
+            drop_lsn
+        } else {
+            Lsn(u64::MAX)
+        }
+    }
+
+    fn is_dropped(&self) -> bool {
+        let inner = self.inner.lock().unwrap();
+        inner.drop_lsn.is_some()
+    }
+
+    /// Look up given page in the cache.
+    fn get_page_reconstruct_data(
+        &self,
+        blknum: u32,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<Option<Lsn>> {
+        // Scan the BTreeMap backwards, starting from reconstruct_data.lsn.
+        let mut need_base_image_lsn: Option<Lsn> = Some(lsn);
+
+        assert!(self.seg.blknum_in_seg(blknum));
+
+        {
+            let inner = self.inner.lock().unwrap();
+            let minkey = (blknum, Lsn(0));
+            let maxkey = (blknum, lsn);
+            let mut iter = inner
+                .page_versions
+                .range((Included(&minkey), Included(&maxkey)));
+            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
+                if let Some(img) = &entry.page_image {
+                    reconstruct_data.page_img = Some(img.clone());
+                    need_base_image_lsn = None;
+                    break;
+                } else if let Some(rec) = &entry.record {
+                    reconstruct_data.records.push(rec.clone());
+                    if rec.will_init {
+                        // This WAL record initializes the page, so no need to go further back
+                        need_base_image_lsn = None;
+                        break;
+                    } else {
+                        need_base_image_lsn = Some(*entry_lsn);
+                    }
+                } else {
+                    // No base image, and no WAL record. Huh?
+                    bail!("no page image or WAL record for requested page");
+                }
+            }
+
+            // release lock on 'page_versions'
+        }
+
+        Ok(need_base_image_lsn)
+    }
+
+    /// Get size of the relation at given LSN
+    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let inner = self.inner.lock().unwrap();
+        let mut iter = inner.segsizes.range((Included(&Lsn(0)), Included(&lsn)));
+
+        if let Some((_entry_lsn, entry)) = iter.next_back() {
+            let result = *entry;
+            drop(inner);
+            trace!("get_seg_size: {} at {} -> {}", self.seg, lsn, result);
+            Ok(result)
+        } else {
+            bail!("No size found for {} at {} in memory", self.seg, lsn);
+        }
+    }
+
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
+        let inner = self.inner.lock().unwrap();
+
+        // Is the requested LSN after the segment was dropped?
+        if let Some(drop_lsn) = inner.drop_lsn {
+            if lsn >= drop_lsn {
+                return Ok(false);
+            }
+        }
+
+        // Otherwise, it exists
+        Ok(true)
+    }
+}
+
+impl InMemoryLayer {
+    ///
+    /// Create a new, empty, in-memory layer
+    ///
+    pub fn create(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        seg: SegmentTag,
+        start_lsn: Lsn,
+    ) -> Result<InMemoryLayer> {
+        trace!(
+            "initializing new empty InMemoryLayer for writing {} on timeline {} at {}",
+            seg,
+            timelineid,
+            start_lsn
+        );
+
+        Ok(InMemoryLayer {
+            conf,
+            timelineid,
+            tenantid,
+            seg,
+            start_lsn,
+            inner: Mutex::new(InMemoryLayerInner {
+                drop_lsn: None,
+                page_versions: BTreeMap::new(),
+                segsizes: BTreeMap::new(),
+            }),
+        })
+    }
+
+    // Write operations
+
+    /// Remember new page version, as a WAL record over previous version
+    pub fn put_wal_record(&self, blknum: u32, rec: WALRecord) -> Result<()> {
+        self.put_page_version(
+            blknum,
+            rec.lsn,
+            PageVersion {
+                page_image: None,
+                record: Some(rec),
+            },
+        )
+    }
+
+    /// Remember new page version, as a full page image
+    pub fn put_page_image(&self, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()> {
+        self.put_page_version(
+            blknum,
+            lsn,
+            PageVersion {
+                page_image: Some(img),
+                record: None,
+            },
+        )
+    }
+
+    /// Common subroutine of the public put_wal_record() and put_page_image() functions.
+    /// Adds the page version to the in-memory tree
+    pub fn put_page_version(&self, blknum: u32, lsn: Lsn, pv: PageVersion) -> Result<()> {
+        assert!(self.seg.blknum_in_seg(blknum));
+
+        trace!(
+            "put_page_version blk {} of {} at {}/{}",
+            blknum,
+            self.seg.rel,
+            self.timelineid,
+            lsn
+        );
+        let mut inner = self.inner.lock().unwrap();
+
+        let old = inner.page_versions.insert((blknum, lsn), pv);
+
+        if old.is_some() {
+            // We already had an entry for this LSN. That's odd..
+            warn!(
+                "Page version of rel {} blk {} at {} already exists",
+                self.seg.rel, blknum, lsn
+            );
+        }
+
+        // Also update the relation size, if this extended the relation.
+        if self.seg.rel.is_blocky() {
+            let newsize = blknum - self.seg.segno * RELISH_SEG_SIZE + 1;
+
+            let mut iter = inner.segsizes.range((Included(&Lsn(0)), Included(&lsn)));
+
+            let oldsize;
+            if let Some((_entry_lsn, entry)) = iter.next_back() {
+                oldsize = *entry;
+            } else {
+                oldsize = 0;
+                //bail!("No old size found for {} at {}", self.tag, lsn);
+            }
+            if newsize > oldsize {
+                trace!(
+                    "enlarging segment {} from {} to {} blocks at {}",
+                    self.seg,
+                    oldsize,
+                    newsize,
+                    lsn
+                );
+                inner.segsizes.insert(lsn, newsize);
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Remember that the relation was truncated at given LSN
+    pub fn put_truncation(&self, lsn: Lsn, segsize: u32) -> anyhow::Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+        let old = inner.segsizes.insert(lsn, segsize);
+
+        if old.is_some() {
+            // We already had an entry for this LSN. That's odd..
+            warn!("Inserting truncation, but had an entry for the LSN already");
+        }
+
+        Ok(())
+    }
+
+    /// Remember that the segment was dropped at given LSN
+    pub fn put_unlink(&self, lsn: Lsn) -> anyhow::Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+
+        assert!(inner.drop_lsn.is_none());
+        inner.drop_lsn = Some(lsn);
+
+        info!("dropped segment {} at {}", self.seg, lsn);
+
+        Ok(())
+    }
+
+    ///
+    /// Initialize a new InMemoryLayer for, by copying the state at the given
+    /// point in time from given existing layer.
+    ///
+    pub fn copy_snapshot(
+        conf: &'static PageServerConf,
+        timeline: &LayeredTimeline,
+        src: &dyn Layer,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        lsn: Lsn,
+    ) -> Result<InMemoryLayer> {
+        trace!(
+            "initializing new InMemoryLayer for writing {} on timeline {} at {}",
+            src.get_seg_tag(),
+            timelineid,
+            lsn
+        );
+        let mut page_versions = BTreeMap::new();
+        let mut segsizes = BTreeMap::new();
+
+        let seg = src.get_seg_tag();
+
+        let startblk;
+        let size;
+        if seg.rel.is_blocky() {
+            size = src.get_seg_size(lsn)?;
+            segsizes.insert(lsn, size);
+            startblk = seg.segno * RELISH_SEG_SIZE;
+        } else {
+            size = 1;
+            startblk = 0;
+        }
+
+        for blknum in startblk..(startblk + size) {
+            let img = timeline.materialize_page(seg, blknum, lsn, src)?;
+            let pv = PageVersion {
+                page_image: Some(img),
+                record: None,
+            };
+            page_versions.insert((blknum, lsn), pv);
+        }
+
+        Ok(InMemoryLayer {
+            conf,
+            timelineid,
+            tenantid,
+            seg: src.get_seg_tag(),
+            start_lsn: lsn,
+            inner: Mutex::new(InMemoryLayerInner {
+                drop_lsn: None,
+                page_versions: page_versions,
+                segsizes: segsizes,
+            }),
+        })
+    }
+
+    ///
+    /// Write the this in-memory layer to disk, as a snapshot layer.
+    ///
+    /// The cutoff point for the layer that's written to disk is 'end_lsn'.
+    ///
+    /// Returns new layers that replace this one. Always returns a
+    /// SnapshotLayer containing the page versions that were written to disk,
+    /// but if there were page versions newer than 'end_lsn', also return a new
+    /// in-memory layer containing those page versions. The caller replaces
+    /// this layer with the returned layers in the layer map.
+    ///
+    pub fn freeze(
+        &self,
+        cutoff_lsn: Lsn,
+        // This is needed just to call materialize_page()
+        timeline: &LayeredTimeline,
+    ) -> Result<(Option<Arc<SnapshotLayer>>, Option<Arc<InMemoryLayer>>)> {
+        info!(
+            "freezing in memory layer for {} on timeline {} at {}",
+            self.seg, self.timelineid, cutoff_lsn
+        );
+
+        let inner = self.inner.lock().unwrap();
+
+        // Normally, use the cutoff LSN as the end of the frozen layer.
+        // But if the relation was dropped, we know that there are no
+        // more changes coming in for it, and in particular we know that
+        // there are no changes "in flight" for the LSN anymore, so we use
+        // the drop LSN instead. The drop-LSN could be ahead of the
+        // caller-specified LSN!
+        let dropped = inner.drop_lsn.is_some();
+        let end_lsn = if dropped {
+            inner.drop_lsn.unwrap()
+        } else {
+            cutoff_lsn
+        };
+
+        // Divide all the page versions into old and new at the 'end_lsn' cutoff point.
+        let mut before_page_versions;
+        let mut before_segsizes;
+        let mut after_page_versions;
+        let mut after_segsizes;
+        if !dropped {
+            before_segsizes = BTreeMap::new();
+            after_segsizes = BTreeMap::new();
+            for (lsn, size) in inner.segsizes.iter() {
+                if *lsn > end_lsn {
+                    after_segsizes.insert(*lsn, *size);
+                } else {
+                    before_segsizes.insert(*lsn, *size);
+                }
+            }
+
+            before_page_versions = BTreeMap::new();
+            after_page_versions = BTreeMap::new();
+            for ((blknum, lsn), pv) in inner.page_versions.iter() {
+                if *lsn > end_lsn {
+                    after_page_versions.insert((*blknum, *lsn), pv.clone());
+                } else {
+                    before_page_versions.insert((*blknum, *lsn), pv.clone());
+                }
+            }
+        } else {
+            before_page_versions = inner.page_versions.clone();
+            before_segsizes = inner.segsizes.clone();
+            after_segsizes = BTreeMap::new();
+            after_page_versions = BTreeMap::new();
+        }
+
+        // we can release the lock now.
+        drop(inner);
+
+        // Write the page versions before the cutoff to disk.
+        let snapfile = SnapshotLayer::create(
+            self.conf,
+            self.timelineid,
+            self.tenantid,
+            self.seg,
+            self.start_lsn,
+            end_lsn,
+            dropped,
+            before_page_versions,
+            before_segsizes,
+        )?;
+
+        // If there were any "new" page versions, initialize a new in-memory layer to hold
+        // them
+        let new_open = if !after_segsizes.is_empty() || !after_page_versions.is_empty() {
+            info!("created new in-mem layer for {} {}-", self.seg, end_lsn);
+
+            let new_open = Self::copy_snapshot(
+                self.conf,
+                timeline,
+                &snapfile,
+                self.timelineid,
+                self.tenantid,
+                end_lsn,
+            )?;
+            let mut new_inner = new_open.inner.lock().unwrap();
+            new_inner.page_versions.append(&mut after_page_versions);
+            new_inner.segsizes.append(&mut after_segsizes);
+            drop(new_inner);
+
+            Some(Arc::new(new_open))
+        } else {
+            None
+        };
+
+        let new_historic = Some(Arc::new(snapfile));
+
+        Ok((new_historic, new_open))
+    }
+
+    /// debugging function to print out the contents of the layer
+    #[allow(unused)]
+    pub fn dump(&self) -> String {
+        let mut result = format!(
+            "----- inmemory layer for {} {}-> ----\n",
+            self.seg, self.start_lsn
+        );
+
+        let inner = self.inner.lock().unwrap();
+
+        for (k, v) in inner.segsizes.iter() {
+            result += &format!("{}: {}\n", k, v);
+        }
+        for (k, v) in inner.page_versions.iter() {
+            result += &format!(
+                "blk {} at {}: {}/{}\n",
+                k.0,
+                k.1,
+                v.page_image.is_some(),
+                v.record.is_some()
+            );
+        }
+
+        result
+    }
+}

pageserver/src/layered_repository/layer_map.rs

@@ -0,0 +1,281 @@
+//!
+//! The layer map tracks what layers exist for all the relations in a timeline.
+//!
+//! When the timeline is first accessed, the server lists of all snapshot files
+//! in the timelines/<timelineid> directory, and populates this map with
+//! SnapshotLayers corresponding to each file. When new WAL is received,
+//! we create InMemoryLayers to hold the incoming records. Now and then,
+//! in the checkpoint() function, the in-memory layers are frozen, forming
+//! new snapshot layers and corresponding files are written to disk.
+//!
+
+use crate::layered_repository::storage_layer::{Layer, SegmentTag};
+use crate::layered_repository::{InMemoryLayer, SnapshotLayer};
+use crate::relish::*;
+use anyhow::Result;
+use log::*;
+use std::collections::HashSet;
+use std::collections::{BTreeMap, HashMap};
+use std::ops::Bound::Included;
+use std::sync::Arc;
+use zenith_utils::lsn::Lsn;
+
+///
+/// LayerMap tracks what layers exist or a timeline. The last layer that is
+/// open for writes is always an InMemoryLayer, and is tracked separately
+/// because there can be only one for each segment. The older layers,
+/// stored on disk, are kept in a BTreeMap keyed by the layer's start LSN.
+///
+pub struct LayerMap {
+    segs: HashMap<SegmentTag, SegEntry>,
+}
+
+struct SegEntry {
+    pub open: Option<Arc<InMemoryLayer>>,
+    pub historic: BTreeMap<Lsn, Arc<SnapshotLayer>>,
+}
+
+impl LayerMap {
+    ///
+    /// Look up using the given segment tag and LSN. This differs from a plain
+    /// key-value lookup in that if there is any layer that covers the
+    /// given LSN, or precedes the given LSN, it is returned. In other words,
+    /// you don't need to know the exact start LSN of the layer.
+    ///
+    pub fn get(&self, tag: &SegmentTag, lsn: Lsn) -> Option<Arc<dyn Layer>> {
+        let segentry = self.segs.get(tag)?;
+
+        if let Some(open) = &segentry.open {
+            if open.get_start_lsn() <= lsn {
+                let x: Arc<dyn Layer> = Arc::clone(&open) as _;
+                return Some(x);
+            }
+        }
+
+        if let Some((_k, v)) = segentry
+            .historic
+            .range((Included(Lsn(0)), Included(lsn)))
+            .next_back()
+        {
+            let x: Arc<dyn Layer> = Arc::clone(&v) as _;
+            Some(x)
+        } else {
+            None
+        }
+    }
+
+    ///
+    /// Get the open layer for given segment for writing. Or None if no open
+    /// layer exists.
+    ///
+    pub fn get_open(&self, tag: &SegmentTag) -> Option<Arc<InMemoryLayer>> {
+        let segentry = self.segs.get(tag)?;
+
+        if let Some(open) = &segentry.open {
+            Some(Arc::clone(open))
+        } else {
+            None
+        }
+    }
+
+    ///
+    /// Insert an open in-memory layer
+    ///
+    pub fn insert_open(&mut self, layer: Arc<InMemoryLayer>) {
+        let tag = layer.get_seg_tag();
+
+        if let Some(segentry) = self.segs.get_mut(&tag) {
+            if let Some(_old) = &segentry.open {
+                // FIXME: shouldn't exist, but check
+            }
+            segentry.open = Some(layer);
+        } else {
+            let segentry = SegEntry {
+                open: Some(layer),
+                historic: BTreeMap::new(),
+            };
+            self.segs.insert(tag, segentry);
+        }
+    }
+
+    ///
+    /// Insert an on-disk layer
+    ///
+    pub fn insert_historic(&mut self, layer: Arc<SnapshotLayer>) {
+        let tag = layer.get_seg_tag();
+        let start_lsn = layer.get_start_lsn();
+
+        if let Some(segentry) = self.segs.get_mut(&tag) {
+            segentry.historic.insert(start_lsn, layer);
+        } else {
+            let mut historic = BTreeMap::new();
+            historic.insert(start_lsn, layer);
+
+            let segentry = SegEntry {
+                open: None,
+                historic,
+            };
+            self.segs.insert(tag, segentry);
+        }
+    }
+
+    ///
+    /// Remove an on-disk layer from the map.
+    ///
+    /// This should be called when the corresponding file on disk has been deleted.
+    ///
+    pub fn remove_historic(&mut self, layer: &SnapshotLayer) {
+        let tag = layer.get_seg_tag();
+        let start_lsn = layer.get_start_lsn();
+
+        if let Some(segentry) = self.segs.get_mut(&tag) {
+            segentry.historic.remove(&start_lsn);
+        }
+    }
+
+    pub fn list_rels(&self, spcnode: u32, dbnode: u32) -> Result<HashSet<RelTag>> {
+        let mut rels: HashSet<RelTag> = HashSet::new();
+
+        for (seg, _entry) in self.segs.iter() {
+            if let RelishTag::Relation(reltag) = seg.rel {
+                // FIXME: skip if it was dropped before the requested LSN. But there is no
+                // LSN argument
+
+                if (spcnode == 0 || reltag.spcnode == spcnode)
+                    && (dbnode == 0 || reltag.dbnode == dbnode)
+                {
+                    rels.insert(reltag);
+                }
+            }
+        }
+        Ok(rels)
+    }
+
+    pub fn list_nonrels(&self, _lsn: Lsn) -> Result<HashSet<RelishTag>> {
+        let mut rels: HashSet<RelishTag> = HashSet::new();
+
+        // Scan the timeline directory to get all rels in this timeline.
+        for (seg, _entry) in self.segs.iter() {
+            // FIXME: skip if it was dropped before the requested LSN.
+
+            if let RelishTag::Relation(_) = seg.rel {
+            } else {
+                rels.insert(seg.rel);
+            }
+        }
+        Ok(rels)
+    }
+
+    /// Is there a newer layer for given segment?
+    pub fn newer_layer_exists(&self, seg: SegmentTag, lsn: Lsn) -> bool {
+        if let Some(segentry) = self.segs.get(&seg) {
+            if let Some(_open) = &segentry.open {
+                return true;
+            }
+
+            for (newer_lsn, layer) in segentry
+                .historic
+                .range((Included(lsn), Included(Lsn(u64::MAX))))
+            {
+                if layer.get_end_lsn() > lsn {
+                    trace!(
+                        "found later layer for {}, {} {}-{}",
+                        seg,
+                        lsn,
+                        newer_lsn,
+                        layer.get_end_lsn()
+                    );
+                    return true;
+                } else {
+                    trace!("found singleton layer for {}, {} {}", seg, lsn, newer_lsn);
+                    continue;
+                }
+            }
+        }
+        trace!("no later layer found for {}, {}", seg, lsn);
+        false
+    }
+
+    pub fn iter_open_layers(&mut self) -> OpenLayerIter {
+        OpenLayerIter {
+            last: None,
+            segiter: self.segs.iter_mut(),
+        }
+    }
+
+    pub fn iter_historic_layers(&self) -> HistoricLayerIter {
+        HistoricLayerIter {
+            segiter: self.segs.iter(),
+            iter: None,
+        }
+    }
+}
+
+impl Default for LayerMap {
+    fn default() -> Self {
+        LayerMap {
+            segs: HashMap::new(),
+        }
+    }
+}
+
+pub struct OpenLayerIter<'a> {
+    last: Option<&'a mut SegEntry>,
+
+    segiter: std::collections::hash_map::IterMut<'a, SegmentTag, SegEntry>,
+}
+
+impl<'a> OpenLayerIter<'a> {
+    pub fn replace(&mut self, replacement: Option<Arc<InMemoryLayer>>) {
+        let segentry = self.last.as_mut().unwrap();
+        segentry.open = replacement;
+    }
+
+    pub fn insert_historic(&mut self, new_layer: Arc<SnapshotLayer>) {
+        let start_lsn = new_layer.get_start_lsn();
+
+        let segentry = self.last.as_mut().unwrap();
+        segentry.historic.insert(start_lsn, new_layer);
+    }
+}
+
+impl<'a> Iterator for OpenLayerIter<'a> {
+    type Item = Arc<InMemoryLayer>;
+
+    fn next(&mut self) -> std::option::Option<<Self as std::iter::Iterator>::Item> {
+        while let Some((_seg, entry)) = self.segiter.next() {
+            if let Some(open) = &entry.open {
+                let op = Arc::clone(&open);
+                self.last = Some(entry);
+                return Some(op);
+            }
+        }
+        self.last = None;
+        None
+    }
+}
+
+pub struct HistoricLayerIter<'a> {
+    segiter: std::collections::hash_map::Iter<'a, SegmentTag, SegEntry>,
+    iter: Option<std::collections::btree_map::Iter<'a, Lsn, Arc<SnapshotLayer>>>,
+}
+
+impl<'a> Iterator for HistoricLayerIter<'a> {
+    type Item = Arc<SnapshotLayer>;
+
+    fn next(&mut self) -> std::option::Option<<Self as std::iter::Iterator>::Item> {
+        loop {
+            if let Some(x) = &mut self.iter {
+                if let Some(x) = x.next() {
+                    return Some(Arc::clone(&*x.1));
+                }
+            }
+            if let Some(seg) = self.segiter.next() {
+                self.iter = Some(seg.1.historic.iter());
+                continue;
+            } else {
+                return None;
+            }
+        }
+    }
+}

pageserver/src/layered_repository/snapshot_layer.rs

@@ -0,0 +1,547 @@
+//!
+//! A SnapshotLayer represents one snapshot file on disk. One file holds all page
+//! version and size information of one relation, in a range of LSN.
+//! The name "snapshot file" is a bit of a misnomer because a snapshot file doesn't
+//! contain a snapshot at a specific LSN, but rather all the page versions in a range
+//! of LSNs.
+//!
+//! Currently, a snapshot file contains full information needed to reconstruct any
+//! page version in the LSN range, without consulting any other snapshot files. When
+//! a new snapshot file is created for writing, the full contents of relation are
+//! materialized as it is at the beginning of the LSN range. That can be very expensive,
+//! we should find a way to store differential files. But this keeps the read-side
+//! of things simple. You can find the correct snapshot file based on RelishTag and
+//! timeline+LSN, and once you've located it, you have all the data you need to in that
+//! file.
+//!
+//! When a snapshot file needs to be accessed, we slurp the whole file into memory, into
+//! the SnapshotLayer struct. See load() and unload() functions.
+//!
+//! On disk, the snapshot files are stored in timelines/<timelineid> directory.
+//! Currently, there are no subdirectories, and each snapshot file is named like this:
+//!
+//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<start LSN>_<end LSN>
+//!
+//! For example:
+//!
+//!    1663_13990_2609_0_000000000169C348_000000000169C349
+//!
+//! If a relation is dropped, we add a '_DROPPED' to the end of the filename to indicate that.
+//! So the above example would become:
+//!
+//!    1663_13990_2609_0_000000000169C348_000000000169C349_DROPPED
+//!
+//! The end LSN indicates when it was dropped in that case, we don't store it in the
+//! file contents in any way.
+//!
+//! A snapshot file is constructed using the 'bookfile' crate. Each file consists of two
+//! parts: the page versions and the relation sizes. They are stored as separate chapters.
+//!
+use crate::layered_repository::storage_layer::{
+    Layer, PageReconstructData, PageVersion, SegmentTag,
+};
+use crate::relish::*;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
+use anyhow::{bail, Result};
+use log::*;
+use std::collections::BTreeMap;
+use std::fmt;
+use std::fs;
+use std::fs::File;
+use std::io::Write;
+use std::ops::Bound::Included;
+use std::path::PathBuf;
+use std::sync::{Arc, Mutex, MutexGuard};
+
+use bookfile::{Book, BookWriter};
+
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
+
+// Magic constant to identify a Zenith snapshot file
+static SNAPSHOT_FILE_MAGIC: u32 = 0x5A616E01;
+
+static PAGE_VERSIONS_CHAPTER: u64 = 1;
+static REL_SIZES_CHAPTER: u64 = 2;
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+struct SnapshotFileName {
+    seg: SegmentTag,
+    start_lsn: Lsn,
+    end_lsn: Lsn,
+    dropped: bool,
+}
+
+impl SnapshotFileName {
+    fn from_str(fname: &str) -> Option<Self> {
+        // Split the filename into parts
+        //
+        //    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>
+        //
+        // or if it was dropped:
+        //
+        //    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>_DROPPED
+        //
+        let rel;
+        let mut parts;
+        if let Some(rest) = fname.strip_prefix("rel_") {
+            parts = rest.split('_');
+            rel = RelishTag::Relation(RelTag {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+                relnode: parts.next()?.parse::<u32>().ok()?,
+                forknum: parts.next()?.parse::<u8>().ok()?,
+            });
+        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
+            parts = rest.split('_');
+            rel = RelishTag::FileNodeMap {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
+            parts = rest.split('_');
+            rel = RelishTag::TwoPhase {
+                xid: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
+            parts = rest.split('_');
+            rel = RelishTag::Checkpoint;
+        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
+            parts = rest.split('_');
+            rel = RelishTag::ControlFile;
+        } else {
+            return None;
+        }
+
+        let segno = parts.next()?.parse::<u32>().ok()?;
+
+        let seg = SegmentTag { rel, segno };
+
+        let start_lsn = Lsn::from_hex(parts.next()?).ok()?;
+        let end_lsn = Lsn::from_hex(parts.next()?).ok()?;
+
+        let mut dropped = false;
+        if let Some(suffix) = parts.next() {
+            if suffix == "DROPPED" {
+                dropped = true;
+            } else {
+                warn!("unrecognized filename in timeline dir: {}", fname);
+                return None;
+            }
+        }
+        if parts.next().is_some() {
+            warn!("unrecognized filename in timeline dir: {}", fname);
+            return None;
+        }
+
+        Some(SnapshotFileName {
+            seg,
+            start_lsn,
+            end_lsn,
+            dropped,
+        })
+    }
+
+    fn to_string(&self) -> String {
+        let basename = match self.seg.rel {
+            RelishTag::Relation(reltag) => format!(
+                "rel_{}_{}_{}_{}",
+                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
+            ),
+            RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno,
+            } => format!("pg_xact_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno,
+            } => format!("pg_multixact_members_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno,
+            } => format!("pg_multixact_offsets_{:04X}", segno),
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
+            RelishTag::Checkpoint => format!("pg_control_checkpoint"),
+            RelishTag::ControlFile => format!("pg_control"),
+        };
+
+        format!(
+            "{}_{}_{:016X}_{:016X}{}",
+            basename,
+            self.seg.segno,
+            u64::from(self.start_lsn),
+            u64::from(self.end_lsn),
+            if self.dropped { "_DROPPED" } else { "" }
+        )
+    }
+}
+
+impl fmt::Display for SnapshotFileName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.to_string())
+    }
+}
+
+///
+/// SnapshotLayer is the in-memory data structure associated with an
+/// on-disk snapshot file.  We keep a SnapshotLayer in memory for each
+/// file, in the LayerMap. If a layer is in "loaded" state, we have a
+/// copy of the file in memory, in 'inner'. Otherwise the struct is
+/// just a placeholder for a file that exists on disk, and it needs to
+/// be loaded before using it in queries.
+///
+pub struct SnapshotLayer {
+    conf: &'static PageServerConf,
+    pub tenantid: ZTenantId,
+    pub timelineid: ZTimelineId,
+    pub seg: SegmentTag,
+
+    //
+    // This entry contains all the changes from 'start_lsn' to 'end_lsn'. The
+    // start is inclusive, and end is exclusive.
+    pub start_lsn: Lsn,
+    pub end_lsn: Lsn,
+
+    dropped: bool,
+
+    inner: Mutex<SnapshotLayerInner>,
+}
+
+pub struct SnapshotLayerInner {
+    /// If false, the 'page_versions' and 'relsizes' have not been
+    /// loaded into memory yet.
+    loaded: bool,
+
+    /// All versions of all pages in the file are are kept here.
+    /// Indexed by block number and LSN.
+    page_versions: BTreeMap<(u32, Lsn), PageVersion>,
+
+    /// `relsizes` tracks the size of the relation at different points in time.
+    relsizes: BTreeMap<Lsn, u32>,
+}
+
+impl Layer for SnapshotLayer {
+    fn get_timeline_id(&self) -> ZTimelineId {
+        return self.timelineid;
+    }
+
+    fn get_seg_tag(&self) -> SegmentTag {
+        return self.seg;
+    }
+
+    fn is_dropped(&self) -> bool {
+        return self.dropped;
+    }
+
+    fn get_start_lsn(&self) -> Lsn {
+        return self.start_lsn;
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
+        return self.end_lsn;
+    }
+
+    /// Look up given page in the cache.
+    fn get_page_reconstruct_data(
+        &self,
+        blknum: u32,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<Option<Lsn>> {
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let mut need_base_image_lsn: Option<Lsn> = Some(lsn);
+        {
+            let inner = self.load()?;
+            let minkey = (blknum, Lsn(0));
+            let maxkey = (blknum, lsn);
+            let mut iter = inner
+                .page_versions
+                .range((Included(&minkey), Included(&maxkey)));
+            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
+                if let Some(img) = &entry.page_image {
+                    reconstruct_data.page_img = Some(img.clone());
+                    need_base_image_lsn = None;
+                    break;
+                } else if let Some(rec) = &entry.record {
+                    reconstruct_data.records.push(rec.clone());
+                    if rec.will_init {
+                        // This WAL record initializes the page, so no need to go further back
+                        need_base_image_lsn = None;
+                        break;
+                    } else {
+                        need_base_image_lsn = Some(*entry_lsn);
+                    }
+                } else {
+                    // No base image, and no WAL record. Huh?
+                    bail!("no page image or WAL record for requested page");
+                }
+            }
+
+            // release lock on 'inner'
+        }
+
+        Ok(need_base_image_lsn)
+    }
+
+    /// Get size of the relation at given LSN
+    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let inner = self.load()?;
+        let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn)));
+
+        if let Some((_entry_lsn, entry)) = iter.next_back() {
+            let result = *entry;
+            drop(inner);
+            trace!("get_seg_size: {} at {} -> {}", self.seg, lsn, result);
+            Ok(result)
+        } else {
+            error!(
+                "No size found for {} at {} in snapshot layer {} {}-{}",
+                self.seg, lsn, self.seg, self.start_lsn, self.end_lsn
+            );
+            bail!(
+                "No size found for {} at {} in snapshot layer",
+                self.seg,
+                lsn
+            );
+        }
+    }
+
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
+        // Is the requested LSN after the rel was dropped?
+        if self.dropped && lsn >= self.end_lsn {
+            return Ok(false);
+        }
+
+        // Otherwise, it exists.
+        Ok(true)
+    }
+}
+
+impl SnapshotLayer {
+    fn path(&self) -> PathBuf {
+        Self::path_for(
+            self.conf,
+            self.timelineid,
+            self.tenantid,
+            &SnapshotFileName {
+                seg: self.seg,
+                start_lsn: self.start_lsn,
+                end_lsn: self.end_lsn,
+                dropped: self.dropped,
+            },
+        )
+    }
+
+    fn path_for(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        fname: &SnapshotFileName,
+    ) -> PathBuf {
+        conf.timeline_path(&timelineid, &tenantid)
+            .join(fname.to_string())
+    }
+
+    /// Create a new snapshot file, using the given btreemaps containing the page versions and
+    /// relsizes.
+    ///
+    /// This is used to write the in-memory layer to disk. The in-memory layer uses the same
+    /// data structure with two btreemaps as we do, so passing the btreemaps is currently
+    /// expedient.
+    pub fn create(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        seg: SegmentTag,
+        start_lsn: Lsn,
+        end_lsn: Lsn,
+        dropped: bool,
+        page_versions: BTreeMap<(u32, Lsn), PageVersion>,
+        relsizes: BTreeMap<Lsn, u32>,
+    ) -> Result<SnapshotLayer> {
+        let snapfile = SnapshotLayer {
+            conf: conf,
+            timelineid: timelineid,
+            tenantid: tenantid,
+            seg: seg,
+            start_lsn: start_lsn,
+            end_lsn,
+            dropped,
+            inner: Mutex::new(SnapshotLayerInner {
+                loaded: true,
+                page_versions: page_versions,
+                relsizes: relsizes,
+            }),
+        };
+        let inner = snapfile.inner.lock().unwrap();
+
+        // Write the in-memory btreemaps into a file
+        let path = snapfile.path();
+
+        // Note: This overwrites any existing file. There shouldn't be any.
+        // FIXME: throw an error instead?
+        let file = File::create(&path)?;
+        let book = BookWriter::new(file, SNAPSHOT_FILE_MAGIC)?;
+
+        // Write out page versions
+        let mut chapter = book.new_chapter(PAGE_VERSIONS_CHAPTER);
+        let buf = BTreeMap::ser(&inner.page_versions)?;
+        chapter.write_all(&buf)?;
+        let book = chapter.close()?;
+
+        // and relsizes to separate chapter
+        let mut chapter = book.new_chapter(REL_SIZES_CHAPTER);
+        let buf = BTreeMap::ser(&inner.relsizes)?;
+        chapter.write_all(&buf)?;
+        let book = chapter.close()?;
+
+        book.close()?;
+
+        trace!("saved {}", &path.display());
+
+        drop(inner);
+
+        Ok(snapfile)
+    }
+
+    ///
+    /// Load the contents of the file into memory
+    ///
+    fn load(&self) -> Result<MutexGuard<SnapshotLayerInner>> {
+        // quick exit if already loaded
+        let mut inner = self.inner.lock().unwrap();
+
+        if inner.loaded {
+            return Ok(inner);
+        }
+
+        let path = Self::path_for(
+            self.conf,
+            self.timelineid,
+            self.tenantid,
+            &SnapshotFileName {
+                seg: self.seg,
+                start_lsn: self.start_lsn,
+                end_lsn: self.end_lsn,
+                dropped: self.dropped,
+            },
+        );
+
+        let file = File::open(&path)?;
+        let book = Book::new(file)?;
+
+        let chapter = book.read_chapter(PAGE_VERSIONS_CHAPTER)?;
+        let page_versions = BTreeMap::des(&chapter)?;
+
+        let chapter = book.read_chapter(REL_SIZES_CHAPTER)?;
+        let relsizes = BTreeMap::des(&chapter)?;
+
+        debug!("loaded from {}", &path.display());
+
+        *inner = SnapshotLayerInner {
+            loaded: true,
+            page_versions,
+            relsizes,
+        };
+
+        Ok(inner)
+    }
+
+    /// Create SnapshotLayers representing all files on disk
+    ///
+    // TODO: returning an Iterator would be more idiomatic
+    pub fn list_snapshot_files(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+    ) -> Result<Vec<Arc<SnapshotLayer>>> {
+        let path = conf.timeline_path(&timelineid, &tenantid);
+
+        let mut snapfiles: Vec<Arc<SnapshotLayer>> = Vec::new();
+        for direntry in fs::read_dir(path)? {
+            let fname = direntry?.file_name();
+            let fname = fname.to_str().unwrap();
+
+            if let Some(snapfilename) = SnapshotFileName::from_str(fname) {
+                let snapfile = SnapshotLayer {
+                    conf,
+                    timelineid,
+                    tenantid,
+                    seg: snapfilename.seg,
+                    start_lsn: snapfilename.start_lsn,
+                    end_lsn: snapfilename.end_lsn,
+                    dropped: snapfilename.dropped,
+                    inner: Mutex::new(SnapshotLayerInner {
+                        loaded: false,
+                        page_versions: BTreeMap::new(),
+                        relsizes: BTreeMap::new(),
+                    }),
+                };
+
+                snapfiles.push(Arc::new(snapfile));
+            }
+        }
+        return Ok(snapfiles);
+    }
+
+    pub fn delete(&self) -> Result<()> {
+        // delete underlying file
+        fs::remove_file(self.path())?;
+        Ok(())
+    }
+
+    ///
+    /// Release most of the memory used by this layer. If it's accessed again later,
+    /// it will need to be loaded back.
+    ///
+    pub fn unload(&self) -> Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.page_versions = BTreeMap::new();
+        inner.relsizes = BTreeMap::new();
+        inner.loaded = false;
+        Ok(())
+    }
+
+    /// debugging function to print out the contents of the layer
+    #[allow(unused)]
+    pub fn dump(&self) -> String {
+        let mut result = format!(
+            "----- snapshot layer for {} {}-{} ----\n",
+            self.seg, self.start_lsn, self.end_lsn
+        );
+
+        let inner = self.inner.lock().unwrap();
+        for (k, v) in inner.relsizes.iter() {
+            result += &format!("{}: {}\n", k, v);
+        }
+        //for (k, v) in inner.page_versions.iter() {
+        //    result += &format!("blk {} at {}: {}/{}\n", k.0, k.1, v.page_image.is_some(), v.record.is_some());
+        //}
+
+        result
+    }
+}

pageserver/src/layered_repository/storage_layer.rs

@@ -0,0 +1,128 @@
+//!
+//! Common traits and structs for layers
+//!
+
+use crate::relish::RelishTag;
+use crate::repository::WALRecord;
+use crate::ZTimelineId;
+use anyhow::Result;
+use bytes::Bytes;
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+use zenith_utils::lsn::Lsn;
+
+// Size of one segment in pages (10 MB)
+pub const RELISH_SEG_SIZE: u32 = 10 * 1024 * 1024 / 8192;
+
+///
+/// Each relish stored in the repository is divided into fixed-sized "segments",
+/// with 10 MB of key-space, or 1280 8k pages each.
+///
+#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy)]
+pub struct SegmentTag {
+    pub rel: RelishTag,
+    pub segno: u32,
+}
+
+impl fmt::Display for SegmentTag {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}.{}", self.rel, self.segno)
+    }
+}
+
+impl SegmentTag {
+    pub const fn from_blknum(rel: RelishTag, blknum: u32) -> SegmentTag {
+        SegmentTag {
+            rel,
+            segno: blknum / RELISH_SEG_SIZE,
+        }
+    }
+
+    pub fn blknum_in_seg(&self, blknum: u32) -> bool {
+        blknum / RELISH_SEG_SIZE == self.segno
+    }
+}
+
+///
+/// Represents a version of a page at a specific LSN. The LSN is the key of the
+/// entry in the 'page_versions' hash, it is not duplicated here.
+///
+/// A page version can be stored as a full page image, or as WAL record that needs
+/// to be applied over the previous page version to reconstruct this version.
+///
+/// It's also possible to have both a WAL record and a page image in the same
+/// PageVersion. That happens if page version is originally stored as a WAL record
+/// but it is later reconstructed by a GetPage@LSN request by performing WAL
+/// redo. The get_page_at_lsn() code will store the reconstructed pag image next to
+/// the WAL record in that case. TODO: That's pretty accidental, not the result
+/// of any grand design. If we want to keep reconstructed page versions around, we
+/// probably should have a separate buffer cache so that we could control the
+/// replacement policy globally. Or if we keep a reconstructed page image, we
+/// could throw away the WAL record.
+///
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PageVersion {
+    /// an 8kb page image
+    pub page_image: Option<Bytes>,
+    /// WAL record to get from previous page version to this one.
+    pub record: Option<WALRecord>,
+}
+
+///
+/// Data needed to reconstruct a page version
+///
+/// 'page_img' is the old base image of the page to start the WAL replay with.
+/// It can be None, if the first WAL record initializes the page (will_init)
+/// 'records' contains the records to apply over the base image.
+///
+pub struct PageReconstructData {
+    pub records: Vec<WALRecord>,
+    pub page_img: Option<Bytes>,
+}
+
+///
+/// A Layer holds all page versions for one segment of a relish, in a range of LSNs.
+/// There are two kinds of layers, in-memory and snapshot layers. In-memory
+/// layers are used to ingest incoming WAL, and provide fast access
+/// to the recent page versions. Snaphot layers are stored on disk, and
+/// are immutable. This trait presents the common functionality of
+/// in-memory and snapshot layers.
+///
+/// Each layer contains a full snapshot of the segment at the start
+/// LSN. In addition to that, it contains WAL (or more page images)
+/// needed to recontruct any page version up to the end LSN.
+///
+pub trait Layer: Send + Sync {
+    // These functions identify the relish segment and the LSN range
+    // that this Layer holds.
+    fn get_timeline_id(&self) -> ZTimelineId;
+    fn get_seg_tag(&self) -> SegmentTag;
+    fn get_start_lsn(&self) -> Lsn;
+    fn get_end_lsn(&self) -> Lsn;
+    fn is_dropped(&self) -> bool;
+
+    ///
+    /// Return data needed to reconstruct given page at LSN.
+    ///
+    /// It is up to the caller to collect more data from previous layer and
+    /// perform WAL redo, if necessary.
+    ///
+    /// If returns Some, the returned data is not complete. The caller needs
+    /// to continue with the returned 'lsn'.
+    ///
+    /// Note that the 'blknum' is the offset of the page from the beginning
+    /// of the *relish*, not the beginning of the segment. The requested
+    /// 'blknum' must be covered by this segment.
+    fn get_page_reconstruct_data(
+        &self,
+        blknum: u32,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<Option<Lsn>>;
+
+    // Functions that correspond to the Timeline trait functions.
+    fn get_seg_size(&self, lsn: Lsn) -> Result<u32>;
+
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool>;
+}

pageserver/src/lib.rs

@@ -1,31 +1,46 @@
-use serde::{Deserialize, Serialize};
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
-use std::fmt;
-use std::net::SocketAddr;
 use std::path::PathBuf;
-use std::str::FromStr;
 use std::time::Duration;
 
+use lazy_static::lazy_static;
+use zenith_metrics::{register_int_gauge_vec, IntGaugeVec};
+
 pub mod basebackup;
 pub mod branches;
+pub mod layered_repository;
+pub mod logger;
+pub mod object_key;
+pub mod object_repository;
+pub mod object_store;
 pub mod page_cache;
 pub mod page_service;
+pub mod relish;
 pub mod repository;
 pub mod restore_local_repo;
-pub mod tui;
-pub mod tui_event;
-mod tui_logger;
+pub mod rocksdb_storage;
 pub mod waldecoder;
 pub mod walreceiver;
 pub mod walredo;
 
+lazy_static! {
+    static ref LIVE_CONNECTIONS_COUNT: IntGaugeVec = register_int_gauge_vec!(
+        "pageserver_live_connections_count",
+        "Number of live network connections",
+        &["pageserver_connection_kind"]
+    )
+    .expect("failed to define a metric");
+}
+
 #[derive(Debug, Clone)]
 pub struct PageServerConf {
     pub daemonize: bool,
-    pub interactive: bool,
-    pub listen_addr: SocketAddr,
+    pub listen_addr: String,
+    pub http_endpoint_addr: String,
     pub gc_horizon: u64,
     pub gc_period: Duration,
+    pub superuser: String,
 
     // Repository directory, relative to current working directory.
     // Normally, the page server changes the current working directory
@@ -36,6 +51,18 @@ pub struct PageServerConf {
     pub workdir: PathBuf,
 
     pub pg_distrib_dir: PathBuf,
+
+    pub auth_type: AuthType,
+
+    pub auth_validation_public_key_path: Option<PathBuf>,
+
+    pub repository_format: RepositoryFormat,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum RepositoryFormat {
+    Layered,
+    RocksDb,
 }
 
 impl PageServerConf {
@@ -43,24 +70,44 @@ impl PageServerConf {
     // Repository paths, relative to workdir.
     //
 
-    fn tag_path(&self, name: &str) -> PathBuf {
-        self.workdir.join("refs").join("tags").join(name)
+    fn tenants_path(&self) -> PathBuf {
+        self.workdir.join("tenants")
     }
 
-    fn branch_path(&self, name: &str) -> PathBuf {
-        self.workdir.join("refs").join("branches").join(name)
+    fn tenant_path(&self, tenantid: &ZTenantId) -> PathBuf {
+        self.tenants_path().join(tenantid.to_string())
     }
 
-    fn timeline_path(&self, timelineid: ZTimelineId) -> PathBuf {
-        self.workdir.join("timelines").join(timelineid.to_string())
+    fn tags_path(&self, tenantid: &ZTenantId) -> PathBuf {
+        self.tenant_path(tenantid).join("refs").join("tags")
     }
 
-    fn snapshots_path(&self, timelineid: ZTimelineId) -> PathBuf {
-        self.timeline_path(timelineid).join("snapshots")
+    fn tag_path(&self, tag_name: &str, tenantid: &ZTenantId) -> PathBuf {
+        self.tags_path(tenantid).join(tag_name)
     }
 
-    fn ancestor_path(&self, timelineid: ZTimelineId) -> PathBuf {
-        self.timeline_path(timelineid).join("ancestor")
+    fn branches_path(&self, tenantid: &ZTenantId) -> PathBuf {
+        self.tenant_path(tenantid).join("refs").join("branches")
+    }
+
+    fn branch_path(&self, branch_name: &str, tenantid: &ZTenantId) -> PathBuf {
+        self.branches_path(tenantid).join(branch_name)
+    }
+
+    fn timelines_path(&self, tenantid: &ZTenantId) -> PathBuf {
+        self.tenant_path(tenantid).join("timelines")
+    }
+
+    fn timeline_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
+        self.timelines_path(tenantid).join(timelineid.to_string())
+    }
+
+    fn ancestor_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
+        self.timeline_path(timelineid, tenantid).join("ancestor")
+    }
+
+    fn wal_dir_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
+        self.timeline_path(timelineid, tenantid).join("wal")
     }
 
     //
@@ -75,64 +122,3 @@ impl PageServerConf {
         self.pg_distrib_dir.join("lib")
     }
 }
-
-/// Zenith Timeline ID is a 128-bit random ID.
-///
-/// Zenith timeline IDs are different from PostgreSQL timeline
-/// IDs. They serve a similar purpose though: they differentiate
-/// between different "histories" of the same cluster.  However,
-/// PostgreSQL timeline IDs are a bit cumbersome, because they are only
-/// 32-bits wide, and they must be in ascending order in any given
-/// timeline history.  Those limitations mean that we cannot generate a
-/// new PostgreSQL timeline ID by just generating a random number. And
-/// that in turn is problematic for the "pull/push" workflow, where you
-/// have a local copy of a zenith repository, and you periodically sync
-/// the local changes with a remote server. When you work "detached"
-/// from the remote server, you cannot create a PostgreSQL timeline ID
-/// that's guaranteed to be different from all existing timelines in
-/// the remote server. For example, if two people are having a clone of
-/// the repository on their laptops, and they both create a new branch
-/// with different name. What timeline ID would they assign to their
-/// branches? If they pick the same one, and later try to push the
-/// branches to the same remote server, they will get mixed up.
-///
-/// To avoid those issues, Zenith has its own concept of timelines that
-/// is separate from PostgreSQL timelines, and doesn't have those
-/// limitations. A zenith timeline is identified by a 128-bit ID, which
-/// is usually printed out as a hex string.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
-pub struct ZTimelineId([u8; 16]);
-
-impl FromStr for ZTimelineId {
-    type Err = hex::FromHexError;
-
-    fn from_str(s: &str) -> Result<ZTimelineId, Self::Err> {
-        let timelineid = hex::decode(s)?;
-
-        let mut buf: [u8; 16] = [0u8; 16];
-        buf.copy_from_slice(timelineid.as_slice());
-        Ok(ZTimelineId(buf))
-    }
-}
-
-impl ZTimelineId {
-    pub fn from(b: [u8; 16]) -> ZTimelineId {
-        ZTimelineId(b)
-    }
-
-    pub fn get_from_buf(buf: &mut dyn bytes::Buf) -> ZTimelineId {
-        let mut arr = [0u8; 16];
-        buf.copy_to_slice(&mut arr);
-        ZTimelineId::from(arr)
-    }
-
-    pub fn as_arr(&self) -> [u8; 16] {
-        self.0
-    }
-}
-
-impl fmt::Display for ZTimelineId {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_str(&hex::encode(self.0))
-    }
-}

pageserver/src/logger.rs

@@ -0,0 +1,45 @@
+use crate::PageServerConf;
+
+use anyhow::{Context, Result};
+use slog::{Drain, FnValue};
+use std::fs::{File, OpenOptions};
+
+pub fn init_logging(
+    _conf: &PageServerConf,
+    log_filename: &str,
+) -> Result<(slog_scope::GlobalLoggerGuard, File)> {
+    // Don't open the same file for output multiple times;
+    // the different fds could overwrite each other's output.
+    let log_file = OpenOptions::new()
+        .create(true)
+        .append(true)
+        .open(&log_filename)
+        .with_context(|| format!("failed to open {:?}", &log_filename))?;
+
+    let logger_file = log_file.try_clone().unwrap();
+
+    let decorator = slog_term::PlainSyncDecorator::new(logger_file);
+    let drain = slog_term::FullFormat::new(decorator).build();
+    let drain = slog::Filter::new(drain, |record: &slog::Record| {
+        if record.level().is_at_least(slog::Level::Info) {
+            return true;
+        }
+        false
+    });
+    let drain = std::sync::Mutex::new(drain).fuse();
+    let logger = slog::Logger::root(
+        drain,
+        slog::o!(
+            "location" =>
+                FnValue(move |record| {
+                    format!("{}, {}:{}",
+                            record.module(),
+                            record.file(),
+                            record.line()
+                    )
+                }
+                )
+        ),
+    );
+    Ok((slog_scope::set_global_logger(logger), log_file))
+}

pageserver/src/object_key.rs

@@ -0,0 +1,49 @@
+//!
+//! Common structs shared by object_repository.rs and object_store.rs.
+//!
+
+use crate::relish::RelishTag;
+use serde::{Deserialize, Serialize};
+use zenith_utils::zid::ZTimelineId;
+
+///
+/// ObjectKey is the key type used to identify objects stored in an object
+/// repository. It is shared between object_repository.rs and object_store.rs.
+/// It is mostly opaque to ObjectStore, it just stores and retrieves objects
+/// using the key given by the caller.
+///
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ObjectKey {
+    pub timeline: ZTimelineId,
+    pub tag: ObjectTag,
+}
+
+///
+/// ObjectTag is a part of ObjectKey that is specific to the type of
+/// the stored object.
+///
+/// NB: the order of the enum values is significant!  In particular,
+/// rocksdb_storage.rs assumes that TimelineMetadataTag is first
+///
+/// Buffer is the kind of object that is accessible by the public
+/// get_page_at_lsn() / put_page_image() / put_wal_record() functions in
+/// the repository.rs interface. The rest are internal objects stored in
+/// the key-value store, to store various metadata. They're not directly
+/// accessible outside object_repository.rs
+///
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub enum ObjectTag {
+    // dummy tag preceeding all other keys
+    FirstTag,
+
+    // Metadata about a timeline. Not versioned.
+    TimelineMetadataTag,
+
+    // These objects store metadata about one relish. Currently it's used
+    // just to track the relish's size. It's not used for non-blocky relishes
+    // at all.
+    RelationMetadata(RelishTag),
+
+    // These are the pages exposed in the public Repository/Timeline interface.
+    Buffer(RelishTag, u32),
+}

pageserver/src/object_store.rs

@@ -0,0 +1,92 @@
+//! Low-level key-value storage abstraction.
+//!
+use crate::object_key::*;
+use crate::relish::*;
+use anyhow::Result;
+use std::collections::HashSet;
+use std::iter::Iterator;
+use zenith_utils::lsn::Lsn;
+use zenith_utils::zid::ZTimelineId;
+
+///
+/// Low-level storage abstraction.
+///
+/// All the data in the repository is stored in a key-value store. This trait
+/// abstracts the details of the key-value store.
+///
+/// A simple key-value store would support just GET and PUT operations with
+/// a key, but the upper layer needs slightly complicated read operations
+///
+/// The most frequently used function is 'object_versions'. It is used
+/// to look up a page version. It is LSN aware, in that the caller
+/// specifies an LSN, and the function returns all values for that
+/// block with the same or older LSN.
+///
+pub trait ObjectStore: Send + Sync {
+    ///
+    /// Store a value with given key.
+    ///
+    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()>;
+
+    /// Read entry with the exact given key.
+    ///
+    /// This is used for retrieving metadata with special key that doesn't
+    /// correspond to any real relation.
+    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>>;
+
+    /// Read key greater or equal than specified
+    fn get_next_key(&self, key: &ObjectKey) -> Result<Option<ObjectKey>>;
+
+    /// Iterate through all page versions of one object.
+    ///
+    /// Returns all page versions in descending LSN order, along with the LSN
+    /// of each page version.
+    fn object_versions<'a>(
+        &'a self,
+        key: &ObjectKey,
+        lsn: Lsn,
+    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>>;
+
+    /// Iterate through versions of all objects in a timeline.
+    ///
+    /// Returns objects in increasing key-version order.
+    /// Returns all versions up to and including the specified LSN.
+    fn objects<'a>(
+        &'a self,
+        timeline: ZTimelineId,
+        lsn: Lsn,
+    ) -> Result<Box<dyn Iterator<Item = Result<(ObjectTag, Lsn, Vec<u8>)>> + 'a>>;
+
+    /// Iterate through all keys with given tablespace and database ID, and LSN <= 'lsn'.
+    /// Both dbnode and spcnode can be InvalidId (0) which means get all relations in tablespace/cluster
+    ///
+    /// This is used to implement 'create database'
+    fn list_rels(
+        &self,
+        timelineid: ZTimelineId,
+        spcnode: u32,
+        dbnode: u32,
+        lsn: Lsn,
+    ) -> Result<HashSet<RelTag>>;
+
+    /// Iterate through non-rel relishes
+    ///
+    /// This is used to prepare tarball for new node startup.
+    fn list_nonrels<'a>(&'a self, timelineid: ZTimelineId, lsn: Lsn) -> Result<HashSet<RelishTag>>;
+
+    /// Iterate through objects tags. If nonrel_only, then only non-relationa data is iterated.
+    ///
+    /// This is used to implement GC and preparing tarball for new node startup
+    /// Returns objects in increasing key-version order.
+    fn list_objects<'a>(
+        &'a self,
+        timelineid: ZTimelineId,
+        lsn: Lsn,
+    ) -> Result<Box<dyn Iterator<Item = ObjectTag> + 'a>>;
+
+    /// Unlink object (used by GC). This mehod may actually delete object or just mark it for deletion.
+    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()>;
+
+    // Compact storage and remove versions marged for deletion
+    fn compact(&self);
+}

pageserver/src/page_cache.rs

@@ -1,32 +1,91 @@
 //! This module acts as a switchboard to access different repositories managed by this
-//! page server. Currently, a Page Server can only manage one repository, so there
-//! isn't much here. If we implement multi-tenancy, this will probably be changed into
-//! a hash map, keyed by the tenant ID.
+//! page server.
 
-use crate::repository::rocksdb::RocksRepository;
+use crate::branches;
+use crate::layered_repository::LayeredRepository;
+use crate::object_repository::ObjectRepository;
 use crate::repository::Repository;
+use crate::rocksdb_storage::RocksObjectStore;
 use crate::walredo::PostgresRedoManager;
-use crate::PageServerConf;
+use crate::{PageServerConf, RepositoryFormat};
+use anyhow::{anyhow, bail, Result};
 use lazy_static::lazy_static;
+use log::info;
+use std::collections::HashMap;
+use std::fs;
+use std::str::FromStr;
 use std::sync::{Arc, Mutex};
+use zenith_utils::zid::ZTenantId;
 
 lazy_static! {
-    pub static ref REPOSITORY: Mutex<Option<Arc<dyn Repository + Send + Sync>>> = Mutex::new(None);
+    pub static ref REPOSITORY: Mutex<HashMap<ZTenantId, Arc<dyn Repository>>> =
+        Mutex::new(HashMap::new());
 }
 
 pub fn init(conf: &'static PageServerConf) {
     let mut m = REPOSITORY.lock().unwrap();
 
-    // Set up a WAL redo manager, for applying WAL records.
-    let walredo_mgr = PostgresRedoManager::new(conf);
+    for dir_entry in fs::read_dir(conf.tenants_path()).unwrap() {
+        let tenantid =
+            ZTenantId::from_str(dir_entry.unwrap().file_name().to_str().unwrap()).unwrap();
 
-    // we have already changed current dir to the repository.
-    let repo = RocksRepository::new(conf, Arc::new(walredo_mgr));
+        // Set up a WAL redo manager, for applying WAL records.
+        let walredo_mgr = PostgresRedoManager::new(conf, tenantid);
 
-    *m = Some(Arc::new(repo));
+        // Set up an object repository, for actual data storage.
+        let repo: Arc<dyn Repository + Sync + Send> = match conf.repository_format {
+            RepositoryFormat::Layered => {
+                let repo = Arc::new(LayeredRepository::new(
+                    conf,
+                    Arc::new(walredo_mgr),
+                    tenantid,
+                ));
+                LayeredRepository::launch_checkpointer_thread(conf, repo.clone());
+                repo
+            }
+            RepositoryFormat::RocksDb => {
+                let obj_store = RocksObjectStore::open(conf, &tenantid).unwrap();
+
+                Arc::new(ObjectRepository::new(
+                    conf,
+                    Arc::new(obj_store),
+                    Arc::new(walredo_mgr),
+                    tenantid,
+                ))
+            }
+        };
+
+        info!("initialized storage for tenant: {}", &tenantid);
+        m.insert(tenantid, repo);
+    }
 }
 
-pub fn get_repository() -> Arc<dyn Repository + Send + Sync> {
+pub fn create_repository_for_tenant(
+    conf: &'static PageServerConf,
+    tenantid: ZTenantId,
+) -> Result<()> {
+    let mut m = REPOSITORY.lock().unwrap();
+
+    // First check that the tenant doesn't exist already
+    if m.get(&tenantid).is_some() {
+        bail!("tenant {} already exists", tenantid);
+    }
+    let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenantid));
+    let repo = branches::create_repo(conf, tenantid, wal_redo_manager)?;
+
+    m.insert(tenantid, repo);
+
+    Ok(())
+}
+
+pub fn insert_repository_for_tenant(tenantid: ZTenantId, repo: Arc<dyn Repository>) {
+    let o = &mut REPOSITORY.lock().unwrap();
+    o.insert(tenantid, repo);
+}
+
+pub fn get_repository_for_tenant(tenantid: &ZTenantId) -> Result<Arc<dyn Repository>> {
     let o = &REPOSITORY.lock().unwrap();
-    Arc::clone(o.as_ref().unwrap())
+    o.get(tenantid)
+        .map(|repo| Arc::clone(repo))
+        .ok_or_else(|| anyhow!("repository not found for tenant name {}", tenantid))
 }

pageserver/src/relish.rs

@@ -0,0 +1,235 @@
+//!
+//! Zenith stores PostgreSQL relations, and some other files, in the
+//! repository.  The relations (i.e. tables and indexes) take up most
+//! of the space in a typical installation, while the other files are
+//! small. We call each relation and other file that is stored in the
+//! repository a "relish". It comes from "rel"-ish, as in "kind of a
+//! rel", because it covers relations as well as other things that are
+//! not relations, but are treated similarly for the purposes of the
+//! storage layer.
+//!
+//! This source file contains the definition of the RelishTag struct,
+//! which uniquely identifies a relish.
+//!
+//! Relishes come in two flavors: blocky and non-blocky. Relations and
+//! SLRUs are blocky, that is, they are divided into 8k blocks, and
+//! the repository tracks their size. Other relishes are non-blocky:
+//! the content of the whole relish is stored as one blob. Block
+//! number must be passed as 0 for all operations on a non-blocky
+//! relish. The one "block" that you store in a non-blocky relish can
+//! have arbitrary size, but they are expected to be small, or you
+//! will have performance issues.
+//!
+//! All relishes are versioned by LSN in the repository.
+//!
+
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+use postgres_ffi::relfile_utils::forknumber_to_name;
+use postgres_ffi::{Oid, TransactionId};
+
+///
+/// RelishTag identifies one relish.
+///
+#[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub enum RelishTag {
+    // Relations correspond to PostgreSQL relation forks. Each
+    // PostgreSQL relation fork is considered a separate relish.
+    Relation(RelTag),
+
+    // SLRUs include pg_clog, pg_multixact/members, and
+    // pg_multixact/offsets. There are other SLRUs in PostgreSQL, but
+    // they don't need to be stored permanently (e.g. pg_subtrans),
+    // or we do not support them in zenith yet (pg_commit_ts).
+    //
+    // These are currently never requested directly by the compute
+    // nodes, although in principle that would be possible. However,
+    // when a new compute node is created, these are included in the
+    // tarball that we send to the compute node to initialize the
+    // PostgreSQL data directory.
+    //
+    // Each SLRU segment in PostgreSQL is considered a separate
+    // relish. For example, pg_clog/0000, pg_clog/0001, and so forth.
+    //
+    // SLRU segments are divided into blocks, like relations.
+    Slru { slru: SlruKind, segno: u32 },
+
+    // Miscellaneous other files that need to be included in the
+    // tarball at compute node creation. These are non-blocky, and are
+    // expected to be small.
+
+    //
+    // FileNodeMap represents PostgreSQL's 'pg_filenode.map'
+    // files. They are needed to map catalog table OIDs to filenode
+    // numbers. Usually the mapping is done by looking up a relation's
+    // 'relfilenode' field in the 'pg_class' system table, but that
+    // doesn't work for 'pg_class' itself and a few other such system
+    // relations. See PostgreSQL relmapper.c for details.
+    //
+    // Each database has a map file for its local mapped catalogs,
+    // and there is a separate map file for shared catalogs.
+    //
+    // These files are always 512 bytes long (although we don't check
+    // or care about that in the page server).
+    //
+    FileNodeMap { spcnode: Oid, dbnode: Oid },
+
+    //
+    // State files for prepared transactions (e.g pg_twophase/1234)
+    //
+    TwoPhase { xid: TransactionId },
+
+    // The control file, stored in global/pg_control
+    ControlFile,
+
+    // Special entry that represents PostgreSQL checkpoint. It doesn't
+    // correspond to to any physical file in PostgreSQL, but we use it
+    // to track fields needed to restore the checkpoint data in the
+    // control file, when a compute node is created.
+    Checkpoint,
+}
+
+impl RelishTag {
+    pub const fn is_blocky(&self) -> bool {
+        match self {
+            // These relishes work with blocks
+            RelishTag::Relation(_) | RelishTag::Slru { slru: _, segno: _ } => true,
+
+            // and these don't
+            RelishTag::FileNodeMap {
+                spcnode: _,
+                dbnode: _,
+            }
+            | RelishTag::TwoPhase { xid: _ }
+            | RelishTag::ControlFile
+            | RelishTag::Checkpoint => false,
+        }
+    }
+
+    // Physical relishes represent files and use
+    // RelationSizeEntry to track existing and dropped files.
+    // They can be both blocky and non-blocky.
+    pub const fn is_physical(&self) -> bool {
+        match self {
+            // These relishes represent physical files
+            RelishTag::Relation(_)
+            | RelishTag::Slru { .. }
+            | RelishTag::FileNodeMap { .. }
+            | RelishTag::TwoPhase { .. } => true,
+
+            // and these don't
+            RelishTag::ControlFile | RelishTag::Checkpoint => false,
+        }
+    }
+
+    // convenience function to check if this relish is a normal relation.
+    pub const fn is_relation(&self) -> bool {
+        if let RelishTag::Relation(_) = self {
+            true
+        } else {
+            false
+        }
+    }
+}
+
+///
+/// Relation data file segment id throughout the Postgres cluster.
+///
+/// Every data file in Postgres is uniquely identified by 4 numbers:
+/// - relation id / node (`relnode`)
+/// - database id (`dbnode`)
+/// - tablespace id (`spcnode`), in short this is a unique id of a separate
+///   directory to store data files.
+/// - forknumber (`forknum`) is used to split different kinds of data of the same relation
+///   between some set of files (`relnode`, `relnode_fsm`, `relnode_vm`).
+///
+/// In native Postgres code `RelFileNode` structure and individual `ForkNumber` value
+/// are used for the same purpose.
+/// [See more related comments here](https:///github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/relfilenode.h#L57).
+///
+#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy, Serialize, Deserialize)]
+pub struct RelTag {
+    pub forknum: u8,
+    pub spcnode: Oid,
+    pub dbnode: Oid,
+    pub relnode: Oid,
+}
+
+/// Display RelTag in the same format that's used in most PostgreSQL debug messages:
+///
+/// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
+///
+impl fmt::Display for RelTag {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if let Some(forkname) = forknumber_to_name(self.forknum) {
+            write!(
+                f,
+                "{}/{}/{}_{}",
+                self.spcnode, self.dbnode, self.relnode, forkname
+            )
+        } else {
+            write!(f, "{}/{}/{}", self.spcnode, self.dbnode, self.relnode)
+        }
+    }
+}
+
+/// Display RelTag in the same format that's used in most PostgreSQL debug messages:
+///
+/// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
+///
+impl fmt::Display for RelishTag {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            RelishTag::Relation(rel) => rel.fmt(f),
+            RelishTag::Slru { slru, segno } => {
+                // e.g. pg_clog/0001
+                write!(f, "{}/{:04X}", slru.to_str(), segno)
+            }
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                write!(f, "relmapper file for spc {} db {}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => {
+                write!(f, "pg_twophase/{:08X}", xid)
+            }
+            RelishTag::ControlFile => {
+                write!(f, "control file")
+            }
+            RelishTag::Checkpoint => {
+                write!(f, "checkpoint")
+            }
+        }
+    }
+}
+
+///
+/// Non-relation transaction status files (clog (a.k.a. pg_xact) and
+/// pg_multixact) in Postgres are handled by SLRU (Simple LRU) buffer,
+/// hence the name.
+///
+/// These files are global for a postgres instance.
+///
+/// These files are divided into segments, which are divided into
+/// pages of the same BLCKSZ as used for relation files.
+///
+#[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub enum SlruKind {
+    Clog,
+    MultiXactMembers,
+    MultiXactOffsets,
+}
+
+impl SlruKind {
+    pub fn to_str(&self) -> &'static str {
+        match self {
+            Self::Clog => "pg_xact",
+            Self::MultiXactMembers => "pg_multixact/members",
+            Self::MultiXactOffsets => "pg_multixact/offsets",
+        }
+    }
+}
+
+pub const FIRST_NONREL_RELISH_TAG: RelishTag = RelishTag::Slru {
+    slru: SlruKind::Clog,
+    segno: 0,
+};

pageserver/src/repository.rs

@@ -1,55 +1,141 @@
-pub mod rocksdb;
-
-use crate::waldecoder::{DecodedWALRecord, Oid, TransactionId, XlCreateDatabase, XlSmgrTruncate};
-use crate::ZTimelineId;
+use crate::object_key::*;
+use crate::relish::*;
 use anyhow::Result;
 use bytes::{Buf, BufMut, Bytes, BytesMut};
-use postgres_ffi::pg_constants;
-use postgres_ffi::relfile_utils::forknumber_to_name;
-use std::fmt;
+use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
+use std::iter::Iterator;
+use std::ops::AddAssign;
 use std::sync::Arc;
+use std::time::Duration;
 use zenith_utils::lsn::Lsn;
+use zenith_utils::zid::ZTimelineId;
 
 ///
 /// A repository corresponds to one .zenith directory. One repository holds multiple
 /// timelines, forked off from the same initial call to 'initdb'.
-pub trait Repository {
+pub trait Repository: Send + Sync {
     /// Get Timeline handle for given zenith timeline ID.
-    ///
-    /// The Timeline is expected to be already "open", i.e. `get_or_restore_timeline`
-    /// should've been called on it earlier already.
     fn get_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>>;
 
-    /// Get Timeline handle for given zenith timeline ID.
-    ///
-    /// Creates a new Timeline object if it's not "open" already.
-    fn get_or_restore_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>>;
+    /// Create a new, empty timeline. The caller is responsible for loading data into it
+    fn create_empty_timeline(
+        &self,
+        timelineid: ZTimelineId,
+        start_lsn: Lsn,
+    ) -> Result<Arc<dyn Timeline>>;
 
-    /// Create an empty timeline, without loading any data into it from possible on-disk snapshot.
-    ///
-    /// For unit tests.
-    #[cfg(test)]
-    fn create_empty_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>>;
+    /// Branch a timeline
+    fn branch_timeline(&self, src: ZTimelineId, dst: ZTimelineId, start_lsn: Lsn) -> Result<()>;
 
+    /// perform one garbage collection iteration.
+    /// garbage collection is periodically performed by gc thread,
+    /// but it can be explicitly requested through page server api.
+    ///
+    /// 'timelineid' specifies the timeline to GC, or None for all.
+    /// `horizon` specifies delta from last lsn to preserve all object versions (pitr interval).
+    /// `compact` parameter is used to force compaction of storage.
+    /// some storage implementation are based on lsm tree and require periodic merge (compaction).
+    /// usually storage implementation determines itself when compaction should be performed.
+    /// but for gc tests it way be useful to force compaction just after completion of gc iteration
+    /// to make sure that all detected garbage is removed.
+    /// so right now `compact` is set to true when gc explicitly requested through page srver api,
+    /// and is st to false in gc threads which infinitely repeats gc iterations in loop.
+    fn gc_iteration(
+        &self,
+        timelineid: Option<ZTimelineId>,
+        horizon: u64,
+        compact: bool,
+    ) -> Result<GcResult>;
+
+    // TODO get timelines?
     //fn get_stats(&self) -> RepositoryStats;
 }
 
-pub trait Timeline {
+///
+/// Result of performing GC
+///
+#[derive(Default)]
+pub struct GcResult {
+    // FIXME: These counters make sense for the ObjectRepository. They are not used
+    // by the LayeredRepository.
+    pub n_relations: u64,
+    pub inspected: u64,
+    pub truncated: u64,
+    pub deleted: u64,
+    pub prep_deleted: u64,        // RelishTag::Twophase
+    pub slru_deleted: u64,        // RelishTag::Slru
+    pub chkp_deleted: u64,        // RelishTag::Checkpoint
+    pub control_deleted: u64,     // RelishTag::ControlFile
+    pub filenodemap_deleted: u64, // RelishTag::FileNodeMap
+    pub dropped: u64,
+
+    // These are used for the LayeredRepository instead
+    pub snapshot_relfiles_total: u64,
+    pub snapshot_relfiles_needed_by_cutoff: u64,
+    pub snapshot_relfiles_needed_by_branches: u64,
+    pub snapshot_relfiles_not_updated: u64,
+    pub snapshot_relfiles_removed: u64, // # of snapshot files removed because they have been made obsolete by newer snapshot files.
+    pub snapshot_relfiles_dropped: u64, // # of snapshot files removed because the relation was dropped
+
+    pub snapshot_nonrelfiles_total: u64,
+    pub snapshot_nonrelfiles_needed_by_cutoff: u64,
+    pub snapshot_nonrelfiles_needed_by_branches: u64,
+    pub snapshot_nonrelfiles_not_updated: u64,
+    pub snapshot_nonrelfiles_removed: u64, // # of snapshot files removed because they have been made obsolete by newer snapshot files.
+    pub snapshot_nonrelfiles_dropped: u64, // # of snapshot files removed because the relation was dropped
+
+    pub elapsed: Duration,
+}
+
+impl AddAssign for GcResult {
+    fn add_assign(&mut self, other: Self) {
+        self.n_relations += other.n_relations;
+        self.truncated += other.truncated;
+        self.deleted += other.deleted;
+        self.dropped += other.dropped;
+
+        self.snapshot_relfiles_total += other.snapshot_relfiles_total;
+        self.snapshot_relfiles_needed_by_cutoff += other.snapshot_relfiles_needed_by_cutoff;
+        self.snapshot_relfiles_needed_by_branches += other.snapshot_relfiles_needed_by_branches;
+        self.snapshot_relfiles_not_updated += other.snapshot_relfiles_not_updated;
+        self.snapshot_relfiles_removed += other.snapshot_relfiles_removed;
+        self.snapshot_relfiles_dropped += other.snapshot_relfiles_dropped;
+
+        self.snapshot_nonrelfiles_total += other.snapshot_nonrelfiles_total;
+        self.snapshot_nonrelfiles_needed_by_cutoff += other.snapshot_nonrelfiles_needed_by_cutoff;
+        self.snapshot_nonrelfiles_needed_by_branches +=
+            other.snapshot_nonrelfiles_needed_by_branches;
+        self.snapshot_nonrelfiles_not_updated += other.snapshot_nonrelfiles_not_updated;
+        self.snapshot_nonrelfiles_removed += other.snapshot_nonrelfiles_removed;
+        self.snapshot_nonrelfiles_dropped += other.snapshot_nonrelfiles_dropped;
+
+        self.elapsed += other.elapsed;
+    }
+}
+
+pub trait Timeline: Send + Sync {
     //------------------------------------------------------------------------------
     // Public GET functions
     //------------------------------------------------------------------------------
 
     /// Look up given page in the cache.
-    fn get_page_at_lsn(&self, tag: BufferTag, lsn: Lsn) -> Result<Bytes>;
+    fn get_page_at_lsn(&self, tag: RelishTag, blknum: u32, lsn: Lsn) -> Result<Bytes>;
 
-    /// Get size of relation
-    fn get_relsize(&self, tag: RelTag, lsn: Lsn) -> Result<u32>;
+    /// Look up given page in the cache.
+    fn get_page_at_lsn_nowait(&self, tag: RelishTag, blknum: u32, lsn: Lsn) -> Result<Bytes>;
+
+    /// Get size of a relish
+    fn get_relish_size(&self, tag: RelishTag, lsn: Lsn) -> Result<Option<u32>>;
 
     /// Does relation exist?
-    fn get_relsize_exists(&self, tag: RelTag, lsn: Lsn) -> Result<bool>;
+    fn get_rel_exists(&self, tag: RelishTag, lsn: Lsn) -> Result<bool>;
 
-    /// Get page image at the particular LSN
-    fn get_page_image(&self, tag: BufferTag, lsn: Lsn) -> Result<Option<Bytes>>;
+    /// Get a list of all distinct relations in given tablespace and database.
+    fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashSet<RelTag>>;
+
+    /// Get a list of non-relational objects
+    fn list_nonrels<'a>(&'a self, lsn: Lsn) -> Result<HashSet<RelishTag>>;
 
     //------------------------------------------------------------------------------
     // Public PUT functions, to update the repository with new page versions.
@@ -61,94 +147,27 @@ pub trait Timeline {
     ///
     /// This will implicitly extend the relation, if the page is beyond the
     /// current end-of-file.
-    fn put_wal_record(&self, tag: BufferTag, rec: WALRecord);
+    fn put_wal_record(&self, tag: RelishTag, blknum: u32, rec: WALRecord) -> Result<()>;
 
     /// Like put_wal_record, but with ready-made image of the page.
-    fn put_page_image(&self, tag: BufferTag, lsn: Lsn, img: Bytes);
-
-    /// Truncate relation
-    fn put_truncation(&self, rel: RelTag, lsn: Lsn, nblocks: u32) -> Result<()>;
-
-    /// Create a new database from a template database
-    ///
-    /// In PostgreSQL, CREATE DATABASE works by scanning the data directory and
-    /// copying all relation files from the template database. This is the equivalent
-    /// of that.
-    fn put_create_database(
+    fn put_page_image(
         &self,
+        tag: RelishTag,
+        blknum: u32,
         lsn: Lsn,
-        db_id: Oid,
-        tablespace_id: Oid,
-        src_db_id: Oid,
-        src_tablespace_id: Oid,
+        img: Bytes,
+        update_meta: bool,
     ) -> Result<()>;
 
-    ///
-    /// Helper function to parse a WAL record and call the above functions for all the
-    /// relations/pages that the record affects.
-    ///
-    fn save_decoded_record(
-        &self,
-        decoded: DecodedWALRecord,
-        recdata: Bytes,
-        lsn: Lsn,
-    ) -> Result<()> {
-        // Figure out which blocks the record applies to, and "put" a separate copy
-        // of the record for each block.
-        for blk in decoded.blocks.iter() {
-            let tag = BufferTag {
-                rel: RelTag {
-                    spcnode: blk.rnode_spcnode,
-                    dbnode: blk.rnode_dbnode,
-                    relnode: blk.rnode_relnode,
-                    forknum: blk.forknum as u8,
-                },
-                blknum: blk.blkno,
-            };
+    /// Truncate relation
+    fn put_truncation(&self, rel: RelishTag, lsn: Lsn, nblocks: u32) -> Result<()>;
 
-            let rec = WALRecord {
-                lsn,
-                will_init: blk.will_init || blk.apply_image,
-                rec: recdata.clone(),
-                main_data_offset: decoded.main_data_offset as u32,
-            };
+    /// Unlink relish.
+    /// This method is used for marking dropped relations and truncated SLRU segments
+    fn put_unlink(&self, tag: RelishTag, lsn: Lsn) -> Result<()>;
 
-            self.put_wal_record(tag, rec);
-        }
-
-        // Handle a few special record types
-        if decoded.xl_rmid == pg_constants::RM_SMGR_ID
-            && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK)
-                == pg_constants::XLOG_SMGR_TRUNCATE
-        {
-            let truncate = XlSmgrTruncate::decode(&decoded);
-            if (truncate.flags & pg_constants::SMGR_TRUNCATE_HEAP) != 0 {
-                let rel = RelTag {
-                    spcnode: truncate.rnode.spcnode,
-                    dbnode: truncate.rnode.dbnode,
-                    relnode: truncate.rnode.relnode,
-                    forknum: pg_constants::MAIN_FORKNUM,
-                };
-                self.put_truncation(rel, lsn, truncate.blkno)?;
-            }
-        } else if decoded.xl_rmid == pg_constants::RM_DBASE_ID
-            && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK)
-                == pg_constants::XLOG_DBASE_CREATE
-        {
-            let createdb = XlCreateDatabase::decode(&decoded);
-            self.put_create_database(
-                lsn,
-                createdb.db_id,
-                createdb.tablespace_id,
-                createdb.src_db_id,
-                createdb.src_tablespace_id,
-            )?;
-        }
-        // Now that this record has been handled, let the repository know that
-        // it is up-to-date to this LSN
-        self.advance_last_record_lsn(lsn);
-        Ok(())
-    }
+    /// Put raw data
+    fn put_raw_data(&self, tag: ObjectTag, lsn: Lsn, data: &[u8]) -> Result<()>;
 
     /// Remember the all WAL before the given LSN has been processed.
     ///
@@ -167,15 +186,54 @@ pub trait Timeline {
     fn advance_last_record_lsn(&self, lsn: Lsn);
     fn get_last_record_lsn(&self) -> Lsn;
 
-    /// Get range [begin,end) of stored blocks. Used mostly for SMGR pseudorelations
-    /// but can be also applied to normal relations.
-    fn get_range(&self, rel: RelTag, lsn: Lsn) -> Result<(u32, u32)>;
+    // Like `advance_last_record_lsn`, but points to the start position of last record
+    fn get_prev_record_lsn(&self) -> Lsn;
 
-    /// Get vector of databases (represented using RelTag only dbnode and spcnode fields are used)
-    fn get_databases(&self, lsn: Lsn) -> Result<Vec<RelTag>>;
+    ///
+    /// Flush to disk all data that was written with the put_* functions
+    ///
+    /// NOTE: This has nothing to do with checkpoint in PostgreSQL. We don't
+    /// know anything about them here in the repository.
+    fn checkpoint(&self) -> Result<()>;
 
-    /// Get vector of prepared twophase transactions
-    fn get_twophase(&self, lsn: Lsn) -> Result<Vec<TransactionId>>;
+    /// Events for all relations in the timeline.
+    /// Contains updates from start up to the last valid LSN
+    /// at time of history() call. This lsn can be read via the lsn() function.
+    ///
+    /// Relation size is increased implicitly and decreased with Truncate updates.
+    // TODO ordering guarantee?
+    fn history<'a>(&'a self) -> Result<Box<dyn History + 'a>>;
+
+    //
+    // Wait until WAL has been received up to the given LSN.
+    //
+    fn wait_lsn(&self, lsn: Lsn) -> Result<Lsn>;
+}
+
+pub trait History: Iterator<Item = Result<Modification>> {
+    /// The last_valid_lsn at the time of history() call.
+    fn lsn(&self) -> Lsn;
+}
+
+//
+// Structure representing any update operation of object storage.
+// It is used to copy object storage content in PUSH method.
+//
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Modification {
+    pub tag: ObjectTag,
+    pub lsn: Lsn,
+    pub data: Vec<u8>,
+}
+
+impl Modification {
+    pub fn new(entry: (ObjectTag, Lsn, Vec<u8>)) -> Modification {
+        Modification {
+            tag: entry.0,
+            lsn: entry.1,
+            data: entry.2,
+        }
+    }
 }
 
 #[derive(Clone)]
@@ -186,81 +244,7 @@ pub struct RepositoryStats {
     pub num_getpage_requests: Lsn,
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy)]
-pub struct RelTag {
-    pub forknum: u8,
-    pub spcnode: u32,
-    pub dbnode: u32,
-    pub relnode: u32,
-}
-
-impl RelTag {
-    pub fn pack(&self, buf: &mut BytesMut) {
-        buf.put_u8(self.forknum);
-        buf.put_u32(self.spcnode);
-        buf.put_u32(self.dbnode);
-        buf.put_u32(self.relnode);
-    }
-    pub fn unpack(buf: &mut Bytes) -> RelTag {
-        RelTag {
-            forknum: buf.get_u8(),
-            spcnode: buf.get_u32(),
-            dbnode: buf.get_u32(),
-            relnode: buf.get_u32(),
-        }
-    }
-}
-
-/// Display RelTag in the same format that's used in most PostgreSQL debug messages:
-///
-/// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
-///
-impl fmt::Display for RelTag {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        if let Some(forkname) = forknumber_to_name(self.forknum) {
-            write!(
-                f,
-                "{}/{}/{}_{}",
-                self.spcnode, self.dbnode, self.relnode, forkname
-            )
-        } else {
-            write!(f, "{}/{}/{}", self.spcnode, self.dbnode, self.relnode)
-        }
-    }
-}
-
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
-pub struct BufferTag {
-    pub rel: RelTag,
-    pub blknum: u32,
-}
-
-impl BufferTag {
-    pub fn fork(forknum: u8) -> BufferTag {
-        BufferTag {
-            rel: RelTag {
-                forknum,
-                spcnode: 0,
-                dbnode: 0,
-                relnode: 0,
-            },
-            blknum: 0,
-        }
-    }
-
-    pub fn pack(&self, buf: &mut BytesMut) {
-        self.rel.pack(buf);
-        buf.put_u32(self.blknum);
-    }
-    pub fn unpack(buf: &mut Bytes) -> BufferTag {
-        BufferTag {
-            rel: RelTag::unpack(buf),
-            blknum: buf.get_u32(),
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct WALRecord {
     pub lsn: Lsn, // LSN at the *end* of the record
     pub will_init: bool,
@@ -300,31 +284,34 @@ impl WALRecord {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::layered_repository::LayeredRepository;
+    use crate::object_repository::ObjectRepository;
+    use crate::object_repository::{ObjectValue, PageEntry, RelationSizeEntry};
+    use crate::rocksdb_storage::RocksObjectStore;
     use crate::walredo::{WalRedoError, WalRedoManager};
-    use crate::PageServerConf;
+    use crate::{PageServerConf, RepositoryFormat};
     use postgres_ffi::pg_constants;
     use std::fs;
     use std::path::PathBuf;
     use std::str::FromStr;
     use std::time::Duration;
+    use zenith_utils::bin_ser::BeSer;
+    use zenith_utils::postgres_backend::AuthType;
+    use zenith_utils::zid::ZTenantId;
 
     /// Arbitrary relation tag, for testing.
-    const TESTREL_A: RelTag = RelTag {
+    const TESTREL_A: RelishTag = RelishTag::Relation(RelTag {
         spcnode: 0,
         dbnode: 111,
         relnode: 1000,
         forknum: 0,
-    };
-
-    /// Convenience function to create a BufferTag for testing.
-    /// Helps to keeps the tests shorter.
-    #[allow(non_snake_case)]
-    fn TEST_BUF(blknum: u32) -> BufferTag {
-        BufferTag {
-            rel: TESTREL_A,
-            blknum,
-        }
-    }
+    });
+    const TESTREL_B: RelishTag = RelishTag::Relation(RelTag {
+        spcnode: 0,
+        dbnode: 111,
+        relnode: 1001,
+        forknum: 0,
+    });
 
     /// Convenience function to create a page image with given string as the only content
     #[allow(non_snake_case)]
@@ -336,97 +323,127 @@ mod tests {
         buf.freeze()
     }
 
-    fn get_test_repo(test_name: &str) -> Result<Box<dyn Repository>> {
+    static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
+
+    fn get_test_repo(
+        test_name: &str,
+        repository_format: RepositoryFormat,
+    ) -> Result<Box<dyn Repository>> {
         let repo_dir = PathBuf::from(format!("../tmp_check/test_{}", test_name));
         let _ = fs::remove_dir_all(&repo_dir);
         fs::create_dir_all(&repo_dir)?;
+        fs::create_dir_all(&repo_dir.join("timelines"))?;
 
         let conf = PageServerConf {
             daemonize: false,
-            interactive: false,
             gc_horizon: 64 * 1024 * 1024,
             gc_period: Duration::from_secs(10),
-            listen_addr: "127.0.0.1:5430".parse().unwrap(),
-            workdir: repo_dir.into(),
+            listen_addr: "127.0.0.1:5430".to_string(),
+            http_endpoint_addr: "127.0.0.1:9898".to_string(),
+            superuser: "zenith_admin".to_string(),
+            workdir: repo_dir,
             pg_distrib_dir: "".into(),
+            auth_type: AuthType::Trust,
+            auth_validation_public_key_path: None,
+            repository_format,
         };
         // Make a static copy of the config. This can never be free'd, but that's
         // OK in a test.
         let conf: &'static PageServerConf = Box::leak(Box::new(conf));
+        let tenantid = ZTenantId::generate();
+        fs::create_dir_all(conf.tenant_path(&tenantid)).unwrap();
 
         let walredo_mgr = TestRedoManager {};
 
-        let repo = rocksdb::RocksRepository::new(conf, Arc::new(walredo_mgr));
+        let repo: Box<dyn Repository + Sync + Send> = match conf.repository_format {
+            RepositoryFormat::Layered => Box::new(LayeredRepository::new(
+                conf,
+                Arc::new(walredo_mgr),
+                tenantid,
+            )),
+            RepositoryFormat::RocksDb => {
+                let obj_store = RocksObjectStore::create(conf, &tenantid)?;
 
-        Ok(Box::new(repo))
+                Box::new(ObjectRepository::new(
+                    conf,
+                    Arc::new(obj_store),
+                    Arc::new(walredo_mgr),
+                    tenantid,
+                ))
+            }
+        };
+
+        Ok(repo)
     }
 
     /// Test get_relsize() and truncation.
-    ///
-    /// FIXME: The RocksRepository implementation returns wrong relation size, if
-    /// you make a request with an old LSN. It seems to ignore the requested LSN
-    /// and always return result as of latest LSN. For such cases, the expected
-    /// results below match the current RocksRepository behavior, so that the test
-    /// passes, and the actually correct answers are in comments like
-    /// "// CORRECT: <correct answer>"
     #[test]
-    fn test_relsize() -> Result<()> {
+    fn test_relsize_rocksdb() -> Result<()> {
+        let repo = get_test_repo("test_relsize_rocksdb", RepositoryFormat::RocksDb)?;
+        test_relsize(&*repo)
+    }
+
+    #[test]
+    fn test_relsize_layered() -> Result<()> {
+        let repo = get_test_repo("test_relsize_layered", RepositoryFormat::Layered)?;
+        test_relsize(&*repo)
+    }
+
+    fn test_relsize(repo: &dyn Repository) -> Result<()> {
         // get_timeline() with non-existent timeline id should fail
         //repo.get_timeline("11223344556677881122334455667788");
 
         // Create timeline to work on
-        let repo = get_test_repo("test_relsize")?;
         let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid)?;
+        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
 
         tline.init_valid_lsn(Lsn(1));
-        tline.put_page_image(TEST_BUF(0), Lsn(2), TEST_IMG("foo blk 0 at 2"));
-        tline.put_page_image(TEST_BUF(0), Lsn(2), TEST_IMG("foo blk 0 at 2"));
-        tline.put_page_image(TEST_BUF(0), Lsn(3), TEST_IMG("foo blk 0 at 3"));
-        tline.put_page_image(TEST_BUF(1), Lsn(4), TEST_IMG("foo blk 1 at 4"));
-        tline.put_page_image(TEST_BUF(2), Lsn(5), TEST_IMG("foo blk 2 at 5"));
+        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"), true)?;
+        tline.put_page_image(TESTREL_A, 1, Lsn(4), TEST_IMG("foo blk 1 at 4"), true)?;
+        tline.put_page_image(TESTREL_A, 2, Lsn(5), TEST_IMG("foo blk 2 at 5"), true)?;
 
         tline.advance_last_valid_lsn(Lsn(5));
 
-        // rocksdb implementation erroneosly returns 'true' here
-        assert_eq!(tline.get_relsize_exists(TESTREL_A, Lsn(1))?, true); // CORRECT: false
-                                                                        // likewise, it returns wrong size here
-        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(1))?, 3); // CORRECT: 0 (or error?)
+        // The relation was created at LSN 2, not visible at LSN 1 yet.
+        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(1))?, false);
+        assert!(tline.get_relish_size(TESTREL_A, Lsn(1))?.is_none());
 
-        assert_eq!(tline.get_relsize_exists(TESTREL_A, Lsn(2))?, true);
-        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(2))?, 3); // CORRECT: 1
-        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(5))?, 3);
+        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(2))?, true);
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(2))?.unwrap(), 1);
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(5))?.unwrap(), 3);
 
         // Check page contents at each LSN
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(0), Lsn(2))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(2))?,
             TEST_IMG("foo blk 0 at 2")
         );
 
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(0), Lsn(3))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(3))?,
             TEST_IMG("foo blk 0 at 3")
         );
 
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(0), Lsn(4))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(4))?,
             TEST_IMG("foo blk 0 at 3")
         );
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(1), Lsn(4))?,
+            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(4))?,
             TEST_IMG("foo blk 1 at 4")
         );
 
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(0), Lsn(5))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(5))?,
             TEST_IMG("foo blk 0 at 3")
         );
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(1), Lsn(5))?,
+            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(5))?,
             TEST_IMG("foo blk 1 at 4")
         );
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(2), Lsn(5))?,
+            tline.get_page_at_lsn(TESTREL_A, 2, Lsn(5))?,
             TEST_IMG("foo blk 2 at 5")
         );
 
@@ -435,20 +452,20 @@ mod tests {
         tline.advance_last_valid_lsn(Lsn(6));
 
         // Check reported size and contents after truncation
-        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(6))?, 2);
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(6))?.unwrap(), 2);
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(0), Lsn(6))?,
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(6))?,
             TEST_IMG("foo blk 0 at 3")
         );
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(1), Lsn(6))?,
+            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(6))?,
             TEST_IMG("foo blk 1 at 4")
         );
 
         // should still see the truncated block with older LSN
-        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(5))?, 2); // CORRECT: 3
+        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(5))?.unwrap(), 3);
         assert_eq!(
-            tline.get_page_at_lsn(TEST_BUF(2), Lsn(5))?,
+            tline.get_page_at_lsn(TESTREL_A, 2, Lsn(5))?,
             TEST_IMG("foo blk 2 at 5")
         );
 
@@ -461,23 +478,33 @@ mod tests {
     /// This isn't very interesting with the RocksDb implementation, as we don't pay
     /// any attention to Postgres segment boundaries there.
     #[test]
-    fn test_large_rel() -> Result<()> {
-        let repo = get_test_repo("test_large_rel")?;
+    fn test_large_rel_rocksdb() -> Result<()> {
+        let repo = get_test_repo("test_large_rel_rocksdb", RepositoryFormat::RocksDb)?;
+        test_large_rel(&*repo)
+    }
+
+    #[test]
+    fn test_large_rel_layered() -> Result<()> {
+        let repo = get_test_repo("test_large_rel_layered", RepositoryFormat::Layered)?;
+        test_large_rel(&*repo)
+    }
+
+    fn test_large_rel(repo: &dyn Repository) -> Result<()> {
         let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid)?;
+        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
 
         tline.init_valid_lsn(Lsn(1));
 
-        let mut lsn = 0;
-        for i in 0..pg_constants::RELSEG_SIZE + 1 {
-            let img = TEST_IMG(&format!("foo blk {} at {}", i, Lsn(lsn)));
+        let mut lsn = 1;
+        for blknum in 0..pg_constants::RELSEG_SIZE + 1 {
+            let img = TEST_IMG(&format!("foo blk {} at {}", blknum, Lsn(lsn)));
             lsn += 1;
-            tline.put_page_image(TEST_BUF(i as u32), Lsn(lsn), img);
+            tline.put_page_image(TESTREL_A, blknum as u32, Lsn(lsn), img, true)?;
         }
         tline.advance_last_valid_lsn(Lsn(lsn));
 
         assert_eq!(
-            tline.get_relsize(TESTREL_A, Lsn(lsn))?,
+            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
             pg_constants::RELSEG_SIZE + 1
         );
 
@@ -486,7 +513,7 @@ mod tests {
         tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE)?;
         tline.advance_last_valid_lsn(Lsn(lsn));
         assert_eq!(
-            tline.get_relsize(TESTREL_A, Lsn(lsn))?,
+            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
             pg_constants::RELSEG_SIZE
         );
 
@@ -495,10 +522,186 @@ mod tests {
         tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE - 1)?;
         tline.advance_last_valid_lsn(Lsn(lsn));
         assert_eq!(
-            tline.get_relsize(TESTREL_A, Lsn(lsn))?,
+            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
             pg_constants::RELSEG_SIZE - 1
         );
 
+        // Truncate to 1500, and then truncate all the way down to 0, one block at a time
+        // This tests the behavior at segment boundaries
+        let mut size: i32 = 3000;
+        while size >= 0 {
+            lsn += 1;
+            tline.put_truncation(TESTREL_A, Lsn(lsn), size as u32)?;
+            tline.advance_last_valid_lsn(Lsn(lsn));
+            assert_eq!(
+                tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
+                size as u32
+            );
+
+            size -= 1;
+        }
+
+        Ok(())
+    }
+
+    fn skip_nonrel_objects<'a>(
+        snapshot: Box<dyn History + 'a>,
+    ) -> Result<impl Iterator<Item = <dyn History as Iterator>::Item> + 'a> {
+        Ok(snapshot.skip_while(|r| match r {
+            Ok(m) => match m.tag {
+                ObjectTag::RelationMetadata(_) => false,
+                _ => true,
+            },
+            _ => panic!("Iteration error"),
+        }))
+    }
+
+    #[test]
+    fn test_branch_rocksdb() -> Result<()> {
+        let repo = get_test_repo("test_branch_rocksdb", RepositoryFormat::RocksDb)?;
+        test_branch(&*repo)
+    }
+
+    #[test]
+    fn test_branch_layered() -> Result<()> {
+        let repo = get_test_repo("test_branch_layered", RepositoryFormat::Layered)?;
+        test_branch(&*repo)
+    }
+
+    ///
+    /// Test branch creation
+    ///
+    fn test_branch(repo: &dyn Repository) -> Result<()> {
+        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
+        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
+
+        // Import initial dummy checkpoint record, otherwise the get_timeline() call
+        // after branching fails below
+        tline.put_page_image(RelishTag::Checkpoint, 0, Lsn(1), ZERO_PAGE.clone(), false)?;
+
+        // Create a relation on the timeline
+        tline.init_valid_lsn(Lsn(1));
+        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"), true)?;
+        tline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("foo blk 0 at 4"), true)?;
+
+        // Create another relation
+        tline.put_page_image(TESTREL_B, 0, Lsn(2), TEST_IMG("foobar blk 0 at 2"), true)?;
+
+        tline.advance_last_valid_lsn(Lsn(4));
+
+        // Branch the history, modify relation differently on the new timeline
+        let newtimelineid = ZTimelineId::from_str("AA223344556677881122334455667788").unwrap();
+        repo.branch_timeline(timelineid, newtimelineid, Lsn(3))?;
+        let newtline = repo.get_timeline(newtimelineid)?;
+
+        newtline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("bar blk 0 at 4"), true)?;
+        newtline.advance_last_valid_lsn(Lsn(4));
+
+        // Check page contents on both branches
+        assert_eq!(
+            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(4))?,
+            TEST_IMG("foo blk 0 at 4")
+        );
+
+        assert_eq!(
+            newtline.get_page_at_lsn(TESTREL_A, 0, Lsn(4))?,
+            TEST_IMG("bar blk 0 at 4")
+        );
+
+        assert_eq!(
+            newtline.get_page_at_lsn(TESTREL_B, 0, Lsn(4))?,
+            TEST_IMG("foobar blk 0 at 2")
+        );
+
+        assert_eq!(newtline.get_relish_size(TESTREL_B, Lsn(4))?.unwrap(), 1);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_history_rocksdb() -> Result<()> {
+        let repo = get_test_repo("test_history_rocksdb", RepositoryFormat::RocksDb)?;
+        test_history(&*repo)
+    }
+    #[test]
+    // TODO: This doesn't work with the layered storage, the functions needed for push/pull
+    // functionality haven't been implemented yet.
+    #[ignore]
+    fn test_history_layered() -> Result<()> {
+        let repo = get_test_repo("test_history_layered", RepositoryFormat::Layered)?;
+        test_history(&*repo)
+    }
+    fn test_history(repo: &dyn Repository) -> Result<()> {
+        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
+        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
+
+        let snapshot = tline.history()?;
+        assert_eq!(snapshot.lsn(), Lsn(0));
+        let mut snapshot = skip_nonrel_objects(snapshot)?;
+        assert_eq!(None, snapshot.next().transpose()?);
+
+        // add a page and advance the last valid LSN
+        let rel = TESTREL_A;
+        tline.put_page_image(rel, 1, Lsn(1), TEST_IMG("blk 1 @ lsn 1"), true)?;
+        tline.advance_last_valid_lsn(Lsn(1));
+
+        let expected_page = Modification {
+            tag: ObjectTag::Buffer(rel, 1),
+            lsn: Lsn(1),
+            data: ObjectValue::ser(&ObjectValue::Page(PageEntry::Page(TEST_IMG(
+                "blk 1 @ lsn 1",
+            ))))?,
+        };
+        let expected_init_size = Modification {
+            tag: ObjectTag::RelationMetadata(rel),
+            lsn: Lsn(1),
+            data: ObjectValue::ser(&ObjectValue::RelationSize(RelationSizeEntry::Size(2)))?,
+        };
+        let expected_trunc_size = Modification {
+            tag: ObjectTag::RelationMetadata(rel),
+            lsn: Lsn(2),
+            data: ObjectValue::ser(&ObjectValue::RelationSize(RelationSizeEntry::Size(0)))?,
+        };
+
+        let snapshot = tline.history()?;
+        assert_eq!(snapshot.lsn(), Lsn(1));
+        let mut snapshot = skip_nonrel_objects(snapshot)?;
+        assert_eq!(
+            Some(&expected_init_size),
+            snapshot.next().transpose()?.as_ref()
+        );
+        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
+        assert_eq!(None, snapshot.next().transpose()?);
+
+        // truncate to zero, but don't advance the last valid LSN
+        tline.put_truncation(rel, Lsn(2), 0)?;
+        let snapshot = tline.history()?;
+        assert_eq!(snapshot.lsn(), Lsn(1));
+        let mut snapshot = skip_nonrel_objects(snapshot)?;
+        assert_eq!(
+            Some(&expected_init_size),
+            snapshot.next().transpose()?.as_ref()
+        );
+        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
+        assert_eq!(None, snapshot.next().transpose()?);
+
+        // advance the last valid LSN and the truncation should be observable
+        tline.advance_last_valid_lsn(Lsn(2));
+        let snapshot = tline.history()?;
+        assert_eq!(snapshot.lsn(), Lsn(2));
+        let mut snapshot = skip_nonrel_objects(snapshot)?;
+        assert_eq!(
+            Some(&expected_init_size),
+            snapshot.next().transpose()?.as_ref()
+        );
+        assert_eq!(
+            Some(&expected_trunc_size),
+            snapshot.next().transpose()?.as_ref()
+        );
+        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
+        assert_eq!(None, snapshot.next().transpose()?);
+
         Ok(())
     }
 
@@ -508,15 +711,16 @@ mod tests {
     impl WalRedoManager for TestRedoManager {
         fn request_redo(
             &self,
-            tag: BufferTag,
+            rel: RelishTag,
+            blknum: u32,
             lsn: Lsn,
             base_img: Option<Bytes>,
             records: Vec<WALRecord>,
         ) -> Result<Bytes, WalRedoError> {
             let s = format!(
-                "redo for rel {} blk {} to get to {}, with {} and {} records",
-                tag.rel,
-                tag.blknum,
+                "redo for {} blk {} to get to {}, with {} and {} records",
+                rel,
+                blknum,
                 lsn,
                 if base_img.is_some() {
                     "base image"

pageserver/src/repository/rocksdb.rs

@@ -1,978 +0,0 @@
-//
-// A Repository holds all the different page versions and WAL records
-//
-// This implementation uses RocksDB to store WAL wal records and
-// full page images, keyed by the RelFileNode, blocknumber, and the
-// LSN.
-
-use crate::repository::{BufferTag, RelTag, Repository, Timeline, WALRecord};
-use crate::restore_local_repo::restore_timeline;
-use crate::waldecoder::{Oid, TransactionId};
-use crate::walredo::WalRedoManager;
-use crate::PageServerConf;
-use crate::ZTimelineId;
-// use crate::PageServerConf;
-// use crate::branches;
-use anyhow::{bail, Context, Result};
-use bytes::{Buf, BufMut, Bytes, BytesMut};
-use log::*;
-use postgres_ffi::nonrelfile_utils::transaction_id_get_status;
-use postgres_ffi::*;
-use std::cmp::min;
-use std::collections::HashMap;
-use std::convert::TryInto;
-use std::sync::atomic::AtomicU64;
-use std::sync::atomic::Ordering;
-use std::sync::{Arc, Mutex};
-use std::thread;
-use std::time::{Duration, Instant};
-use zenith_utils::lsn::{AtomicLsn, Lsn};
-use zenith_utils::seqwait::SeqWait;
-
-// Timeout when waiting or WAL receiver to catch up to an LSN given in a GetPage@LSN call.
-static TIMEOUT: Duration = Duration::from_secs(60);
-
-pub struct RocksRepository {
-    conf: &'static PageServerConf,
-    timelines: Mutex<HashMap<ZTimelineId, Arc<RocksTimeline>>>,
-
-    walredo_mgr: Arc<dyn WalRedoManager>,
-}
-
-pub struct RocksTimeline {
-    // RocksDB handle
-    db: rocksdb::DB,
-
-    // WAL redo manager
-    walredo_mgr: Arc<dyn WalRedoManager>,
-
-    // What page versions do we hold in the cache? If we get a request > last_valid_lsn,
-    // we need to wait until we receive all the WAL up to the request. The SeqWait
-    // provides functions for that. TODO: If we get a request for an old LSN, such that
-    // the versions have already been garbage collected away, we should throw an error,
-    // but we don't track that currently.
-    //
-    // last_record_lsn points to the end of last processed WAL record.
-    // It can lag behind last_valid_lsn, if the WAL receiver has received some WAL
-    // after the end of last record, but not the whole next record yet. In the
-    // page cache, we care about last_valid_lsn, but if the WAL receiver needs to
-    // restart the streaming, it needs to restart at the end of last record, so
-    // we track them separately. last_record_lsn should perhaps be in
-    // walreceiver.rs instead of here, but it seems convenient to keep all three
-    // values together.
-    //
-    last_valid_lsn: SeqWait<Lsn>,
-    last_record_lsn: AtomicLsn,
-
-    // Counters, for metrics collection.
-    pub num_entries: AtomicU64,
-    pub num_page_images: AtomicU64,
-    pub num_wal_records: AtomicU64,
-    pub num_getpage_requests: AtomicU64,
-}
-
-//
-// We store two kinds of entries in the repository:
-//
-// 1. Ready-made images of the block
-// 2. WAL records, to be applied on top of the "previous" entry
-//
-// Some WAL records will initialize the page from scratch. For such records,
-// the 'will_init' flag is set. They don't need the previous page image before
-// applying. The 'will_init' flag is set for records containing a full-page image,
-// and for records with the BKPBLOCK_WILL_INIT flag. These differ from PageImages
-// stored directly in the cache entry in that you still need to run the WAL redo
-// routine to generate the page image.
-//
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
-struct CacheKey {
-    pub tag: BufferTag,
-    pub lsn: Lsn,
-}
-
-impl CacheKey {
-    fn pack(&self, buf: &mut BytesMut) {
-        self.tag.pack(buf);
-        buf.put_u64(self.lsn.0);
-    }
-    fn unpack(buf: &mut Bytes) -> CacheKey {
-        CacheKey {
-            tag: BufferTag::unpack(buf),
-            lsn: Lsn::from(buf.get_u64()),
-        }
-    }
-
-    fn from_slice(slice: &[u8]) -> Self {
-        let mut buf = Bytes::copy_from_slice(slice);
-        Self::unpack(&mut buf)
-    }
-
-    fn to_bytes(&self) -> BytesMut {
-        let mut buf = BytesMut::new();
-        self.pack(&mut buf);
-        buf
-    }
-}
-
-enum CacheEntryContent {
-    PageImage(Bytes),
-    WALRecord(WALRecord),
-    Truncation,
-}
-
-// The serialized representation of a CacheEntryContent begins with
-// single byte that indicates what kind of entry it is. There is also
-// an UNUSED_VERSION_FLAG that is not represented in the CacheEntryContent
-// at all, you must peek into the first byte of the serialized representation
-// to read it.
-const CONTENT_PAGE_IMAGE: u8 = 1u8;
-const CONTENT_WAL_RECORD: u8 = 2u8;
-const CONTENT_TRUNCATION: u8 = 3u8;
-
-const CONTENT_KIND_MASK: u8 = 3u8; // bitmask that covers the above
-
-const UNUSED_VERSION_FLAG: u8 = 4u8;
-
-impl CacheEntryContent {
-    pub fn pack(&self, buf: &mut BytesMut) {
-        match self {
-            CacheEntryContent::PageImage(image) => {
-                buf.put_u8(CONTENT_PAGE_IMAGE);
-                buf.put_u16(image.len() as u16);
-                buf.put_slice(&image[..]);
-            }
-            CacheEntryContent::WALRecord(rec) => {
-                buf.put_u8(CONTENT_WAL_RECORD);
-                rec.pack(buf);
-            }
-            CacheEntryContent::Truncation => {
-                buf.put_u8(CONTENT_TRUNCATION);
-            }
-        }
-    }
-    pub fn unpack(buf: &mut Bytes) -> CacheEntryContent {
-        let kind = buf.get_u8() & CONTENT_KIND_MASK;
-
-        match kind {
-            CONTENT_PAGE_IMAGE => {
-                let len = buf.get_u16() as usize;
-                let mut dst = vec![0u8; len];
-                buf.copy_to_slice(&mut dst);
-                CacheEntryContent::PageImage(Bytes::from(dst))
-            }
-            CONTENT_WAL_RECORD => CacheEntryContent::WALRecord(WALRecord::unpack(buf)),
-            CONTENT_TRUNCATION => CacheEntryContent::Truncation,
-            _ => unreachable!(),
-        }
-    }
-
-    fn from_slice(slice: &[u8]) -> Self {
-        let mut buf = Bytes::copy_from_slice(slice);
-        Self::unpack(&mut buf)
-    }
-
-    fn to_bytes(&self) -> BytesMut {
-        let mut buf = BytesMut::new();
-        self.pack(&mut buf);
-        buf
-    }
-}
-
-impl RocksRepository {
-    pub fn new(
-        conf: &'static PageServerConf,
-        walredo_mgr: Arc<dyn WalRedoManager>,
-    ) -> RocksRepository {
-        RocksRepository {
-            conf: conf,
-            timelines: Mutex::new(HashMap::new()),
-            walredo_mgr,
-        }
-    }
-}
-
-// Get handle to a given timeline. It is assumed to already exist.
-impl Repository for RocksRepository {
-    fn get_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>> {
-        let timelines = self.timelines.lock().unwrap();
-
-        match timelines.get(&timelineid) {
-            Some(timeline) => Ok(timeline.clone()),
-            None => bail!("timeline not found"),
-        }
-    }
-
-    fn get_or_restore_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>> {
-        let mut timelines = self.timelines.lock().unwrap();
-
-        match timelines.get(&timelineid) {
-            Some(timeline) => Ok(timeline.clone()),
-            None => {
-                let timeline = RocksTimeline::new(self.conf, timelineid, self.walredo_mgr.clone());
-
-                restore_timeline(self.conf, &timeline, timelineid)?;
-
-                let timeline_rc = Arc::new(timeline);
-
-                timelines.insert(timelineid, timeline_rc.clone());
-
-                if self.conf.gc_horizon != 0 {
-                    let timeline_rc_copy = timeline_rc.clone();
-                    let conf = self.conf;
-                    let _gc_thread = thread::Builder::new()
-                        .name("Garbage collection thread".into())
-                        .spawn(move || {
-                            // FIXME
-                            timeline_rc_copy.do_gc(conf).expect("GC thread died");
-                        })
-                        .unwrap();
-                }
-                Ok(timeline_rc)
-            }
-        }
-    }
-
-    #[cfg(test)]
-    fn create_empty_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>> {
-        let mut timelines = self.timelines.lock().unwrap();
-
-        let timeline = RocksTimeline::new(&self.conf, timelineid, self.walredo_mgr.clone());
-
-        let timeline_rc = Arc::new(timeline);
-        let r = timelines.insert(timelineid, timeline_rc.clone());
-        assert!(r.is_none());
-
-        // don't start the garbage collector for unit tests, either.
-
-        Ok(timeline_rc)
-    }
-}
-
-impl RocksTimeline {
-    fn open_rocksdb(conf: &PageServerConf, timelineid: ZTimelineId) -> rocksdb::DB {
-        let path = conf.timeline_path(timelineid);
-        let mut opts = rocksdb::Options::default();
-        opts.create_if_missing(true);
-        opts.set_use_fsync(true);
-        opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
-        opts.set_compaction_filter("ttl", move |_level: u32, _key: &[u8], val: &[u8]| {
-            if (val[0] & UNUSED_VERSION_FLAG) != 0 {
-                rocksdb::compaction_filter::Decision::Remove
-            } else {
-                rocksdb::compaction_filter::Decision::Keep
-            }
-        });
-        rocksdb::DB::open(&opts, &path).unwrap()
-    }
-
-    fn new(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        walredo_mgr: Arc<dyn WalRedoManager>,
-    ) -> RocksTimeline {
-        RocksTimeline {
-            db: RocksTimeline::open_rocksdb(conf, timelineid),
-
-            walredo_mgr,
-
-            last_valid_lsn: SeqWait::new(Lsn(0)),
-            last_record_lsn: AtomicLsn::new(0),
-
-            num_entries: AtomicU64::new(0),
-            num_page_images: AtomicU64::new(0),
-            num_wal_records: AtomicU64::new(0),
-            num_getpage_requests: AtomicU64::new(0),
-        }
-    }
-}
-
-impl RocksTimeline {
-    ///
-    /// Collect all the WAL records that are needed to reconstruct a page
-    /// image for the given cache entry.
-    ///
-    /// Returns an old page image (if any), and a vector of WAL records to apply
-    /// over it.
-    ///
-    fn collect_records_for_apply(
-        &self,
-        tag: BufferTag,
-        lsn: Lsn,
-    ) -> (Option<Bytes>, Vec<WALRecord>) {
-        let key = CacheKey { tag, lsn };
-        let mut base_img: Option<Bytes> = None;
-        let mut records: Vec<WALRecord> = Vec::new();
-
-        let mut iter = self.db.raw_iterator();
-        iter.seek_for_prev(key.to_bytes());
-
-        // Scan backwards, collecting the WAL records, until we hit an
-        // old page image.
-        while iter.valid() {
-            let key = CacheKey::from_slice(iter.key().unwrap());
-            if key.tag != tag {
-                break;
-            }
-            let content = CacheEntryContent::from_slice(iter.value().unwrap());
-            if let CacheEntryContent::PageImage(img) = content {
-                // We have a base image. No need to dig deeper into the list of
-                // records
-                base_img = Some(img);
-                break;
-            } else if let CacheEntryContent::WALRecord(rec) = content {
-                records.push(rec.clone());
-                // If this WAL record initializes the page, no need to dig deeper.
-                if rec.will_init {
-                    break;
-                }
-            } else {
-                panic!("no base image and no WAL record on cache entry");
-            }
-            iter.prev();
-        }
-        records.reverse();
-        (base_img, records)
-    }
-
-    // Internal functions
-
-    //
-    // Internal function to get relation size at given LSN.
-    //
-    // The caller must ensure that WAL has been received up to 'lsn'.
-    //
-    fn relsize_get_nowait(&self, rel: RelTag, lsn: Lsn) -> Result<u32> {
-        assert!(lsn <= self.last_valid_lsn.load());
-
-        let mut key = CacheKey {
-            tag: BufferTag {
-                rel,
-                blknum: u32::MAX,
-            },
-            lsn,
-        };
-        let mut iter = self.db.raw_iterator();
-        loop {
-            iter.seek_for_prev(key.to_bytes());
-            if iter.valid() {
-                let thiskey = CacheKey::from_slice(iter.key().unwrap());
-                if thiskey.tag.rel == rel {
-                    let content = CacheEntryContent::from_slice(iter.value().unwrap());
-                    if let CacheEntryContent::Truncation = content {
-                        if thiskey.tag.blknum > 0 {
-                            key.tag.blknum = thiskey.tag.blknum - 1;
-                            continue;
-                        }
-                        break;
-                    }
-                    let relsize = thiskey.tag.blknum + 1;
-                    debug!("Size of relation {} at {} is {}", rel, lsn, relsize);
-                    return Ok(relsize);
-                }
-            }
-            break;
-        }
-        debug!("Size of relation {} at {} is zero", rel, lsn);
-        Ok(0)
-    }
-
-    fn do_gc(&self, conf: &'static PageServerConf) -> Result<Bytes> {
-        loop {
-            thread::sleep(conf.gc_period);
-            let last_lsn = self.get_last_valid_lsn();
-
-            // checked_sub() returns None on overflow.
-            if let Some(horizon) = last_lsn.checked_sub(conf.gc_horizon) {
-                let mut maxkey = CacheKey {
-                    tag: BufferTag {
-                        rel: RelTag {
-                            spcnode: u32::MAX,
-                            dbnode: u32::MAX,
-                            relnode: u32::MAX,
-                            forknum: u8::MAX,
-                        },
-                        blknum: u32::MAX,
-                    },
-                    lsn: Lsn::MAX,
-                };
-                let now = Instant::now();
-                let mut reconstructed = 0u64;
-                let mut truncated = 0u64;
-                let mut inspected = 0u64;
-                let mut deleted = 0u64;
-                loop {
-                    let mut iter = self.db.raw_iterator();
-                    iter.seek_for_prev(maxkey.to_bytes());
-                    if iter.valid() {
-                        let key = CacheKey::from_slice(iter.key().unwrap());
-                        let v = iter.value().unwrap();
-
-                        inspected += 1;
-
-                        // Construct boundaries for old records cleanup
-                        maxkey.tag = key.tag;
-                        let last_lsn = key.lsn;
-                        maxkey.lsn = min(horizon, last_lsn); // do not remove last version
-
-                        let mut minkey = maxkey.clone();
-                        minkey.lsn = Lsn(0); // first version
-
-                        // Special handling of delete of PREPARE WAL record
-                        if last_lsn < horizon
-                            && key.tag.rel.forknum == pg_constants::PG_TWOPHASE_FORKNUM
-                        {
-                            if (v[0] & UNUSED_VERSION_FLAG) == 0 {
-                                let mut v = v.to_owned();
-                                v[0] |= UNUSED_VERSION_FLAG;
-                                self.db.put(key.to_bytes(), &v[..])?;
-                                deleted += 1;
-                            }
-                            maxkey = minkey;
-                            continue;
-                        }
-                        // reconstruct most recent page version
-                        if (v[0] & CONTENT_KIND_MASK) == CONTENT_WAL_RECORD {
-                            // force reconstruction of most recent page version
-                            let (base_img, records) =
-                                self.collect_records_for_apply(key.tag, key.lsn);
-
-                            trace!(
-                                "Reconstruct most recent page {} blk {} at {} from {} records",
-                                key.tag.rel,
-                                key.tag.blknum,
-                                key.lsn,
-                                records.len()
-                            );
-
-                            let new_img = self
-                                .walredo_mgr
-                                .request_redo(key.tag, key.lsn, base_img, records)?;
-                            self.put_page_image(key.tag, key.lsn, new_img.clone());
-
-                            reconstructed += 1;
-                        }
-
-                        iter.seek_for_prev(maxkey.to_bytes());
-                        if iter.valid() {
-                            // do not remove last version
-                            if last_lsn > horizon {
-                                // locate most recent record before horizon
-                                let key = CacheKey::from_slice(iter.key().unwrap());
-                                if key.tag == maxkey.tag {
-                                    let v = iter.value().unwrap();
-                                    if (v[0] & CONTENT_KIND_MASK) == CONTENT_WAL_RECORD {
-                                        let (base_img, records) =
-                                            self.collect_records_for_apply(key.tag, key.lsn);
-                                        trace!("Reconstruct horizon page {} blk {} at {} from {} records",
-                                              key.tag.rel, key.tag.blknum, key.lsn, records.len());
-                                        let new_img = self
-                                            .walredo_mgr
-                                            .request_redo(key.tag, key.lsn, base_img, records)?;
-                                        self.put_page_image(key.tag, key.lsn, new_img.clone());
-
-                                        truncated += 1;
-                                    } else {
-                                        trace!(
-                                            "Keeping horizon page {} blk {} at {}",
-                                            key.tag.rel,
-                                            key.tag.blknum,
-                                            key.lsn
-                                        );
-                                    }
-                                }
-                            } else {
-                                trace!(
-                                    "Last page {} blk {} at {}, horizon {}",
-                                    key.tag.rel,
-                                    key.tag.blknum,
-                                    key.lsn,
-                                    horizon
-                                );
-                            }
-                            // remove records prior to horizon
-                            loop {
-                                iter.prev();
-                                if !iter.valid() {
-                                    break;
-                                }
-                                let key = CacheKey::from_slice(iter.key().unwrap());
-                                if key.tag != maxkey.tag {
-                                    break;
-                                }
-                                let v = iter.value().unwrap();
-                                if (v[0] & UNUSED_VERSION_FLAG) == 0 {
-                                    let mut v = v.to_owned();
-                                    v[0] |= UNUSED_VERSION_FLAG;
-                                    self.db.put(key.to_bytes(), &v[..])?;
-                                    deleted += 1;
-                                    trace!(
-                                        "deleted: {} blk {} at {}",
-                                        key.tag.rel,
-                                        key.tag.blknum,
-                                        key.lsn
-                                    );
-                                } else {
-                                    break;
-                                }
-                            }
-                        }
-                        maxkey = minkey;
-                    } else {
-                        break;
-                    }
-                }
-                info!("Garbage collection completed in {:?}:\n{} version chains inspected, {} pages reconstructed, {} version histories truncated, {} versions deleted",
-					  now.elapsed(), inspected, reconstructed, truncated, deleted);
-            }
-        }
-    }
-
-    //
-    // Wait until WAL has been received up to the given LSN.
-    //
-    fn wait_lsn(&self, mut lsn: Lsn) -> Result<Lsn> {
-        // When invalid LSN is requested, it means "don't wait, return latest version of the page"
-        // This is necessary for bootstrap.
-        if lsn == Lsn(0) {
-            let last_valid_lsn = self.last_valid_lsn.load();
-            trace!(
-                "walreceiver doesn't work yet last_valid_lsn {}, requested {}",
-                last_valid_lsn,
-                lsn
-            );
-            lsn = last_valid_lsn;
-        }
-        //trace!("Start waiting for LSN {}, valid LSN is {}", lsn,  self.last_valid_lsn.load());
-        self.last_valid_lsn
-            .wait_for_timeout(lsn, TIMEOUT)
-            .with_context(|| {
-                format!(
-                    "Timed out while waiting for WAL record at LSN {} to arrive",
-                    lsn
-                )
-            })?;
-        //trace!("Stop waiting for LSN {}, valid LSN is {}", lsn,  self.last_valid_lsn.load());
-
-        Ok(lsn)
-    }
-}
-
-impl Timeline for RocksTimeline {
-    // Public GET interface functions
-
-    ///
-    /// GetPage@LSN
-    ///
-    /// Returns an 8k page image
-    ///
-    fn get_page_at_lsn(&self, tag: BufferTag, req_lsn: Lsn) -> Result<Bytes> {
-        self.num_getpage_requests.fetch_add(1, Ordering::Relaxed);
-
-        let lsn = self.wait_lsn(req_lsn)?;
-
-        // Look up cache entry. If it's a page image, return that. If it's a WAL record,
-        // ask the WAL redo service to reconstruct the page image from the WAL records.
-        let key = CacheKey { tag, lsn };
-
-        let mut iter = self.db.raw_iterator();
-        iter.seek_for_prev(key.to_bytes());
-
-        if iter.valid() {
-            let key = CacheKey::from_slice(iter.key().unwrap());
-            if key.tag == tag {
-                let content = CacheEntryContent::from_slice(iter.value().unwrap());
-                let page_img: Bytes;
-                if let CacheEntryContent::PageImage(img) = content {
-                    page_img = img;
-                } else if let CacheEntryContent::WALRecord(_rec) = content {
-                    // Request the WAL redo manager to apply the WAL records for us.
-                    let (base_img, records) = self.collect_records_for_apply(tag, lsn);
-                    page_img = self.walredo_mgr.request_redo(tag, lsn, base_img, records)?;
-
-                    self.put_page_image(tag, lsn, page_img.clone());
-                } else {
-                    // No base image, and no WAL record. Huh?
-                    bail!("no page image or WAL record for requested page");
-                }
-                // FIXME: assumes little-endian. Only used for the debugging log though
-                let page_lsn_hi =
-                    u32::from_le_bytes(page_img.get(0..4).unwrap().try_into().unwrap());
-                let page_lsn_lo =
-                    u32::from_le_bytes(page_img.get(4..8).unwrap().try_into().unwrap());
-                debug!(
-                    "Returning page with LSN {:X}/{:X} for {} blk {}",
-                    page_lsn_hi, page_lsn_lo, tag.rel, tag.blknum
-                );
-                return Ok(page_img);
-            }
-        }
-        static ZERO_PAGE: [u8; 8192] = [0u8; 8192];
-        debug!(
-            "Page {} blk {} at {}({}) not found",
-            tag.rel, tag.blknum, req_lsn, lsn
-        );
-        Ok(Bytes::from_static(&ZERO_PAGE))
-        /* return Err("could not find page image")?; */
-    }
-
-    ///
-    /// Get size of relation at given LSN.
-    ///
-    fn get_relsize(&self, rel: RelTag, lsn: Lsn) -> Result<u32> {
-        let lsn = self.wait_lsn(lsn)?;
-        self.relsize_get_nowait(rel, lsn)
-    }
-
-    /// Get vector of prepared twophase transactions
-    fn get_twophase(&self, lsn: Lsn) -> Result<Vec<TransactionId>> {
-        let key = CacheKey {
-            // minimal key
-            tag: BufferTag {
-                rel: RelTag {
-                    forknum: pg_constants::PG_TWOPHASE_FORKNUM,
-                    spcnode: 0,
-                    dbnode: 0,
-                    relnode: 0,
-                },
-                blknum: 0,
-            },
-            lsn: Lsn(0),
-        };
-        let mut gxacts = Vec::new();
-
-        let mut iter = self.db.raw_iterator();
-        iter.seek(key.to_bytes());
-        while iter.valid() {
-            let key = CacheKey::from_slice(iter.key().unwrap());
-            if key.tag.rel.forknum != pg_constants::PG_TWOPHASE_FORKNUM {
-                break; // we are done with this fork
-            }
-            if key.lsn <= lsn {
-                let xid = key.tag.blknum;
-                let tag = BufferTag {
-                    rel: RelTag {
-                        forknum: pg_constants::PG_XACT_FORKNUM,
-                        spcnode: 0,
-                        dbnode: 0,
-                        relnode: 0,
-                    },
-                    blknum: xid / pg_constants::CLOG_XACTS_PER_PAGE,
-                };
-                let clog_page = self.get_page_at_lsn(tag, lsn)?;
-                let status = transaction_id_get_status(xid, &clog_page[..]);
-                if status == pg_constants::TRANSACTION_STATUS_IN_PROGRESS {
-                    gxacts.push(xid);
-                }
-            }
-            iter.next();
-        }
-        return Ok(gxacts);
-    }
-
-    /// Get databases. This function is used to local pg_filenode.map files
-    fn get_databases(&self, lsn: Lsn) -> Result<Vec<RelTag>> {
-        let key = CacheKey {
-            // minimal key
-            tag: BufferTag {
-                rel: RelTag {
-                    forknum: pg_constants::PG_FILENODEMAP_FORKNUM,
-                    spcnode: 0,
-                    dbnode: 0,
-                    relnode: 0,
-                },
-                blknum: 0,
-            },
-            lsn: Lsn(0),
-        };
-        let mut dbs = Vec::new();
-
-        let mut iter = self.db.raw_iterator();
-        iter.seek(key.to_bytes());
-        let mut prev_tag = key.tag.rel;
-        while iter.valid() {
-            let key = CacheKey::from_slice(iter.key().unwrap());
-            if key.tag.rel.forknum != pg_constants::PG_FILENODEMAP_FORKNUM {
-                break; // we are done with this fork
-            }
-            if key.tag.rel != prev_tag && key.lsn <= lsn {
-                prev_tag = key.tag.rel;
-                dbs.push(prev_tag); // collect unique tags
-            }
-            iter.next();
-        }
-        return Ok(dbs);
-    }
-
-    /// Get range [begin,end) of stored blocks. Used mostly for SMGR pseudorelations
-    /// but can be also applied to normal relations.
-    fn get_range(&self, rel: RelTag, lsn: Lsn) -> Result<(u32, u32)> {
-        let _lsn = self.wait_lsn(lsn)?;
-        let mut key = CacheKey {
-            // minimal key to start with
-            tag: BufferTag { rel, blknum: 0 },
-            lsn: Lsn(0),
-        };
-        let mut iter = self.db.raw_iterator();
-        iter.seek(key.to_bytes()); // locate first entry
-        if iter.valid() {
-            let thiskey = CacheKey::from_slice(iter.key().unwrap());
-            let tag = thiskey.tag;
-            if tag.rel == rel {
-                // still trversing this relation
-                let first_blknum = tag.blknum;
-                key.tag.blknum = u32::MAX; // maximal key
-                let mut iter = self.db.raw_iterator();
-                iter.seek_for_prev(key.to_bytes()); // localte last entry
-                if iter.valid() {
-                    let thiskey = CacheKey::from_slice(iter.key().unwrap());
-                    let last_blknum = thiskey.tag.blknum;
-                    return Ok((first_blknum, last_blknum + 1)); // upper boundary is exclusive
-                }
-            }
-        }
-        Ok((0, 0)) // empty range
-    }
-
-    ///
-    /// Does relation exist at given LSN?
-    ///
-    /// FIXME: this actually returns true, if the relation exists at *any* LSN
-    fn get_relsize_exists(&self, rel: RelTag, req_lsn: Lsn) -> Result<bool> {
-        let lsn = self.wait_lsn(req_lsn)?;
-
-        let key = CacheKey {
-            tag: BufferTag {
-                rel,
-                blknum: u32::MAX,
-            },
-            lsn,
-        };
-        let mut iter = self.db.raw_iterator();
-        iter.seek_for_prev(key.to_bytes());
-        if iter.valid() {
-            let key = CacheKey::from_slice(iter.key().unwrap());
-            if key.tag.rel == rel {
-                debug!("Relation {} exists at {}", rel, lsn);
-                return Ok(true);
-            }
-        }
-        debug!("Relation {} doesn't exist at {}", rel, lsn);
-        Ok(false)
-    }
-
-    // Other public functions, for updating the repository.
-    // These are used by the WAL receiver and WAL redo.
-
-    ///
-    /// Adds a WAL record to the repository
-    ///
-    fn put_wal_record(&self, tag: BufferTag, rec: WALRecord) {
-        let lsn = rec.lsn;
-        let key = CacheKey { tag, lsn };
-
-        let content = CacheEntryContent::WALRecord(rec);
-
-        let _res = self.db.put(key.to_bytes(), content.to_bytes());
-        trace!(
-            "put_wal_record rel {} blk {} at {}",
-            tag.rel,
-            tag.blknum,
-            lsn
-        );
-
-        self.num_entries.fetch_add(1, Ordering::Relaxed);
-        self.num_wal_records.fetch_add(1, Ordering::Relaxed);
-    }
-
-    ///
-    /// Adds a relation-wide WAL record (like truncate) to the repository,
-    /// associating it with all pages started with specified block number
-    ///
-    fn put_truncation(&self, rel: RelTag, lsn: Lsn, nblocks: u32) -> Result<()> {
-        // What was the size of the relation before this record?
-        let last_lsn = self.last_valid_lsn.load();
-        let old_rel_size = self.relsize_get_nowait(rel, last_lsn)?;
-
-        let content = CacheEntryContent::Truncation;
-        // set new relation size
-        trace!("Truncate relation {} to {} blocks at {}", rel, nblocks, lsn);
-
-        for blknum in nblocks..old_rel_size {
-            let key = CacheKey {
-                tag: BufferTag { rel, blknum },
-                lsn,
-            };
-            trace!("put_wal_record lsn: {}", key.lsn);
-            let _res = self.db.put(key.to_bytes(), content.to_bytes());
-        }
-        let n = (old_rel_size - nblocks) as u64;
-        self.num_entries.fetch_add(n, Ordering::Relaxed);
-        self.num_wal_records.fetch_add(n, Ordering::Relaxed);
-        Ok(())
-    }
-
-    ///
-    /// Get page image at particular LSN
-    ///
-    fn get_page_image(&self, tag: BufferTag, lsn: Lsn) -> Result<Option<Bytes>> {
-        let key = CacheKey { tag, lsn };
-        if let Some(bytes) = self.db.get(key.to_bytes())? {
-            let content = CacheEntryContent::from_slice(&bytes);
-            if let CacheEntryContent::PageImage(img) = content {
-                return Ok(Some(img));
-            }
-        }
-        return Ok(None);
-    }
-
-    ///
-    /// Memorize a full image of a page version
-    ///
-    fn put_page_image(&self, tag: BufferTag, lsn: Lsn, img: Bytes) {
-        let img_len = img.len();
-        let key = CacheKey { tag, lsn };
-        let content = CacheEntryContent::PageImage(img);
-
-        let mut val_buf = content.to_bytes();
-
-        // Zero size of page image indicates that page can be removed
-        if img_len == 0 {
-            if (val_buf[0] & UNUSED_VERSION_FLAG) != 0 {
-                // records already marked for deletion
-                return;
-            } else {
-                // delete truncated multixact page
-                val_buf[0] |= UNUSED_VERSION_FLAG;
-            }
-        }
-
-        trace!("put_wal_record lsn: {}", key.lsn);
-        let _res = self.db.put(key.to_bytes(), content.to_bytes());
-
-        trace!(
-            "put_page_image rel {} blk {} at {}",
-            tag.rel,
-            tag.blknum,
-            lsn
-        );
-        self.num_page_images.fetch_add(1, Ordering::Relaxed);
-    }
-
-    fn put_create_database(
-        &self,
-        lsn: Lsn,
-        db_id: Oid,
-        tablespace_id: Oid,
-        src_db_id: Oid,
-        src_tablespace_id: Oid,
-    ) -> Result<()> {
-        let mut n = 0;
-        for forknum in &[
-            pg_constants::MAIN_FORKNUM,
-            pg_constants::FSM_FORKNUM,
-            pg_constants::VISIBILITYMAP_FORKNUM,
-            pg_constants::INIT_FORKNUM,
-            pg_constants::PG_FILENODEMAP_FORKNUM,
-        ] {
-            let key = CacheKey {
-                tag: BufferTag {
-                    rel: RelTag {
-                        spcnode: src_tablespace_id,
-                        dbnode: src_db_id,
-                        relnode: 0,
-                        forknum: *forknum,
-                    },
-                    blknum: 0,
-                },
-                lsn: Lsn(0),
-            };
-            let mut iter = self.db.raw_iterator();
-            iter.seek(key.to_bytes());
-            while iter.valid() {
-                let mut key = CacheKey::from_slice(iter.key().unwrap());
-                if key.tag.rel.spcnode != src_tablespace_id || key.tag.rel.dbnode != src_db_id {
-                    break;
-                }
-                key.tag.rel.spcnode = tablespace_id;
-                key.tag.rel.dbnode = db_id;
-                key.lsn = lsn;
-
-                let v = iter.value().unwrap();
-                self.db.put(key.to_bytes(), v)?;
-                n += 1;
-                iter.next();
-            }
-        }
-        info!(
-            "Create database {}/{}, copy {} entries",
-            tablespace_id, db_id, n
-        );
-        Ok(())
-    }
-
-    /// Remember that WAL has been received and added to the timeline up to the given LSN
-    fn advance_last_valid_lsn(&self, lsn: Lsn) {
-        let lsn = Lsn((lsn.0 + 7) & !7); // align position on 8 bytes
-        let old = self.last_valid_lsn.advance(lsn);
-
-        // Can't move backwards.
-        if lsn < old {
-            warn!(
-                "attempted to move last valid LSN backwards (was {}, new {})",
-                old, lsn
-            );
-        }
-    }
-
-    ///
-    /// Remember the (end of) last valid WAL record remembered for the timeline.
-    ///
-    /// NOTE: this updates last_valid_lsn as well.
-    ///
-    fn advance_last_record_lsn(&self, lsn: Lsn) {
-        let lsn = Lsn((lsn.0 + 7) & !7); // align position on 8 bytes
-                                         // Can't move backwards.
-        let old = self.last_record_lsn.fetch_max(lsn);
-        assert!(old <= lsn);
-
-        // Also advance last_valid_lsn
-        let old = self.last_valid_lsn.advance(lsn);
-        // Can't move backwards.
-        if lsn < old {
-            warn!(
-                "attempted to move last record LSN backwards (was {}, new {})",
-                old, lsn
-            );
-        }
-    }
-
-    fn get_last_record_lsn(&self) -> Lsn {
-        self.last_record_lsn.load()
-    }
-
-    fn init_valid_lsn(&self, lsn: Lsn) {
-        let old = self.last_valid_lsn.advance(lsn);
-        assert!(old == Lsn(0));
-        let old = self.last_record_lsn.fetch_max(lsn);
-        assert!(old == Lsn(0));
-    }
-
-    fn get_last_valid_lsn(&self) -> Lsn {
-        self.last_valid_lsn.load()
-    }
-
-    //
-    // Get statistics to be displayed in the user interface.
-    //
-    // FIXME
-    /*
-    fn get_stats(&self) -> TimelineStats {
-        TimelineStats {
-            num_entries: self.num_entries.load(Ordering::Relaxed),
-            num_page_images: self.num_page_images.load(Ordering::Relaxed),
-            num_wal_records: self.num_wal_records.load(Ordering::Relaxed),
-            num_getpage_requests: self.num_getpage_requests.load(Ordering::Relaxed),
-        }
-    }
-    */
-}

pageserver/src/restore_s3.rs

@@ -1,274 +0,0 @@
-//
-// Restore chunks from S3
-//
-// This runs once at Page Server startup. It loads all the "base images" from
-// S3 into the in-memory page cache. It also initializes the "last valid LSN"
-// in the page cache to the LSN of the base image, so that when the WAL receiver
-// is started, it starts streaming from that LSN.
-//
-
-use bytes::{Buf, BytesMut};
-use log::*;
-use regex::Regex;
-use std::env;
-use std::fmt;
-
-use s3::bucket::Bucket;
-use s3::creds::Credentials;
-use s3::region::Region;
-use s3::S3Error;
-
-use tokio::runtime;
-
-use futures::future;
-
-use crate::{page_cache, PageServerConf};
-use postgres_ffi::pg_constants;
-use postgres_ffi::relfile_utils::*;
-
-struct Storage {
-    region: Region,
-    credentials: Credentials,
-    bucket: String,
-}
-
-pub fn restore_main(conf: &PageServerConf) {
-    // Create a new thread pool
-    let runtime = runtime::Runtime::new().unwrap();
-
-    runtime.block_on(async {
-        let result = restore_chunk(conf).await;
-
-        match result {
-            Ok(_) => {}
-            Err(err) => {
-                error!("S3 error: {}", err);
-            }
-        }
-    });
-}
-
-//
-// Restores one chunk from S3.
-//
-// 1. Fetch the last base image >= given LSN
-// 2. Fetch all WAL
-//
-// Load it all into the page cache.
-//
-async fn restore_chunk(conf: &PageServerConf) -> Result<(), S3Error> {
-    let backend = Storage {
-        region: Region::Custom {
-            region: env::var("S3_REGION").unwrap(),
-            endpoint: env::var("S3_ENDPOINT").unwrap(),
-        },
-        credentials: Credentials::new(
-            Some(&env::var("S3_ACCESSKEY").unwrap()),
-            Some(&env::var("S3_SECRET").unwrap()),
-            None,
-            None,
-            None,
-        )
-        .unwrap(),
-        bucket: "zenith-testbucket".to_string(),
-    };
-
-    info!("Restoring from S3...");
-
-    // Create Bucket in REGION for BUCKET
-    let bucket = Bucket::new_with_path_style(&backend.bucket, backend.region, backend.credentials)?;
-
-    // List out contents of directory
-    let results: Vec<s3::serde_types::ListBucketResult> = bucket
-        .list("relationdata/".to_string(), Some("".to_string()))
-        .await?;
-
-    // TODO: get that from backup
-    let sys_id: u64 = 42;
-    let mut oldest_lsn = 0;
-    let mut slurp_futures: Vec<_> = Vec::new();
-
-    for result in results {
-        for object in result.contents {
-            // Download every relation file, slurping them into memory
-
-            let key = object.key;
-            let relpath = key.strip_prefix("relationdata/").unwrap();
-
-            let parsed = parse_rel_file_path(&relpath);
-
-            match parsed {
-                Ok(p) => {
-                    if oldest_lsn == 0 || p.lsn < oldest_lsn {
-                        oldest_lsn = p.lsn;
-                    }
-                    let b = bucket.clone();
-                    let f = slurp_base_file(conf, sys_id, b, key.to_string(), p);
-
-                    slurp_futures.push(f);
-                }
-                Err(e) => {
-                    warn!("unrecognized file: {} ({})", relpath, e);
-                }
-            };
-        }
-    }
-
-    if oldest_lsn == 0 {
-        panic!("no base backup found");
-    }
-
-    let pcache = page_cache::get_pagecache(conf, sys_id);
-    pcache.init_valid_lsn(oldest_lsn);
-
-    info!("{} files to restore...", slurp_futures.len());
-
-    future::join_all(slurp_futures).await;
-    info!("restored!");
-
-    Ok(())
-}
-
-#[derive(Debug)]
-struct ParsedBaseImageFileName {
-    pub spcnode: u32,
-    pub dbnode: u32,
-    pub relnode: u32,
-    pub forknum: u8,
-    pub segno: u32,
-
-    pub lsn: u64,
-}
-
-// formats:
-// <oid>
-// <oid>_<fork name>
-// <oid>.<segment number>
-// <oid>_<fork name>.<segment number>
-
-fn parse_filename(fname: &str) -> Result<(u32, u8, u32, u64), FilePathError> {
-    let re = Regex::new(r"^(?P<relnode>\d+)(_(?P<forkname>[a-z]+))?(\.(?P<segno>\d+))?_(?P<lsnhi>[[:xdigit:]]{8})(?P<lsnlo>[[:xdigit:]]{8})$").unwrap();
-
-    let caps = re
-        .captures(fname)
-        .ok_or_else(|| FilePathError::new("invalid relation data file name"))?;
-
-    let relnode_str = caps.name("relnode").unwrap().as_str();
-    let relnode: u32 = relnode_str.parse()?;
-
-    let forkname = caps.name("forkname").map(|f| f.as_str());
-    let forknum = forkname_to_forknum(forkname)?;
-
-    let segno_match = caps.name("segno");
-    let segno = if segno_match.is_none() {
-        0
-    } else {
-        segno_match.unwrap().as_str().parse::<u32>()?
-    };
-
-    let lsn_hi: u64 = caps.name("lsnhi").unwrap().as_str().parse()?;
-    let lsn_lo: u64 = caps.name("lsnlo").unwrap().as_str().parse()?;
-    let lsn = lsn_hi << 32 | lsn_lo;
-
-    Ok((relnode, forknum, segno, lsn))
-}
-
-fn parse_rel_file_path(path: &str) -> Result<ParsedBaseImageFileName, FilePathError> {
-    /*
-     * Relation data files can be in one of the following directories:
-     *
-     * global/
-     *		shared relations
-     *
-     * base/<db oid>/
-     *		regular relations, default tablespace
-     *
-     * pg_tblspc/<tblspc oid>/<tblspc version>/
-     *		within a non-default tablespace (the name of the directory
-     *		depends on version)
-     *
-     * And the relation data files themselves have a filename like:
-     *
-     * <oid>.<segment number>
-     */
-    if let Some(fname) = path.strip_prefix("global/") {
-        let (relnode, forknum, segno, lsn) = parse_filename(fname)?;
-
-        Ok(ParsedBaseImageFileName {
-            spcnode: pg_constants::GLOBALTABLESPACE_OID,
-            dbnode: 0,
-            relnode,
-            forknum,
-            segno,
-            lsn,
-        })
-    } else if let Some(dbpath) = path.strip_prefix("base/") {
-        let mut s = dbpath.split("/");
-        let dbnode_str = s
-            .next()
-            .ok_or_else(|| FilePathError::new("invalid relation data file name"))?;
-        let dbnode: u32 = dbnode_str.parse()?;
-        let fname = s
-            .next()
-            .ok_or_else(|| FilePathError::new("invalid relation data file name"))?;
-        if s.next().is_some() {
-            return Err(FilePathError::new("invalid relation data file name"));
-        };
-
-        let (relnode, forknum, segno, lsn) = parse_filename(fname)?;
-
-        Ok(ParsedBaseImageFileName {
-            spcnode: pg_constants::DEFAULTTABLESPACE_OID,
-            dbnode,
-            relnode,
-            forknum,
-            segno,
-            lsn,
-        })
-    } else if let Some(_) = path.strip_prefix("pg_tblspc/") {
-        // TODO
-        Err(FilePathError::new("tablespaces not supported"))
-    } else {
-        Err(FilePathError::new("invalid relation data file name"))
-    }
-}
-
-//
-// Load a base file from S3, and insert it into the page cache
-//
-async fn slurp_base_file(
-    conf: &PageServerConf,
-    sys_id: u64,
-    bucket: Bucket,
-    s3path: String,
-    parsed: ParsedBaseImageFileName,
-) {
-    // FIXME: rust-s3 opens a new connection for each request. Should reuse
-    // the reqwest::Client object. But that requires changes to rust-s3 itself.
-    let (data, code) = bucket.get_object(s3path.clone()).await.unwrap();
-
-    trace!("got response: {} on {}", code, &s3path);
-    assert_eq!(200, code);
-
-    let mut bytes = BytesMut::from(data.as_slice()).freeze();
-
-    let mut blknum: u32 = parsed.segno * (1024 * 1024 * 1024 / pg_constants::BLCKSZ as u32);
-
-    let pcache = page_cache::get_pagecache(conf, sys_id);
-
-    while bytes.remaining() >= 8192 {
-        let tag = page_cache::BufferTag {
-            rel: page_cache::RelTag {
-                spcnode: parsed.spcnode,
-                dbnode: parsed.dbnode,
-                relnode: parsed.relnode,
-                forknum: parsed.forknum,
-            },
-            blknum,
-        };
-
-        pcache.put_page_image(tag, parsed.lsn, bytes.copy_to_bytes(8192));
-
-        blknum += 1;
-    }
-}

pageserver/src/rocksdb_storage.rs

@@ -0,0 +1,475 @@
+//!
+//! An implementation of the ObjectStore interface, backed by RocksDB
+//!
+use crate::object_key::*;
+use crate::object_store::ObjectStore;
+use crate::relish::*;
+use crate::PageServerConf;
+use anyhow::{bail, Result};
+use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
+use std::sync::{Arc, Mutex};
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
+use zenith_utils::zid::ZTenantId;
+use zenith_utils::zid::ZTimelineId;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct StorageKey {
+    obj_key: ObjectKey,
+    lsn: Lsn,
+}
+
+impl StorageKey {
+    /// The first key for a given timeline
+    fn timeline_start(timeline: ZTimelineId) -> Self {
+        Self {
+            obj_key: ObjectKey {
+                timeline,
+                tag: ObjectTag::TimelineMetadataTag,
+            },
+            lsn: Lsn(0),
+        }
+    }
+}
+
+///
+/// RocksDB very inefficiently delete random record. Instead of it we have to use merge
+/// filter, which allows to throw away records at LSM merge phase.
+/// Unfortunately, it is hard (if ever possible)  to determine whether version can be removed
+/// at merge time. Version ca be removed if:
+/// 1. It is above PITR horizon (we need to get current LSN and gc_horizon from config)
+/// 2. Page is reconstructed at horizon (all WAL records above horizon are applied and can be removed)
+///
+/// So we have GC process which reconstructs pages at horizon and mark deteriorated WAL record
+/// for deletion. To mark object for deletion we can either set some flag in object itself.
+/// But it is complicated with new object value format, because RocksDB storage knows nothing about
+/// this format. Also updating whole record just to set one bit seems to be inefficient in any case.
+/// This is why we keep keys of marked for deletion versions in HashSet in memory.
+/// When LSM merge filter found key in this map, it removes it from the set preventing memory overflow.
+///
+struct GarbageCollector {
+    garbage: Mutex<HashSet<Vec<u8>>>,
+}
+
+impl GarbageCollector {
+    fn new() -> GarbageCollector {
+        GarbageCollector {
+            garbage: Mutex::new(HashSet::new()),
+        }
+    }
+
+    /// Called by GC to mark version as delete
+    fn mark_for_deletion(&self, key: &[u8]) {
+        let mut garbage = self.garbage.lock().unwrap();
+        garbage.insert(key.to_vec());
+    }
+
+    /// Called by LSM merge filter. If it finds key in the set, then
+    /// it doesn't merge it and removes from this set.
+    fn was_deleted(&self, key: &[u8]) -> bool {
+        let key = key.to_vec();
+        let mut garbage = self.garbage.lock().unwrap();
+        garbage.remove(&key)
+    }
+}
+
+pub struct RocksObjectStore {
+    _conf: &'static PageServerConf,
+
+    // RocksDB handle
+    db: rocksdb::DB,
+    gc: Arc<GarbageCollector>,
+}
+
+impl ObjectStore for RocksObjectStore {
+    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>> {
+        let val = self.db.get(StorageKey::ser(&StorageKey {
+            obj_key: key.clone(),
+            lsn,
+        })?)?;
+        if let Some(val) = val {
+            Ok(val)
+        } else {
+            bail!("could not find page {:?}", key);
+        }
+    }
+
+    fn get_next_key(&self, key: &ObjectKey) -> Result<Option<ObjectKey>> {
+        let mut iter = self.db.raw_iterator();
+        let search_key = StorageKey {
+            obj_key: key.clone(),
+            lsn: Lsn(0),
+        };
+        iter.seek(search_key.ser()?);
+        if !iter.valid() {
+            Ok(None)
+        } else {
+            let key = StorageKey::des(iter.key().unwrap())?;
+            Ok(Some(key.obj_key.clone()))
+        }
+    }
+
+    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()> {
+        self.db.put(
+            StorageKey::ser(&StorageKey {
+                obj_key: key.clone(),
+                lsn,
+            })?,
+            value,
+        )?;
+        Ok(())
+    }
+
+    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()> {
+        self.gc.mark_for_deletion(&StorageKey::ser(&StorageKey {
+            obj_key: key.clone(),
+            lsn,
+        })?);
+        Ok(())
+    }
+
+    /// Iterate through page versions of given page, starting from the given LSN.
+    /// The versions are walked in descending LSN order.
+    fn object_versions<'a>(
+        &'a self,
+        key: &ObjectKey,
+        lsn: Lsn,
+    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>> {
+        let iter = RocksObjectVersionIter::new(&self.db, key, lsn)?;
+        Ok(Box::new(iter))
+    }
+
+    /// Iterate through all timeline objects
+    fn list_objects<'a>(
+        &'a self,
+        timeline: ZTimelineId,
+        lsn: Lsn,
+    ) -> Result<Box<dyn Iterator<Item = ObjectTag> + 'a>> {
+        let iter = RocksObjectIter::new(&self.db, timeline, lsn)?;
+        Ok(Box::new(iter))
+    }
+
+    /// Get a list of all distinct relations in given tablespace and database.
+    ///
+    /// TODO: This implementation is very inefficient, it scans
+    /// through all entries in the given database. In practice, this
+    /// is used for CREATE DATABASE, and usually the template database is small.
+    /// But if it's not, this will be slow.
+    fn list_rels(
+        &self,
+        timelineid: ZTimelineId,
+        spcnode: u32,
+        dbnode: u32,
+        lsn: Lsn,
+    ) -> Result<HashSet<RelTag>> {
+        // FIXME: This scans everything. Very slow
+
+        let mut rels: HashSet<RelTag> = HashSet::new();
+
+        let mut search_rel_tag = RelTag {
+            spcnode,
+            dbnode,
+            relnode: 0,
+            forknum: 0u8,
+        };
+        let mut iter = self.db.raw_iterator();
+        loop {
+            let search_key = StorageKey {
+                obj_key: ObjectKey {
+                    timeline: timelineid,
+                    tag: ObjectTag::RelationMetadata(RelishTag::Relation(search_rel_tag)),
+                },
+                lsn: Lsn(0),
+            };
+            iter.seek(search_key.ser()?);
+            if !iter.valid() {
+                break;
+            }
+            let key = StorageKey::des(iter.key().unwrap())?;
+
+            if let ObjectTag::RelationMetadata(RelishTag::Relation(rel_tag)) = key.obj_key.tag {
+                if spcnode != 0 && rel_tag.spcnode != spcnode
+                    || dbnode != 0 && rel_tag.dbnode != dbnode
+                {
+                    break;
+                }
+                if key.lsn <= lsn {
+                    // visible in this snapshot
+                    rels.insert(rel_tag);
+                }
+                search_rel_tag = rel_tag;
+                // skip to next relation
+                // FIXME: What if relnode is u32::MAX ?
+                search_rel_tag.relnode += 1;
+            } else {
+                // no more relation metadata entries
+                break;
+            }
+        }
+
+        Ok(rels)
+    }
+
+    /// Get a list of all distinct NON-relations in timeline
+    /// that are visible at given lsn.
+    ///
+    /// TODO: This implementation is very inefficient, it scans
+    /// through all non-rel page versions in the system. In practice, this
+    /// is used when initializing a new compute node, and the non-rel files
+    /// are never very large nor change very frequently, so this will do for now.
+    fn list_nonrels(&self, timelineid: ZTimelineId, lsn: Lsn) -> Result<HashSet<RelishTag>> {
+        let mut rels: HashSet<RelishTag> = HashSet::new();
+
+        let search_key = StorageKey {
+            obj_key: ObjectKey {
+                timeline: timelineid,
+                tag: ObjectTag::Buffer(FIRST_NONREL_RELISH_TAG, 0),
+            },
+            lsn: Lsn(0),
+        };
+
+        let mut iter = self.db.raw_iterator();
+        iter.seek(search_key.ser()?);
+        while iter.valid() {
+            let key = StorageKey::des(iter.key().unwrap())?;
+
+            if key.obj_key.timeline != timelineid {
+                // reached end of this timeline in the store
+                break;
+            }
+
+            if let ObjectTag::Buffer(rel_tag, _blknum) = key.obj_key.tag {
+                if key.lsn <= lsn {
+                    // visible in this snapshot
+                    rels.insert(rel_tag);
+                }
+            }
+            // TODO: we could skip to next relation here like we do in list_rels(),
+            // but hopefully there are not that many SLRU segments or other non-rel
+            // entries for it to matter.
+            iter.next();
+        }
+
+        Ok(rels)
+    }
+
+    /// Iterate through versions of all objects in a timeline.
+    ///
+    /// Returns objects in increasing key-version order.
+    /// Returns all versions up to and including the specified LSN.
+    fn objects<'a>(
+        &'a self,
+        timeline: ZTimelineId,
+        lsn: Lsn,
+    ) -> Result<Box<dyn Iterator<Item = Result<(ObjectTag, Lsn, Vec<u8>)>> + 'a>> {
+        let start_key = StorageKey::timeline_start(timeline);
+        let start_key_bytes = StorageKey::ser(&start_key)?;
+        let iter = self.db.iterator(rocksdb::IteratorMode::From(
+            &start_key_bytes,
+            rocksdb::Direction::Forward,
+        ));
+
+        Ok(Box::new(RocksObjects {
+            iter,
+            timeline,
+            lsn,
+        }))
+    }
+
+    fn compact(&self) {
+        self.db.compact_range::<&[u8], &[u8]>(None, None);
+    }
+}
+
+impl RocksObjectStore {
+    /// Open a RocksDB database.
+    pub fn open(conf: &'static PageServerConf, tenantid: &ZTenantId) -> Result<RocksObjectStore> {
+        let opts = Self::get_rocksdb_opts();
+        let obj_store = Self::new(conf, opts, tenantid)?;
+        Ok(obj_store)
+    }
+
+    /// Create a new, empty RocksDB database.
+    pub fn create(conf: &'static PageServerConf, tenantid: &ZTenantId) -> Result<RocksObjectStore> {
+        let path = conf.tenant_path(&tenantid).join("rocksdb-storage");
+        std::fs::create_dir(&path)?;
+
+        let mut opts = Self::get_rocksdb_opts();
+        opts.create_if_missing(true);
+        opts.set_error_if_exists(true);
+        let obj_store = Self::new(conf, opts, tenantid)?;
+        Ok(obj_store)
+    }
+
+    fn new(
+        conf: &'static PageServerConf,
+        mut opts: rocksdb::Options,
+        tenantid: &ZTenantId,
+    ) -> Result<RocksObjectStore> {
+        let path = conf.tenant_path(&tenantid).join("rocksdb-storage");
+        let gc = Arc::new(GarbageCollector::new());
+        let gc_ref = gc.clone();
+        opts.set_compaction_filter("ttl", move |_level: u32, key: &[u8], _val: &[u8]| {
+            if gc_ref.was_deleted(key) {
+                rocksdb::compaction_filter::Decision::Remove
+            } else {
+                rocksdb::compaction_filter::Decision::Keep
+            }
+        });
+        let db = rocksdb::DB::open(&opts, &path)?;
+        let obj_store = RocksObjectStore {
+            _conf: conf,
+            db,
+            gc,
+        };
+        Ok(obj_store)
+    }
+
+    /// common options used by `open` and `create`
+    fn get_rocksdb_opts() -> rocksdb::Options {
+        let mut opts = rocksdb::Options::default();
+        opts.set_use_fsync(true);
+        opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
+        opts
+    }
+}
+
+///
+/// Iterator for `object_versions`. Returns all page versions of a given block, in
+/// reverse LSN order.
+///
+struct RocksObjectVersionIter<'a> {
+    obj_key: ObjectKey,
+    dbiter: rocksdb::DBRawIterator<'a>,
+    first_call: bool,
+}
+impl<'a> RocksObjectVersionIter<'a> {
+    fn new(
+        db: &'a rocksdb::DB,
+        obj_key: &ObjectKey,
+        lsn: Lsn,
+    ) -> Result<RocksObjectVersionIter<'a>> {
+        let key = StorageKey {
+            obj_key: obj_key.clone(),
+            lsn,
+        };
+        let mut dbiter = db.raw_iterator();
+        dbiter.seek_for_prev(StorageKey::ser(&key)?); // locate last entry
+        Ok(RocksObjectVersionIter {
+            first_call: true,
+            obj_key: obj_key.clone(),
+            dbiter,
+        })
+    }
+}
+impl<'a> Iterator for RocksObjectVersionIter<'a> {
+    type Item = (Lsn, Vec<u8>);
+
+    fn next(&mut self) -> std::option::Option<Self::Item> {
+        if self.first_call {
+            self.first_call = false;
+        } else {
+            self.dbiter.prev(); // walk backwards
+        }
+
+        if !self.dbiter.valid() {
+            return None;
+        }
+        let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
+        if key.obj_key.tag != self.obj_key.tag {
+            return None;
+        }
+        let val = self.dbiter.value().unwrap();
+        let result = val.to_vec();
+
+        Some((key.lsn, result))
+    }
+}
+
+struct RocksObjects<'r> {
+    iter: rocksdb::DBIterator<'r>,
+    timeline: ZTimelineId,
+    lsn: Lsn,
+}
+
+impl<'r> Iterator for RocksObjects<'r> {
+    // TODO consider returning Box<[u8]>
+    type Item = Result<(ObjectTag, Lsn, Vec<u8>)>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.next_result().transpose()
+    }
+}
+
+impl<'r> RocksObjects<'r> {
+    fn next_result(&mut self) -> Result<Option<(ObjectTag, Lsn, Vec<u8>)>> {
+        for (key_bytes, v) in &mut self.iter {
+            let key = StorageKey::des(&key_bytes)?;
+
+            if key.obj_key.timeline != self.timeline {
+                return Ok(None);
+            }
+
+            if key.lsn > self.lsn {
+                // TODO can speed up by seeking iterator
+                continue;
+            }
+
+            return Ok(Some((key.obj_key.tag, key.lsn, v.to_vec())));
+        }
+
+        Ok(None)
+    }
+}
+
+///
+/// Iterator for `list_objects`. Returns all objects preceeding specified LSN
+///
+struct RocksObjectIter<'a> {
+    timeline: ZTimelineId,
+    key: StorageKey,
+    lsn: Lsn,
+    dbiter: rocksdb::DBRawIterator<'a>,
+}
+impl<'a> RocksObjectIter<'a> {
+    fn new(db: &'a rocksdb::DB, timeline: ZTimelineId, lsn: Lsn) -> Result<RocksObjectIter<'a>> {
+        let key = StorageKey {
+            obj_key: ObjectKey {
+                timeline,
+                tag: ObjectTag::FirstTag,
+            },
+            lsn: Lsn(0),
+        };
+        let dbiter = db.raw_iterator();
+        Ok(RocksObjectIter {
+            key,
+            timeline,
+            lsn,
+            dbiter,
+        })
+    }
+}
+impl<'a> Iterator for RocksObjectIter<'a> {
+    type Item = ObjectTag;
+
+    fn next(&mut self) -> std::option::Option<Self::Item> {
+        loop {
+            self.dbiter.seek(StorageKey::ser(&self.key).unwrap());
+            if !self.dbiter.valid() {
+                return None;
+            }
+            let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
+            if key.obj_key.timeline != self.timeline {
+                // End of this timeline
+                return None;
+            }
+            self.key = key.clone();
+            self.key.lsn = Lsn(u64::MAX); // next seek should skip all versions
+            if key.lsn <= self.lsn {
+                // visible in this snapshot
+                return Some(key.obj_key.tag);
+            }
+        }
+    }
+}

pageserver/src/tui.rs

@@ -1,307 +0,0 @@
-use crate::tui_event::{Event, Events};
-use crate::tui_logger::TuiLogger;
-use crate::tui_logger::TuiLoggerWidget;
-
-use lazy_static::lazy_static;
-use std::sync::Arc;
-use std::{error::Error, io};
-use termion::{event::Key, input::MouseTerminal, raw::IntoRawMode, screen::AlternateScreen};
-use tui::backend::TermionBackend;
-use tui::buffer::Buffer;
-use tui::layout::{Constraint, Direction, Layout, Rect};
-use tui::style::{Color, Modifier, Style};
-use tui::text::{Span, Spans, Text};
-use tui::widgets::{Block, BorderType, Borders, Paragraph, Widget};
-use tui::Terminal;
-
-use slog::Drain;
-
-lazy_static! {
-    pub static ref PAGESERVICE_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
-    pub static ref WALRECEIVER_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
-    pub static ref WALREDO_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
-    pub static ref CATCHALL_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
-}
-
-pub fn init_logging() -> slog_scope::GlobalLoggerGuard {
-    let pageservice_drain =
-        slog::Filter::new(PAGESERVICE_DRAIN.as_ref(), |record: &slog::Record| {
-            if record.level().is_at_least(slog::Level::Debug)
-                && record.module().starts_with("pageserver::page_service")
-            {
-                return true;
-            }
-            false
-        })
-        .fuse();
-
-    let walredo_drain = slog::Filter::new(WALREDO_DRAIN.as_ref(), |record: &slog::Record| {
-        if record.level().is_at_least(slog::Level::Debug)
-            && record.module().starts_with("pageserver::walredo")
-        {
-            return true;
-        }
-        false
-    })
-    .fuse();
-
-    let walreceiver_drain =
-        slog::Filter::new(WALRECEIVER_DRAIN.as_ref(), |record: &slog::Record| {
-            if record.level().is_at_least(slog::Level::Debug)
-                && record.module().starts_with("pageserver::walreceiver")
-            {
-                return true;
-            }
-            false
-        })
-        .fuse();
-
-    let catchall_drain = slog::Filter::new(CATCHALL_DRAIN.as_ref(), |record: &slog::Record| {
-        if record.level().is_at_least(slog::Level::Info) {
-            return true;
-        }
-        if record.level().is_at_least(slog::Level::Debug)
-            && record.module().starts_with("pageserver")
-        {
-            return true;
-        }
-        false
-    })
-    .fuse();
-
-    let drain = pageservice_drain;
-    let drain = slog::Duplicate::new(drain, walreceiver_drain).fuse();
-    let drain = slog::Duplicate::new(drain, walredo_drain).fuse();
-    let drain = slog::Duplicate::new(drain, catchall_drain).fuse();
-    let drain = slog_async::Async::new(drain).chan_size(1000).build().fuse();
-    let drain = slog::Filter::new(drain, |record: &slog::Record| {
-        if record.level().is_at_least(slog::Level::Info) {
-            return true;
-        }
-        if record.level().is_at_least(slog::Level::Debug)
-            && record.module().starts_with("pageserver")
-        {
-            return true;
-        }
-
-        false
-    })
-    .fuse();
-    let logger = slog::Logger::root(drain, slog::o!());
-    slog_scope::set_global_logger(logger)
-}
-
-pub fn ui_main() -> Result<(), Box<dyn Error>> {
-    // Terminal initialization
-    let stdout = io::stdout().into_raw_mode()?;
-    let stdout = MouseTerminal::from(stdout);
-    let stdout = AlternateScreen::from(stdout);
-    let backend = TermionBackend::new(stdout);
-    let mut terminal = Terminal::new(backend)?;
-
-    // Setup event handlers
-    let events = Events::new();
-
-    loop {
-        terminal.draw(|f| {
-            let size = f.size();
-
-            // +----------------+----------------+
-            // |                |                |
-            // |  top_top_left  | top_top_right  |
-            // |                |                |
-            // +----------------+----------------|
-            // |                |                |
-            // |  top_bot_left  | top_left_right |
-            // |                |                |
-            // +----------------+----------------+
-            // |                                 |
-            // |             bottom              |
-            // |                                 |
-            // +---------------------------------+
-            let chunks = Layout::default()
-                .direction(Direction::Vertical)
-                .constraints([Constraint::Percentage(70), Constraint::Percentage(30)].as_ref())
-                .split(size);
-            let top_chunk = chunks[0];
-            let bottom_chunk = chunks[1];
-
-            let top_chunks = Layout::default()
-                .direction(Direction::Horizontal)
-                .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
-                .split(top_chunk);
-            let top_left_chunk = top_chunks[0];
-            let top_right_chunk = top_chunks[1];
-
-            let c = Layout::default()
-                .direction(Direction::Vertical)
-                .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
-                .split(top_left_chunk);
-            let top_top_left_chunk = c[0];
-            let top_bot_left_chunk = c[1];
-
-            let c = Layout::default()
-                .direction(Direction::Vertical)
-                .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
-                .split(top_right_chunk);
-            let top_top_right_chunk = c[0];
-            let top_bot_right_chunk = c[1];
-
-            f.render_widget(
-                LogWidget::new(PAGESERVICE_DRAIN.as_ref(), "Page Service"),
-                top_top_left_chunk,
-            );
-
-            f.render_widget(
-                LogWidget::new(WALREDO_DRAIN.as_ref(), "WAL Redo"),
-                top_bot_left_chunk,
-            );
-
-            f.render_widget(
-                LogWidget::new(WALRECEIVER_DRAIN.as_ref(), "WAL Receiver"),
-                top_top_right_chunk,
-            );
-
-            f.render_widget(MetricsWidget {}, top_bot_right_chunk);
-
-            f.render_widget(
-                LogWidget::new(CATCHALL_DRAIN.as_ref(), "All Log").show_module(true),
-                bottom_chunk,
-            );
-        })?;
-
-        // If ther user presses 'q', quit.
-
-        // silence clippy's suggestion to rewrite this as an if-statement. Match
-        // makes more sense as soon as we get another command than 'q'.
-        #[allow(clippy::single_match)]
-        #[allow(clippy::collapsible_match)]
-        if let Event::Input(key) = events.next()? {
-            match key {
-                Key::Char('q') => {
-                    break;
-                }
-                _ => (),
-            }
-        }
-    }
-
-    terminal.show_cursor().unwrap();
-    terminal.clear().unwrap();
-
-    Ok(())
-}
-
-#[allow(dead_code)]
-struct LogWidget<'a> {
-    logger: &'a TuiLogger,
-    title: &'a str,
-    show_module: bool,
-}
-
-impl<'a> LogWidget<'a> {
-    fn new(logger: &'a TuiLogger, title: &'a str) -> LogWidget<'a> {
-        LogWidget {
-            logger,
-            title,
-            show_module: false,
-        }
-    }
-
-    fn show_module(mut self, b: bool) -> LogWidget<'a> {
-        self.show_module = b;
-        self
-    }
-}
-
-impl<'a> Widget for LogWidget<'a> {
-    fn render(self, area: Rect, buf: &mut Buffer) {
-        let w = TuiLoggerWidget::default(self.logger)
-            .block(
-                Block::default()
-                    .borders(Borders::ALL)
-                    .title(self.title)
-                    .border_type(BorderType::Rounded),
-            )
-            .show_module(true)
-            .style_error(Style::default().fg(Color::Red))
-            .style_warn(Style::default().fg(Color::Yellow))
-            .style_info(Style::default().fg(Color::Green));
-        w.render(area, buf);
-    }
-}
-
-// Render a widget to show some metrics
-struct MetricsWidget {}
-
-fn _get_metric_u64(title: &str, value: u64) -> Spans {
-    Spans::from(vec![
-        Span::styled(format!("{:<20}", title), Style::default()),
-        Span::raw(": "),
-        Span::styled(
-            value.to_string(),
-            Style::default().add_modifier(Modifier::BOLD),
-        ),
-    ])
-}
-
-// This is not used since LSNs were removed from page cache stats.
-// Maybe it will be used in the future?
-fn _get_metric_str<'a>(title: &str, value: &'a str) -> Spans<'a> {
-    Spans::from(vec![
-        Span::styled(format!("{:<20}", title), Style::default()),
-        Span::raw(": "),
-        Span::styled(value, Style::default().add_modifier(Modifier::BOLD)),
-    ])
-}
-
-impl tui::widgets::Widget for MetricsWidget {
-    fn render(self, area: Rect, buf: &mut Buffer) {
-        let block = Block::default()
-            .borders(Borders::ALL)
-            .title("Page Cache Metrics")
-            .border_type(BorderType::Rounded);
-        let inner_area = block.inner(area);
-
-        block.render(area, buf);
-
-        #[allow(unused_mut)]
-        let mut lines: Vec<Spans> = Vec::new();
-
-        // FIXME
-        //let page_cache_stats = crate::page_cache::get_stats();
-
-        // This is not used since LSNs were removed from page cache stats.
-        // Maybe it will be used in the future?
-        /*
-        let lsnrange = format!(
-            "{} - {}",
-            page_cache_stats.first_valid_lsn, page_cache_stats.last_valid_lsn
-        );
-        let last_valid_recordlsn_str = page_cache_stats.last_record_lsn.to_string();
-        lines.push(get_metric_str("Valid LSN range", &lsnrange));
-        lines.push(get_metric_str("Last record LSN", &last_valid_recordlsn_str));
-        */
-        /*
-        lines.push(get_metric_u64(
-            "# of cache entries",
-            page_cache_stats.num_entries,
-        ));
-        lines.push(get_metric_u64(
-            "# of page images",
-            page_cache_stats.num_page_images,
-        ));
-        lines.push(get_metric_u64(
-            "# of WAL records",
-            page_cache_stats.num_wal_records,
-        ));
-        lines.push(get_metric_u64(
-            "# of GetPage@LSN calls",
-            page_cache_stats.num_getpage_requests,
-        ));
-        */
-        let text = Text::from(lines);
-
-        Paragraph::new(text).render(inner_area, buf);
-    }
-}

pageserver/src/tui_event.rs

@@ -1,96 +0,0 @@
-use std::io;
-use std::sync::mpsc;
-use std::sync::{
-    atomic::{AtomicBool, Ordering},
-    Arc,
-};
-use std::thread;
-use std::time::Duration;
-
-use termion::event::Key;
-use termion::input::TermRead;
-
-pub enum Event<I> {
-    Input(I),
-    Tick,
-}
-
-/// A small event handler that wrap termion input and tick events. Each event
-/// type is handled in its own thread and returned to a common `Receiver`
-#[allow(dead_code)]
-pub struct Events {
-    rx: mpsc::Receiver<Event<Key>>,
-    input_handle: thread::JoinHandle<()>,
-    ignore_exit_key: Arc<AtomicBool>,
-    tick_handle: thread::JoinHandle<()>,
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct Config {
-    pub exit_key: Key,
-    pub tick_rate: Duration,
-}
-
-impl Default for Config {
-    fn default() -> Config {
-        Config {
-            exit_key: Key::Char('q'),
-            tick_rate: Duration::from_millis(250),
-        }
-    }
-}
-
-impl Events {
-    pub fn new() -> Events {
-        Events::with_config(Config::default())
-    }
-
-    pub fn with_config(config: Config) -> Events {
-        let (tx, rx) = mpsc::channel();
-        let ignore_exit_key = Arc::new(AtomicBool::new(false));
-        let input_handle = {
-            let tx = tx.clone();
-            let ignore_exit_key = ignore_exit_key.clone();
-            thread::spawn(move || {
-                let stdin = io::stdin();
-                for evt in stdin.keys() {
-                    if let Ok(key) = evt {
-                        if let Err(err) = tx.send(Event::Input(key)) {
-                            eprintln!("{}", err);
-                            return;
-                        }
-                        if !ignore_exit_key.load(Ordering::Relaxed) && key == config.exit_key {
-                            return;
-                        }
-                    }
-                }
-            })
-        };
-        let tick_handle = {
-            thread::spawn(move || loop {
-                if tx.send(Event::Tick).is_err() {
-                    break;
-                }
-                thread::sleep(config.tick_rate);
-            })
-        };
-        Events {
-            rx,
-            input_handle,
-            ignore_exit_key,
-            tick_handle,
-        }
-    }
-
-    pub fn next(&self) -> Result<Event<Key>, mpsc::RecvError> {
-        self.rx.recv()
-    }
-
-    pub fn disable_exit_key(&mut self) {
-        self.ignore_exit_key.store(true, Ordering::Relaxed);
-    }
-
-    pub fn enable_exit_key(&mut self) {
-        self.ignore_exit_key.store(false, Ordering::Relaxed);
-    }
-}

pageserver/src/tui_logger.rs

@@ -1,199 +0,0 @@
-//
-// A TUI Widget that displays log entries
-//
-// This is heavily inspired by gin66's tui_logger crate at https://github.com/gin66/tui-logger,
-// but I wrote this based on the 'slog' module, which simplified things a lot. tui-logger also
-// implemented the slog Drain trait, but it had a model of one global buffer for the records.
-// With this implementation, each TuiLogger is a separate ring buffer and separate slog Drain.
-// Also, I didn't do any of the "hot log" stuff that gin66's implementation had, you can use an
-// AsyncDrain to buffer and handle overflow if desired.
-//
-use chrono::offset::Local;
-use chrono::DateTime;
-use slog::{Drain, Level, OwnedKVList, Record};
-use slog_async::AsyncRecord;
-use std::collections::VecDeque;
-use std::sync::Mutex;
-use std::time::SystemTime;
-use tui::buffer::Buffer;
-use tui::layout::Rect;
-use tui::style::{Modifier, Style};
-use tui::text::{Span, Spans};
-use tui::widgets::{Block, Paragraph, Widget, Wrap};
-
-// Size of the log ring buffer, in # of records
-static BUFFER_SIZE: usize = 1000;
-
-pub struct TuiLogger {
-    events: Mutex<VecDeque<(SystemTime, AsyncRecord)>>,
-}
-
-impl<'a> Default for TuiLogger {
-    fn default() -> TuiLogger {
-        TuiLogger {
-            events: Mutex::new(VecDeque::with_capacity(BUFFER_SIZE)),
-        }
-    }
-}
-
-impl Drain for TuiLogger {
-    type Ok = ();
-    type Err = slog::Error;
-
-    fn log(&self, record: &Record, values: &OwnedKVList) -> Result<Self::Ok, Self::Err> {
-        let mut events = self.events.lock().unwrap();
-
-        let now = SystemTime::now();
-        let asyncrec = AsyncRecord::from(record, values);
-        events.push_front((now, asyncrec));
-
-        if events.len() > BUFFER_SIZE {
-            events.pop_back();
-        }
-
-        Ok(())
-    }
-}
-
-// TuiLoggerWidget renders a TuiLogger ring buffer
-pub struct TuiLoggerWidget<'b> {
-    block: Option<Block<'b>>,
-    /// Base style of the widget
-    style: Style,
-    /// Level based style
-    style_error: Option<Style>,
-    style_warn: Option<Style>,
-    style_debug: Option<Style>,
-    style_trace: Option<Style>,
-    style_info: Option<Style>,
-    show_module: bool,
-    logger: &'b TuiLogger,
-}
-impl<'b> TuiLoggerWidget<'b> {
-    pub fn default(logger: &'b TuiLogger) -> TuiLoggerWidget<'b> {
-        TuiLoggerWidget {
-            block: None,
-            style: Default::default(),
-            style_error: None,
-            style_warn: None,
-            style_debug: None,
-            style_trace: None,
-            style_info: None,
-            show_module: true,
-            logger,
-        }
-    }
-}
-impl<'b> TuiLoggerWidget<'b> {
-    pub fn block(mut self, block: Block<'b>) -> TuiLoggerWidget<'b> {
-        self.block = Some(block);
-        self
-    }
-    #[allow(unused)]
-    pub fn style(mut self, style: Style) -> TuiLoggerWidget<'b> {
-        self.style = style;
-        self
-    }
-    pub fn style_error(mut self, style: Style) -> TuiLoggerWidget<'b> {
-        self.style_error = Some(style);
-        self
-    }
-    pub fn style_warn(mut self, style: Style) -> TuiLoggerWidget<'b> {
-        self.style_warn = Some(style);
-        self
-    }
-    pub fn style_info(mut self, style: Style) -> TuiLoggerWidget<'b> {
-        self.style_info = Some(style);
-        self
-    }
-    #[allow(unused)]
-    pub fn style_trace(mut self, style: Style) -> TuiLoggerWidget<'b> {
-        self.style_trace = Some(style);
-        self
-    }
-    #[allow(unused)]
-    pub fn style_debug(mut self, style: Style) -> TuiLoggerWidget<'b> {
-        self.style_debug = Some(style);
-        self
-    }
-
-    pub fn show_module(mut self, b: bool) -> TuiLoggerWidget<'b> {
-        self.show_module = b;
-        self
-    }
-}
-impl<'b> Widget for TuiLoggerWidget<'b> {
-    fn render(mut self, area: Rect, buf: &mut Buffer) {
-        buf.set_style(area, self.style);
-        let list_area = match self.block.take() {
-            Some(b) => {
-                let inner_area = b.inner(area);
-                b.render(area, buf);
-                inner_area
-            }
-            None => area,
-        };
-        if list_area.width == 0 || list_area.height == 0 {
-            return;
-        }
-
-        let la_height = list_area.height as usize;
-
-        //
-        // Iterate through the records in the buffer. The records are
-        // pushed to the front, so the newest records come first.
-        //
-        let mut lines: Vec<Spans> = Vec::new();
-
-        let style_msg = Style::default().add_modifier(Modifier::BOLD);
-        {
-            let events = self.logger.events.lock().unwrap();
-
-            for evt in events.iter() {
-                let (timestamp, rec) = evt;
-
-                rec.as_record_values(|rec, _kwlist| {
-                    let mut line: Vec<Span> = Vec::new();
-
-                    let datetime: DateTime<Local> = timestamp.clone().into();
-                    let ts = format!("{}", datetime.format("%H:%M:%S%.3f "));
-                    line.push(Span::raw(ts));
-
-                    let (lvl_style, txt, with_loc) = match rec.level() {
-                        Level::Critical => (self.style_error, "CRIT ", true),
-                        Level::Error => (self.style_error, "ERROR", true),
-                        Level::Warning => (self.style_warn, "WARN ", true),
-                        Level::Info => (self.style_info, "INFO ", false),
-                        Level::Debug => (self.style_debug, "DEBUG", true),
-                        Level::Trace => (self.style_trace, "TRACE", true),
-                    };
-                    line.push(Span::styled(txt, lvl_style.unwrap_or_default()));
-
-                    if self.show_module {
-                        line.push(Span::raw(" "));
-                        line.push(Span::raw(rec.module()));
-                    }
-                    if with_loc {
-                        let loc = format!(" {}:{}", rec.file(), rec.line());
-                        line.push(Span::raw(loc));
-                    }
-                    let msg = format!(" {}", rec.msg());
-                    line.push(Span::styled(msg, style_msg));
-
-                    lines.push(Spans::from(line));
-                });
-                if lines.len() == la_height {
-                    break;
-                }
-            }
-        }
-
-        lines.reverse();
-
-        let text = tui::text::Text::from(lines);
-
-        Paragraph::new(text)
-            .wrap(Wrap { trim: true })
-            .render(list_area, buf);
-    }
-}

pageserver/src/waldecoder.rs

@@ -1,62 +1,37 @@
+//!
+//! WAL decoder. For each WAL record, it decodes the record to figure out which data blocks
+//! the record affects, to add the records to the page cache.
+//!
 use bytes::{Buf, BufMut, Bytes, BytesMut};
+use crc32c::*;
 use log::*;
-use postgres_ffi::xlog_utils::XLogRecord;
-use postgres_ffi::*;
+use postgres_ffi::pg_constants;
+use postgres_ffi::xlog_utils::*;
+use postgres_ffi::XLogLongPageHeaderData;
+use postgres_ffi::XLogPageHeaderData;
+use postgres_ffi::XLogRecord;
+use postgres_ffi::{Oid, TransactionId};
 use std::cmp::min;
-use std::str;
 use thiserror::Error;
 use zenith_utils::lsn::Lsn;
 
-pub type Oid = u32;
-pub type TransactionId = u32;
 pub type BlockNumber = u32;
 pub type OffsetNumber = u16;
 pub type MultiXactId = TransactionId;
 pub type MultiXactOffset = u32;
 pub type MultiXactStatus = u32;
-pub type TimeLineID = u32;
-pub type PgTime = i64;
-
-// From PostgreSQL headers
-
-#[repr(C)]
-#[derive(Debug)]
-pub struct XLogPageHeaderData {
-    xlp_magic: u16,      /* magic value for correctness checks */
-    xlp_info: u16,       /* flag bits, see below */
-    xlp_tli: TimeLineID, /* TimeLineID of first record on page */
-    xlp_pageaddr: u64,   /* XLOG address of this page */
-    xlp_rem_len: u32,    /* total len of remaining data for record */
-}
-
-// FIXME: this assumes MAXIMUM_ALIGNOF 8. There are 4 padding bytes at end
-#[allow(non_upper_case_globals)]
-const SizeOfXLogShortPHD: usize = 2 + 2 + 4 + 8 + 4 + 4;
-
-#[repr(C)]
-#[derive(Debug)]
-pub struct XLogLongPageHeaderData {
-    std: XLogPageHeaderData, /* standard header fields */
-    xlp_sysid: u64,          /* system identifier from pg_control */
-    xlp_seg_size: u32,       /* just as a cross-check */
-    xlp_xlog_blcksz: u32,    /* just as a cross-check */
-}
-
-// FIXME: this assumes MAXIMUM_ALIGNOF 8.
-#[allow(non_upper_case_globals)]
-const SizeOfXLogLongPHD: usize = (2 + 2 + 4 + 8 + 4) + 4 + 8 + 4 + 4;
 
 #[allow(dead_code)]
 pub struct WalStreamDecoder {
     lsn: Lsn,
 
-    startlsn: Lsn, // LSN where this record starts
     contlen: u32,
     padlen: u32,
 
     inputbuf: BytesMut,
-
     recordbuf: BytesMut,
+
+    crc_check: bool,
 }
 
 #[derive(Error, Debug, Clone)]
@@ -71,24 +46,22 @@ pub struct WalDecodeError {
 // FIXME: This isn't a proper rust stream
 //
 impl WalStreamDecoder {
-    pub fn new(lsn: Lsn) -> WalStreamDecoder {
+    pub fn new(lsn: Lsn, crc_check: bool) -> WalStreamDecoder {
         WalStreamDecoder {
             lsn,
 
-            startlsn: Lsn(0),
             contlen: 0,
             padlen: 0,
 
             inputbuf: BytesMut::new(),
             recordbuf: BytesMut::new(),
+
+            crc_check,
         }
     }
 
-    pub fn set_position(&mut self, lsn: Lsn) {
-        self.lsn = lsn;
-        self.contlen = 0;
-        self.padlen = 0;
-        self.inputbuf.clear();
+    pub fn available(&self) -> Lsn {
+        self.lsn + self.inputbuf.remaining() as u64
     }
 
     pub fn feed_bytes(&mut self, buf: &[u8]) {
@@ -99,7 +72,7 @@ impl WalStreamDecoder {
     /// decoder so far.
     ///
     /// Returns one of the following:
-    ///     Ok((u64, Bytes)): a tuple containing the LSN of next record, and the record itself
+    ///     Ok((Lsn, Bytes)): a tuple containing the LSN of next record, and the record itself
     ///     Ok(None): there is not enough data in the input buffer. Feed more by calling the `feed_bytes` function
     ///     Err(WalDecodeError): an error occured while decoding, meaning the input was invalid.
     ///
@@ -109,11 +82,12 @@ impl WalStreamDecoder {
             if self.lsn.segment_offset(pg_constants::WAL_SEGMENT_SIZE) == 0 {
                 // parse long header
 
-                if self.inputbuf.remaining() < SizeOfXLogLongPHD {
+                if self.inputbuf.remaining() < XLOG_SIZE_OF_XLOG_LONG_PHD {
                     return Ok(None);
                 }
 
-                let hdr = self.decode_XLogLongPageHeaderData();
+                let hdr = XLogLongPageHeaderData::from_bytes(&mut self.inputbuf);
+
                 if hdr.std.xlp_pageaddr != self.lsn.0 {
                     return Err(WalDecodeError {
                         msg: "invalid xlog segment header".into(),
@@ -122,24 +96,32 @@ impl WalStreamDecoder {
                 }
                 // TODO: verify the remaining fields in the header
 
-                self.lsn += SizeOfXLogLongPHD as u64;
-                continue;
+                self.lsn += XLOG_SIZE_OF_XLOG_LONG_PHD as u64;
+                if !self.crc_check && self.contlen != hdr.std.xlp_rem_len {
+                    self.contlen = hdr.std.xlp_rem_len; // skip continuation record
+                }
             } else if self.lsn.block_offset() == 0 {
-                if self.inputbuf.remaining() < SizeOfXLogShortPHD {
+                if self.inputbuf.remaining() < XLOG_SIZE_OF_XLOG_SHORT_PHD {
                     return Ok(None);
                 }
 
-                let hdr = self.decode_XLogPageHeaderData();
+                let hdr = XLogPageHeaderData::from_bytes(&mut self.inputbuf);
+
                 if hdr.xlp_pageaddr != self.lsn.0 {
                     return Err(WalDecodeError {
-                        msg: "invalid xlog page header".into(),
+                        msg: format!(
+                            "invalid xlog page header: xlp_pageaddr={} vs. lsn={}",
+                            hdr.xlp_pageaddr, self.lsn
+                        ),
                         lsn: self.lsn,
                     });
                 }
                 // TODO: verify the remaining fields in the header
 
-                self.lsn += SizeOfXLogShortPHD as u64;
-                continue;
+                self.lsn += XLOG_SIZE_OF_XLOG_SHORT_PHD as u64;
+                if !self.crc_check && self.contlen != hdr.xlp_rem_len {
+                    self.contlen = hdr.xlp_rem_len; // skip continuation record
+                }
             } else if self.padlen > 0 {
                 if self.inputbuf.remaining() < self.padlen as usize {
                     return Ok(None);
@@ -157,9 +139,8 @@ impl WalStreamDecoder {
                 }
 
                 // read xl_tot_len FIXME: assumes little-endian
-                self.startlsn = self.lsn;
                 let xl_tot_len = self.inputbuf.get_u32_le();
-                if xl_tot_len < SizeOfXLogRecord {
+                if (xl_tot_len as usize) < XLOG_SIZE_OF_XLOG_RECORD {
                     return Err(WalDecodeError {
                         msg: format!("invalid xl_tot_len {}", xl_tot_len),
                         lsn: self.lsn,
@@ -172,7 +153,6 @@ impl WalStreamDecoder {
                 self.recordbuf.put_u32_le(xl_tot_len);
 
                 self.contlen = xl_tot_len - 4;
-                continue;
             } else {
                 // we're continuing a record, possibly from previous page.
                 let pageleft = self.lsn.remaining_in_block() as u32;
@@ -194,9 +174,10 @@ impl WalStreamDecoder {
                     let recordbuf = recordbuf.freeze();
                     let mut buf = recordbuf.clone();
 
+                    let xlogrec = XLogRecord::from_bytes(&mut buf);
+
                     // XLOG_SWITCH records are special. If we see one, we need to skip
                     // to the next WAL segment.
-                    let xlogrec = XLogRecord::from_bytes(&mut buf);
                     if xlogrec.is_xlog_switch_record() {
                         trace!("saw xlog switch record at {}", self.lsn);
                         self.padlen =
@@ -206,10 +187,29 @@ impl WalStreamDecoder {
                         self.padlen = self.lsn.calc_padding(8u32) as u32;
                     }
 
-                    let result = (self.lsn, recordbuf);
+                    // Check record CRC
+                    if self.crc_check {
+                        let mut crc = crc32c_append(0, &recordbuf[XLOG_RECORD_CRC_OFFS + 4..]);
+                        crc = crc32c_append(crc, &recordbuf[0..XLOG_RECORD_CRC_OFFS]);
+                        if crc != xlogrec.xl_crc {
+                            info!("WAL record crc mismatch n={}, buf.len()={}, lsn={}, rec={:?}, recordbuf={:?}",
+								  n, recordbuf.len(), self.lsn, xlogrec, recordbuf);
+                            return Err(WalDecodeError {
+                                msg: format!(
+                                    "WAL record crc mismatch n={}, buf.len()={}, lsn={}, rec={:?}",
+                                    n,
+                                    buf.len(),
+                                    self.lsn,
+                                    xlogrec
+                                ),
+                                lsn: self.lsn,
+                            });
+                        }
+                    }
+
+                    let result = (self.lsn.align(), recordbuf);
                     return Ok(Some(result));
                 }
-                continue;
             }
         }
         // check record boundaries
@@ -218,40 +218,6 @@ impl WalStreamDecoder {
 
         // deal with xlog_switch records
     }
-
-    #[allow(non_snake_case)]
-    fn decode_XLogPageHeaderData(&mut self) -> XLogPageHeaderData {
-        let buf = &mut self.inputbuf;
-
-        // FIXME: Assume little-endian
-
-        let hdr: XLogPageHeaderData = XLogPageHeaderData {
-            xlp_magic: buf.get_u16_le(),
-            xlp_info: buf.get_u16_le(),
-            xlp_tli: buf.get_u32_le(),
-            xlp_pageaddr: buf.get_u64_le(),
-            xlp_rem_len: buf.get_u32_le(),
-        };
-        // 4 bytes of padding, on 64-bit systems
-        buf.advance(4);
-
-        // FIXME: check that hdr.xlp_rem_len matches self.contlen
-        //println!("next xlog page (xlp_rem_len: {})", hdr.xlp_rem_len);
-
-        hdr
-    }
-
-    #[allow(non_snake_case)]
-    fn decode_XLogLongPageHeaderData(&mut self) -> XLogLongPageHeaderData {
-        let hdr: XLogLongPageHeaderData = XLogLongPageHeaderData {
-            std: self.decode_XLogPageHeaderData(),
-            xlp_sysid: self.inputbuf.get_u64_le(),
-            xlp_seg_size: self.inputbuf.get_u32_le(),
-            xlp_xlog_blcksz: self.inputbuf.get_u32_le(),
-        };
-
-        hdr
-    }
 }
 
 #[allow(dead_code)]
@@ -273,7 +239,7 @@ pub struct DecodedBkpBlock {
     /* Information on full-page image, if any */
     has_image: bool,       /* has image, even for consistency checking */
     pub apply_image: bool, /* has image that should be restored */
-    pub will_init: bool,
+    pub will_init: bool,   /* record doesn't need previous page version to apply */
     //char	   *bkp_image;
     hole_offset: u16,
     hole_length: u16,
@@ -309,10 +275,8 @@ impl DecodedBkpBlock {
     }
 }
 
-#[allow(non_upper_case_globals)]
-const SizeOfXLogRecord: u32 = 24;
-
 pub struct DecodedWALRecord {
+    pub xl_xid: TransactionId,
     pub xl_info: u8,
     pub xl_rmid: u8,
     pub record: Bytes, // raw XLogRecord
@@ -356,9 +320,7 @@ pub struct XlSmgrTruncate {
 }
 
 impl XlSmgrTruncate {
-    pub fn decode(decoded: &DecodedWALRecord) -> XlSmgrTruncate {
-        let mut buf = decoded.record.clone();
-        buf.advance((SizeOfXLogRecord + 2) as usize);
+    pub fn decode(buf: &mut Bytes) -> XlSmgrTruncate {
         XlSmgrTruncate {
             blkno: buf.get_u32_le(),
             rnode: RelFileNode {
@@ -381,9 +343,7 @@ pub struct XlCreateDatabase {
 }
 
 impl XlCreateDatabase {
-    pub fn decode(decoded: &DecodedWALRecord) -> XlCreateDatabase {
-        let mut buf = decoded.record.clone();
-        buf.advance((SizeOfXLogRecord + 2) as usize);
+    pub fn decode(buf: &mut Bytes) -> XlCreateDatabase {
         XlCreateDatabase {
             db_id: buf.get_u32_le(),
             tablespace_id: buf.get_u32_le(),
@@ -469,6 +429,121 @@ impl XlHeapUpdate {
     }
 }
 
+///
+/// Note: Parsing some fields is missing, because they're not needed.
+///
+/// This is similar to the xl_xact_parsed_commit and
+/// xl_xact_parsed_abort structs in PostgreSQL, but we use the same
+/// struct for commits and aborts.
+///
+#[derive(Debug)]
+pub struct XlXactParsedRecord {
+    pub xid: TransactionId,
+    pub info: u8,
+    pub xact_time: TimestampTz,
+    pub xinfo: u32,
+
+    pub db_id: Oid, /* MyDatabaseId */
+    pub ts_id: Oid, /* MyDatabaseTableSpace */
+
+    pub subxacts: Vec<TransactionId>,
+
+    pub xnodes: Vec<RelFileNode>,
+}
+
+impl XlXactParsedRecord {
+    /// Decode a XLOG_XACT_COMMIT/ABORT/COMMIT_PREPARED/ABORT_PREPARED
+    /// record. This should agree with the ParseCommitRecord and ParseAbortRecord
+    /// functions in PostgreSQL (in src/backend/access/rmgr/xactdesc.c)
+    pub fn decode(buf: &mut Bytes, mut xid: TransactionId, xl_info: u8) -> XlXactParsedRecord {
+        let info = xl_info & pg_constants::XLOG_XACT_OPMASK;
+        // The record starts with time of commit/abort
+        let xact_time = buf.get_i64_le();
+        let xinfo;
+        if xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
+            xinfo = buf.get_u32_le();
+        } else {
+            xinfo = 0;
+        }
+        let db_id;
+        let ts_id;
+        if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
+            db_id = buf.get_u32_le();
+            ts_id = buf.get_u32_le();
+        } else {
+            db_id = 0;
+            ts_id = 0;
+        }
+        let mut subxacts = Vec::<TransactionId>::new();
+        if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
+            let nsubxacts = buf.get_i32_le();
+            for _i in 0..nsubxacts {
+                let subxact = buf.get_u32_le();
+                subxacts.push(subxact);
+            }
+        }
+        let mut xnodes = Vec::<RelFileNode>::new();
+        if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
+            let nrels = buf.get_i32_le();
+            for _i in 0..nrels {
+                let spcnode = buf.get_u32_le();
+                let dbnode = buf.get_u32_le();
+                let relnode = buf.get_u32_le();
+                trace!(
+                    "XLOG_XACT_COMMIT relfilenode {}/{}/{}",
+                    spcnode,
+                    dbnode,
+                    relnode
+                );
+                xnodes.push(RelFileNode {
+                    spcnode,
+                    dbnode,
+                    relnode,
+                });
+            }
+        }
+        if xinfo & pg_constants::XACT_XINFO_HAS_INVALS != 0 {
+            let nmsgs = buf.get_i32_le();
+            for _i in 0..nmsgs {
+                let sizeof_shared_invalidation_message = 0;
+                buf.advance(sizeof_shared_invalidation_message);
+            }
+        }
+        if xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE != 0 {
+            xid = buf.get_u32_le();
+            trace!("XLOG_XACT_COMMIT-XACT_XINFO_HAS_TWOPHASE");
+        }
+        XlXactParsedRecord {
+            xid,
+            info,
+            xact_time,
+            xinfo,
+            db_id,
+            ts_id,
+            subxacts,
+            xnodes,
+        }
+    }
+}
+
+#[repr(C)]
+#[derive(Debug)]
+pub struct XlClogTruncate {
+    pub pageno: u32,
+    pub oldest_xid: TransactionId,
+    pub oldest_xid_db: Oid,
+}
+
+impl XlClogTruncate {
+    pub fn decode(buf: &mut Bytes) -> XlClogTruncate {
+        XlClogTruncate {
+            pageno: buf.get_u32_le(),
+            oldest_xid: buf.get_u32_le(),
+            oldest_xid_db: buf.get_u32_le(),
+        }
+    }
+}
+
 #[repr(C)]
 #[derive(Debug)]
 pub struct MultiXactMember {
@@ -515,14 +590,14 @@ impl XlMultiXactCreate {
 #[repr(C)]
 #[derive(Debug)]
 pub struct XlMultiXactTruncate {
-    oldest_multi_db: Oid,
+    pub oldest_multi_db: Oid,
     /* to-be-truncated range of multixact offsets */
-    start_trunc_off: MultiXactId, /* just for completeness' sake */
-    end_trunc_off: MultiXactId,
+    pub start_trunc_off: MultiXactId, /* just for completeness' sake */
+    pub end_trunc_off: MultiXactId,
 
     /* to-be-truncated range of multixact members */
-    start_trunc_memb: MultiXactOffset,
-    end_trunc_memb: MultiXactOffset,
+    pub start_trunc_memb: MultiXactOffset,
+    pub end_trunc_memb: MultiXactOffset,
 }
 
 impl XlMultiXactTruncate {
@@ -537,8 +612,7 @@ impl XlMultiXactTruncate {
     }
 }
 
-//
-// Routines to decode a WAL record and figure out which blocks are modified
+/// Main routine to decode a WAL record and figure out which blocks are modified
 //
 // See xlogrecord.h for details
 // The overall layout of an XLOG record is:
@@ -556,7 +630,7 @@ impl XlMultiXactTruncate {
 //      block data
 //      ...
 //      main data
-pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedWALRecord {
+pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
     let mut rnode_spcnode: u32 = 0;
     let mut rnode_dbnode: u32 = 0;
     let mut rnode_relnode: u32 = 0;
@@ -574,15 +648,10 @@ pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedW
         xlogrec.xl_rmid,
         xlogrec.xl_info
     );
-    if xlogrec.xl_xid > checkpoint.nextXid.value as u32 {
-        // TODO: handle XID wraparound
-        checkpoint.nextXid = FullTransactionId {
-            value: (checkpoint.nextXid.value & 0xFFFFFFFF00000000) | xlogrec.xl_xid as u64,
-        };
-    }
-    let remaining = xlogrec.xl_tot_len - SizeOfXLogRecord;
 
-    if buf.remaining() != remaining as usize {
+    let remaining: usize = xlogrec.xl_tot_len as usize - XLOG_SIZE_OF_XLOG_RECORD;
+
+    if buf.remaining() != remaining {
         //TODO error
     }
 
@@ -790,194 +859,10 @@ pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedW
         assert_eq!(buf.remaining(), main_data_len as usize);
     }
 
-    //5. Handle special CLOG and XACT records
-    if xlogrec.xl_rmid == pg_constants::RM_CLOG_ID {
-        let mut blk = DecodedBkpBlock::new();
-        blk.forknum = pg_constants::PG_XACT_FORKNUM;
-        blk.blkno = buf.get_i32_le() as u32;
-        blk.will_init = true;
-        trace!("RM_CLOG_ID updates block {}", blk.blkno);
-        blocks.push(blk);
-    } else if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
-        let info = xlogrec.xl_info & pg_constants::XLOG_XACT_OPMASK;
-        if info == pg_constants::XLOG_XACT_COMMIT || info == pg_constants::XLOG_XACT_COMMIT_PREPARED {
-			if info == pg_constants::XLOG_XACT_COMMIT {
-				let mut blk = DecodedBkpBlock::new();
-				blk.forknum = pg_constants::PG_XACT_FORKNUM;
-				blk.blkno = xlogrec.xl_xid / pg_constants::CLOG_XACTS_PER_PAGE;
-				trace!(
-					"XLOG_XACT_COMMIT xl_info {} xl_prev {:X}/{:X}  xid {} updates block {} main_data_len {}",
-					xlogrec.xl_info, (xlogrec.xl_prev >> 32),
-					xlogrec.xl_prev & 0xffffffff,
-					xlogrec.xl_xid,
-					blk.blkno,
-					main_data_len
-				);
-				blocks.push(blk);
-			}
-            //parse commit record to extract subtrans entries
-            // xl_xact_commit starts with time of commit
-            let _xact_time = buf.get_i64_le();
-
-            let mut xinfo = 0;
-            if xlogrec.xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
-                xinfo = buf.get_u32_le();
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
-                let _dbid = buf.get_u32_le();
-                let _tsid = buf.get_u32_le();
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
-                let nsubxacts = buf.get_i32_le();
-                let mut prev_blkno = u32::MAX;
-                for _i in 0..nsubxacts {
-                    let subxact = buf.get_u32_le();
-                    let blkno = subxact / pg_constants::CLOG_XACTS_PER_PAGE;
-                    if prev_blkno != blkno {
-                        prev_blkno = blkno;
-                        let mut blk = DecodedBkpBlock::new();
-                        blk.forknum = pg_constants::PG_XACT_FORKNUM;
-                        blk.blkno = blkno;
-                        blocks.push(blk);
-                    }
-                }
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
-                let nrels = buf.get_i32_le();
-                for _i in 0..nrels {
-                    let spcnode = buf.get_u32_le();
-                    let dbnode = buf.get_u32_le();
-                    let relnode = buf.get_u32_le();
-                    //TODO handle this too?
-                    trace!(
-                        "XLOG_XACT_COMMIT relfilenode {}/{}/{}",
-                        spcnode,
-                        dbnode,
-                        relnode
-                    );
-                }
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_INVALS != 0 {
-                let nmsgs = buf.get_i32_le();
-                for _i in 0..nmsgs {
-                    let sizeof_shared_invalidation_message = 0;
-                    buf.advance(sizeof_shared_invalidation_message);
-                }
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE != 0 {
-                let xid = buf.get_u32_le();
-				let mut blk = DecodedBkpBlock::new();
-				blk.forknum = pg_constants::PG_XACT_FORKNUM;
-				blk.blkno = xid / pg_constants::CLOG_XACTS_PER_PAGE;
-				blocks.push(blk);
-                trace!("XLOG_XACT_COMMIT-XACT_XINFO_HAS_TWOPHASE");
-                //TODO handle this to be able to restore pg_twophase on node start
-            }
-        } else if info == pg_constants::XLOG_XACT_ABORT || info == pg_constants::XLOG_XACT_ABORT_PREPARED {
-			if info == pg_constants::XLOG_XACT_ABORT {
-				let mut blk = DecodedBkpBlock::new();
-				blk.forknum = pg_constants::PG_XACT_FORKNUM;
-				blk.blkno = xlogrec.xl_xid / pg_constants::CLOG_XACTS_PER_PAGE;
-				trace!(
-					"XLOG_XACT_ABORT xl_info {} xl_prev {:X}/{:X} xid {} updates block {} main_data_len {}",
-					xlogrec.xl_info, (xlogrec.xl_prev >> 32),
-					xlogrec.xl_prev & 0xffffffff,
-					xlogrec.xl_xid,
-					blk.blkno,
-					main_data_len
-				);
-				blocks.push(blk);
-			}
-            //parse abort record to extract subtrans entries
-            // xl_xact_abort starts with time of commit
-            let _xact_time = buf.get_i64_le();
-
-            let mut xinfo = 0;
-            if xlogrec.xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
-                xinfo = buf.get_u32_le();
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
-                let _dbid = buf.get_u32_le();
-                let _tsid = buf.get_u32_le();
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
-                let nsubxacts = buf.get_i32_le();
-                let mut prev_blkno = u32::MAX;
-                for _i in 0..nsubxacts {
-                    let subxact = buf.get_u32_le();
-                    let blkno = subxact / pg_constants::CLOG_XACTS_PER_PAGE;
-                    if prev_blkno != blkno {
-                        prev_blkno = blkno;
-                        let mut blk = DecodedBkpBlock::new();
-                        blk.forknum = pg_constants::PG_XACT_FORKNUM;
-                        blk.blkno = blkno;
-                        blocks.push(blk);
-                    }
-                }
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
-                let nrels = buf.get_i32_le();
-                for _i in 0..nrels {
-                    let spcnode = buf.get_u32_le();
-                    let dbnode = buf.get_u32_le();
-                    let relnode = buf.get_u32_le();
-                    //TODO save these too
-                    trace!(
-                        "XLOG_XACT_ABORT relfilenode {}/{}/{}",
-                        spcnode,
-                        dbnode,
-                        relnode
-                    );
-                }
-            }
-            if xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE != 0 {
-                let xid = buf.get_u32_le();
-				let mut blk = DecodedBkpBlock::new();
-				blk.forknum = pg_constants::PG_XACT_FORKNUM;
-				blk.blkno = xid / pg_constants::CLOG_XACTS_PER_PAGE;
-				blocks.push(blk);
-                trace!("XLOG_XACT_ABORT-XACT_XINFO_HAS_TWOPHASE");
-            }
-        } else if info == pg_constants::XLOG_XACT_PREPARE {
-            let mut blk = DecodedBkpBlock::new();
-            blk.forknum = pg_constants::PG_TWOPHASE_FORKNUM;
-            blk.blkno = xlogrec.xl_xid;
-            blk.will_init = true;
-            blocks.push(blk);
-			info!("Prepare transaction {}", xlogrec.xl_xid);
-        }
-    } else if xlogrec.xl_rmid == pg_constants::RM_DBASE_ID {
-        let info = xlogrec.xl_info & !pg_constants::XLR_INFO_MASK;
-        if info == pg_constants::XLOG_DBASE_CREATE {
-            //buf points to main_data
-            let db_id = buf.get_u32_le();
-            let tablespace_id = buf.get_u32_le();
-            let src_db_id = buf.get_u32_le();
-            let src_tablespace_id = buf.get_u32_le();
-            trace!(
-                "XLOG_DBASE_CREATE tablespace_id/db_id {}/{} src_db_id {}/{}",
-                tablespace_id,
-                db_id,
-                src_tablespace_id,
-                src_db_id
-            );
-            // in postgres it is implemented as copydir
-            // we need to copy all pages in page_cache
-        } else {
-            trace!("XLOG_DBASE_DROP is not handled yet");
-        }
-    } else if xlogrec.xl_rmid == pg_constants::RM_TBLSPC_ID {
-        let info = xlogrec.xl_info & !pg_constants::XLR_INFO_MASK;
-        if info == pg_constants::XLOG_TBLSPC_CREATE {
-            //buf points to main_data
-            let ts_id = buf.get_u32_le();
-            let ts_path = str::from_utf8(&buf).unwrap();
-            trace!("XLOG_TBLSPC_CREATE ts_id {} ts_path {}", ts_id, ts_path);
-        } else {
-            trace!("XLOG_TBLSPC_DROP is not handled yet");
-        }
-    } else if xlogrec.xl_rmid == pg_constants::RM_HEAP_ID {
-        let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
+    // 5. Handle a few special record types that modify blocks without registering
+    // them with the standard mechanism.
+    if xlogrec.xl_rmid == pg_constants::RM_HEAP_ID {
+        let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
         let blkno = blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32;
         if info == pg_constants::XLOG_HEAP_INSERT {
             let xlrec = XlHeapInsert::decode(&mut buf);
@@ -1031,7 +916,7 @@ pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedW
             }
         }
     } else if xlogrec.xl_rmid == pg_constants::RM_HEAP2_ID {
-        let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
+        let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
         if info == pg_constants::XLOG_HEAP2_MULTI_INSERT {
             let xlrec = XlHeapMultiInsert::decode(&mut buf);
             if (xlrec.flags
@@ -1049,105 +934,10 @@ pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedW
                 blocks.push(blk);
             }
         }
-    } else if xlogrec.xl_rmid == pg_constants::RM_MULTIXACT_ID {
-        let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
-        if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE {
-            let mut blk = DecodedBkpBlock::new();
-            blk.forknum = pg_constants::PG_MXACT_OFFSETS_FORKNUM;
-            blk.blkno = buf.get_u32_le();
-            blk.will_init = true;
-            blocks.push(blk);
-        } else if info == pg_constants::XLOG_MULTIXACT_ZERO_MEM_PAGE {
-            let mut blk = DecodedBkpBlock::new();
-            blk.forknum = pg_constants::PG_MXACT_MEMBERS_FORKNUM;
-            blk.blkno = buf.get_u32_le();
-            blk.will_init = true;
-            blocks.push(blk);
-        } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
-            let xlrec = XlMultiXactCreate::decode(&mut buf);
-            // Update offset page
-            let mut blk = DecodedBkpBlock::new();
-            blk.blkno = xlrec.mid / pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32;
-            blk.forknum = pg_constants::PG_MXACT_OFFSETS_FORKNUM;
-            blocks.push(blk);
-            let first_mbr_blkno = xlrec.moff / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-            let last_mbr_blkno =
-                (xlrec.moff + xlrec.nmembers - 1) / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-            for blkno in first_mbr_blkno..=last_mbr_blkno {
-                // Update members page
-                let mut blk = DecodedBkpBlock::new();
-                blk.forknum = pg_constants::PG_MXACT_MEMBERS_FORKNUM;
-                blk.blkno = blkno;
-                blocks.push(blk);
-            }
-            if xlrec.mid > checkpoint.nextMulti {
-                checkpoint.nextMulti = xlrec.mid;
-            }
-            if xlrec.moff > checkpoint.nextMultiOffset {
-                checkpoint.nextMultiOffset = xlrec.moff;
-            }
-            let max_xid = xlrec
-                .members
-                .iter()
-                .fold(checkpoint.nextXid.value as u32, |acc, mbr| {
-                    if mbr.xid > acc {
-                        mbr.xid
-                    } else {
-                        acc
-                    }
-                });
-            checkpoint.nextXid = FullTransactionId {
-                value: (checkpoint.nextXid.value & 0xFFFFFFFF00000000) | max_xid as u64,
-            };
-        } else if info == pg_constants::XLOG_MULTIXACT_TRUNCATE_ID {
-            let xlrec = XlMultiXactTruncate::decode(&mut buf);
-            checkpoint.oldestXid = xlrec.end_trunc_off;
-            checkpoint.oldestMultiDB = xlrec.oldest_multi_db;
-            let first_off_blkno =
-                xlrec.start_trunc_off / pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32;
-            let last_off_blkno =
-                xlrec.end_trunc_off / pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32;
-            for blkno in first_off_blkno..last_off_blkno {
-                let mut blk = DecodedBkpBlock::new();
-                blk.forknum = pg_constants::PG_MXACT_OFFSETS_FORKNUM;
-                blk.blkno = blkno;
-                blk.will_init = true;
-                blocks.push(blk);
-            }
-            let first_mbr_blkno =
-                xlrec.start_trunc_memb / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-            let last_mbr_blkno =
-                xlrec.end_trunc_memb / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-            for blkno in first_mbr_blkno..last_mbr_blkno {
-                let mut blk = DecodedBkpBlock::new();
-                blk.forknum = pg_constants::PG_MXACT_MEMBERS_FORKNUM;
-                blk.blkno = blkno;
-                blk.will_init = true;
-                blocks.push(blk);
-            }
-        } else {
-            assert!(false);
-        }
-    } else if xlogrec.xl_rmid == pg_constants::RM_RELMAP_ID {
-        let xlrec = XlRelmapUpdate::decode(&mut buf);
-        let mut blk = DecodedBkpBlock::new();
-        blk.forknum = pg_constants::PG_FILENODEMAP_FORKNUM;
-        blk.rnode_spcnode = xlrec.tsid;
-        blk.rnode_dbnode = xlrec.dbid;
-        blk.rnode_relnode = 0;
-        blk.will_init = true;
-        blocks.push(blk);
-    } else if xlogrec.xl_rmid == pg_constants::RM_XLOG_ID {
-        let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
-        if info == pg_constants::XLOG_NEXTOID {
-            let next_oid = buf.get_u32_le();
-            if next_oid > checkpoint.nextOid {
-                checkpoint.nextOid = next_oid;
-            }
-        }
     }
 
     DecodedWALRecord {
+        xl_xid: xlogrec.xl_xid,
         xl_info: xlogrec.xl_info,
         xl_rmid: xlogrec.xl_rmid,
         record,

pageserver/src/walreceiver.rs

@@ -1,17 +1,15 @@
 //!
-//! WAL receiver
-//!
-//! The WAL receiver connects to the WAL safekeeper service, and streams WAL.
-//! For each WAL record, it decodes the record to figure out which data blocks
-//! the record affects, and adds the records to the page cache.
+//! WAL receiver connects to the WAL safekeeper service,
+//! streams WAL, decodes records and saves them in page cache.
 //!
+//! We keep one WAL receiver active per timeline.
 
 use crate::page_cache;
-use crate::repository::*;
+use crate::relish::*;
+use crate::restore_local_repo;
 use crate::waldecoder::*;
-use crate::PageServerConf;
-use crate::ZTimelineId;
-use anyhow::Error;
+use crate::{PageServerConf, RepositoryFormat};
+use anyhow::{Error, Result};
 use lazy_static::lazy_static;
 use log::*;
 use postgres::fallible_iterator::FallibleIterator;
@@ -21,17 +19,17 @@ use postgres_ffi::xlog_utils::*;
 use postgres_ffi::*;
 use postgres_protocol::message::backend::ReplicationMessage;
 use postgres_types::PgLsn;
+use std::cmp::{max, min};
 use std::collections::HashMap;
 use std::fs;
-use std::fs::{File, OpenOptions};
-use std::io::{Seek, SeekFrom, Write};
-use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Mutex;
 use std::thread;
 use std::thread::sleep;
 use std::time::{Duration, SystemTime};
 use zenith_utils::lsn::Lsn;
+use zenith_utils::zid::ZTenantId;
+use zenith_utils::zid::ZTimelineId;
 
 //
 // We keep one WAL Receiver active per timeline.
@@ -50,6 +48,7 @@ pub fn launch_wal_receiver(
     conf: &'static PageServerConf,
     timelineid: ZTimelineId,
     wal_producer_connstr: &str,
+    tenantid: ZTenantId,
 ) {
     let mut receivers = WAL_RECEIVERS.lock().unwrap();
 
@@ -67,7 +66,7 @@ pub fn launch_wal_receiver(
             let _walreceiver_thread = thread::Builder::new()
                 .name("WAL receiver thread".into())
                 .spawn(move || {
-                    thread_main(conf, timelineid);
+                    thread_main(conf, timelineid, &tenantid);
                 })
                 .unwrap();
         }
@@ -88,7 +87,7 @@ fn get_wal_producer_connstr(timelineid: ZTimelineId) -> String {
 //
 // This is the entry point for the WAL receiver thread.
 //
-fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId) {
+fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId, tenantid: &ZTenantId) {
     info!(
         "WAL receiver thread started for timeline : '{}'",
         timelineid
@@ -102,7 +101,7 @@ fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId) {
         // Look up the current WAL producer address
         let wal_producer_connstr = get_wal_producer_connstr(timelineid);
 
-        let res = walreceiver_main(conf, timelineid, &wal_producer_connstr);
+        let res = walreceiver_main(conf, timelineid, &wal_producer_connstr, tenantid);
 
         if let Err(e) = res {
             info!(
@@ -115,23 +114,36 @@ fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId) {
 }
 
 fn walreceiver_main(
-    _conf: &PageServerConf,
+    conf: &PageServerConf,
     timelineid: ZTimelineId,
     wal_producer_connstr: &str,
+    tenantid: &ZTenantId,
 ) -> Result<(), Error> {
     // Connect to the database in replication mode.
     info!("connecting to {:?}", wal_producer_connstr);
-    let connect_cfg = format!("{} replication=true", wal_producer_connstr);
+    let connect_cfg = format!(
+        "{} application_name=pageserver replication=true",
+        wal_producer_connstr
+    );
 
     let mut rclient = Client::connect(&connect_cfg, NoTls)?;
     info!("connected!");
 
+    // Immediately increment the gauge, then create a job to decrement it on thread exit.
+    // One of the pros of `defer!` is that this will *most probably*
+    // get called, even in presence of panics.
+    let gauge = crate::LIVE_CONNECTIONS_COUNT.with_label_values(&["wal_receiver"]);
+    gauge.inc();
+    scopeguard::defer! {
+        gauge.dec();
+    }
+
     let identify = identify_system(&mut rclient)?;
     info!("{:?}", identify);
     let end_of_wal = Lsn::from(u64::from(identify.xlogpos));
     let mut caught_up = false;
 
-    let repository = page_cache::get_repository();
+    let repository = page_cache::get_repository_for_tenant(tenantid)?;
     let timeline = repository.get_timeline(timelineid).unwrap();
 
     //
@@ -140,18 +152,13 @@ fn walreceiver_main(
     // If we had previously received WAL up to some point in the middle of a WAL record, we
     // better start from the end of last full WAL record, not in the middle of one. Hence,
     // use 'last_record_lsn' rather than 'last_valid_lsn' here.
-    let last_rec_lsn = timeline.get_last_record_lsn();
+    let mut last_rec_lsn = timeline.get_last_record_lsn();
     let mut startpoint = last_rec_lsn;
 
     if startpoint == Lsn(0) {
         error!("No previous WAL position");
     }
 
-    startpoint = Lsn::max(
-        startpoint,
-        Lsn(end_of_wal.0 & !(pg_constants::WAL_SEGMENT_SIZE as u64 - 1)),
-    );
-
     // There might be some padding after the last full record, skip it.
     //
     // FIXME: It probably would be better to always start streaming from the beginning
@@ -169,45 +176,67 @@ fn walreceiver_main(
     let copy_stream = rclient.copy_both_simple(&query)?;
     let mut physical_stream = ReplicationIter::new(copy_stream);
 
-    let mut waldecoder = WalStreamDecoder::new(startpoint);
+    let mut waldecoder = WalStreamDecoder::new(startpoint, true);
+
+    let checkpoint_bytes = timeline.get_page_at_lsn_nowait(RelishTag::Checkpoint, 0, startpoint)?;
+    let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
+    trace!("CheckPoint.nextXid = {}", checkpoint.nextXid.value);
 
-    let mut checkpoint = CheckPoint::new(startpoint.0, identify.timeline);
-    let checkpoint_tag = BufferTag::fork(pg_constants::PG_CHECKPOINT_FORKNUM);
-    if let Some(checkpoint_bytes) = timeline.get_page_image(checkpoint_tag, Lsn(0))? {
-        checkpoint = decode_checkpoint(checkpoint_bytes)?;
-        trace!("CheckPoint.nextXid = {}", checkpoint.nextXid.value);
-    } else {
-        error!("No checkpoint record was found in reposistory");
-    }
     while let Some(replication_message) = physical_stream.next()? {
-        match replication_message {
+        let status_update = match replication_message {
             ReplicationMessage::XLogData(xlog_data) => {
                 // Pass the WAL data to the decoder, and see if we can decode
                 // more records as a result.
                 let data = xlog_data.data();
                 let startlsn = Lsn::from(xlog_data.wal_start());
                 let endlsn = startlsn + data.len() as u64;
-
-                write_wal_file(startlsn, timelineid, pg_constants::WAL_SEGMENT_SIZE, data)?;
+                let prev_last_rec_lsn = last_rec_lsn;
 
                 trace!("received XLogData between {} and {}", startlsn, endlsn);
-
                 waldecoder.feed_bytes(data);
 
-                while let Some((lsn, recdata)) = waldecoder.poll_decode()? {
-                    let old_checkpoint_bytes = encode_checkpoint(checkpoint);
-                    let decoded = decode_wal_record(&mut checkpoint, recdata.clone());
-                    timeline.save_decoded_record(decoded, recdata, lsn)?;
+                loop {
+                    match waldecoder.poll_decode() {
+                        Ok(Some((lsn, recdata))) => {
+                            // Save old checkpoint value to compare with it after decoding WAL record
+                            let old_checkpoint_bytes = checkpoint.encode();
+                            let decoded = decode_wal_record(recdata.clone());
+                            restore_local_repo::save_decoded_record(
+                                &mut checkpoint,
+                                &*timeline,
+                                &decoded,
+                                recdata,
+                                lsn,
+                            )?;
+                            last_rec_lsn = lsn;
 
-                    let new_checkpoint_bytes = encode_checkpoint(checkpoint);
-                    if new_checkpoint_bytes != old_checkpoint_bytes {
-                        timeline.put_page_image(checkpoint_tag, Lsn(0), new_checkpoint_bytes);
+                            let new_checkpoint_bytes = checkpoint.encode();
+                            // Check if checkpoint data was updated by save_decoded_record
+                            if new_checkpoint_bytes != old_checkpoint_bytes {
+                                timeline.put_page_image(
+                                    RelishTag::Checkpoint,
+                                    0,
+                                    lsn,
+                                    new_checkpoint_bytes,
+                                    false,
+                                )?;
+                            }
+                        }
+                        Ok(None) => {
+                            trace!(
+                                "End of replication stream {}..{} at {}",
+                                startlsn,
+                                endlsn,
+                                last_rec_lsn
+                            );
+                            break;
+                        }
+                        Err(e) => {
+                            info!("Decode error {}", e);
+                            return Err(e.into());
+                        }
                     }
-                    // Now that this record has been handled, let the page cache know that
-                    // it is up-to-date to this LSN
-                    timeline.advance_last_record_lsn(lsn);
                 }
-
                 // Update the last_valid LSN value in the page cache one more time. We updated
                 // it in the loop above, between each WAL record, but we might have received
                 // a partial record after the last completed record. Our page cache's value
@@ -216,16 +245,52 @@ fn walreceiver_main(
                 // flush ptr.
                 timeline.advance_last_valid_lsn(endlsn);
 
+                // Somewhat arbitrarily, if we have at least 10 complete wal segments (16 MB each),
+                // "checkpoint" the repository to flush all the changes from WAL we've processed
+                // so far to disk. After this, we don't need the original WAL anymore, and it
+                // can be removed. This is probably too aggressive for production, but it's useful
+                // to expose bugs now.
+                //
+                // TODO: We don't actually dare to remove the WAL. It's useful for debugging,
+                // and we might it for logical decoding other things in the future. Although
+                // we should also be able to fetch it back from the WAL safekeepers or S3 if
+                // needed.
+                if prev_last_rec_lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE)
+                    != last_rec_lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE)
+                {
+                    info!("switched segment {} to {}", prev_last_rec_lsn, last_rec_lsn);
+                    let (oldest_segno, newest_segno) = find_wal_file_range(
+                        conf,
+                        &timelineid,
+                        pg_constants::WAL_SEGMENT_SIZE,
+                        last_rec_lsn,
+                        tenantid,
+                    )?;
+
+                    if newest_segno - oldest_segno >= 10 {
+                        // FIXME: The layered repository performs checkpointing in a separate thread, so this
+                        // isn't needed anymore. Remove 'checkpoint' from the Timeline trait altogether?
+                        if conf.repository_format == RepositoryFormat::RocksDb {
+                            timeline.checkpoint()?;
+                        }
+
+                        // TODO: This is where we could remove WAL older than last_rec_lsn.
+                        //remove_wal_files(timelineid, pg_constants::WAL_SEGMENT_SIZE, last_rec_lsn)?;
+                    }
+                }
+
                 if !caught_up && endlsn >= end_of_wal {
                     info!("caught up at LSN {}", endlsn);
                     caught_up = true;
                 }
+
+                Some(endlsn)
             }
 
             ReplicationMessage::PrimaryKeepAlive(keepalive) => {
                 let wal_end = keepalive.wal_end();
                 let timestamp = keepalive.timestamp();
-                let reply_requested: bool = keepalive.reply() != 0;
+                let reply_requested = keepalive.reply() != 0;
 
                 trace!(
                     "received PrimaryKeepAlive(wal_end: {}, timestamp: {:?} reply: {})",
@@ -233,25 +298,73 @@ fn walreceiver_main(
                     timestamp,
                     reply_requested,
                 );
-                if reply_requested {
-                    // TODO: More thought should go into what values are sent here.
-                    let last_lsn = PgLsn::from(u64::from(timeline.get_last_valid_lsn()));
-                    let write_lsn = last_lsn;
-                    let flush_lsn = last_lsn;
-                    let apply_lsn = PgLsn::from(0);
-                    let ts = SystemTime::now();
-                    const NO_REPLY: u8 = 0u8;
 
-                    physical_stream
-                        .standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY)?;
+                if reply_requested {
+                    Some(timeline.get_last_valid_lsn())
+                } else {
+                    None
                 }
             }
-            _ => (),
+
+            _ => None,
+        };
+
+        if let Some(last_lsn) = status_update {
+            // TODO: More thought should go into what values are sent here.
+            let last_lsn = PgLsn::from(u64::from(last_lsn));
+            let write_lsn = last_lsn;
+            let flush_lsn = last_lsn;
+            let apply_lsn = PgLsn::from(0);
+            let ts = SystemTime::now();
+            const NO_REPLY: u8 = 0;
+
+            physical_stream.standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY)?;
         }
     }
     Ok(())
 }
 
+fn find_wal_file_range(
+    conf: &PageServerConf,
+    timeline: &ZTimelineId,
+    wal_seg_size: usize,
+    written_upto: Lsn,
+    tenant: &ZTenantId,
+) -> Result<(u64, u64)> {
+    let written_upto_segno = written_upto.segment_number(wal_seg_size);
+
+    let mut oldest_segno = written_upto_segno;
+    let mut newest_segno = written_upto_segno;
+    // Scan the wal directory, and count how many WAL filed we could remove
+    let wal_dir = conf.wal_dir_path(timeline, tenant);
+    for entry in fs::read_dir(wal_dir)? {
+        let entry = entry?;
+        let path = entry.path();
+
+        if path.is_dir() {
+            continue;
+        }
+
+        let filename = path.file_name().unwrap().to_str().unwrap();
+
+        if IsXLogFileName(filename) {
+            let (segno, _tli) = XLogFromFileName(filename, wal_seg_size);
+
+            if segno > written_upto_segno {
+                // that's strange.
+                warn!("there is a WAL file from future at {}", path.display());
+                continue;
+            }
+
+            oldest_segno = min(oldest_segno, segno);
+            newest_segno = max(newest_segno, segno);
+        }
+    }
+    // FIXME: would be good to assert that there are no gaps in the WAL files
+
+    Ok((oldest_segno, newest_segno))
+}
+
 /// Data returned from the postgres `IDENTIFY_SYSTEM` command
 ///
 /// See the [postgres docs] for more details.
@@ -298,96 +411,3 @@ pub fn identify_system(client: &mut Client) -> Result<IdentifySystem, Error> {
         Err(IdentifyError.into())
     }
 }
-
-fn write_wal_file(
-    startpos: Lsn,
-    timeline: ZTimelineId,
-    wal_seg_size: usize,
-    buf: &[u8],
-) -> anyhow::Result<()> {
-    let mut bytes_left: usize = buf.len();
-    let mut bytes_written: usize = 0;
-    let mut partial;
-    let mut start_pos = startpos;
-    const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ];
-
-    let wal_dir = PathBuf::from(format!("timelines/{}/wal", timeline));
-
-    /* Extract WAL location for this block */
-    let mut xlogoff = start_pos.segment_offset(wal_seg_size);
-
-    while bytes_left != 0 {
-        let bytes_to_write;
-
-        /*
-         * If crossing a WAL boundary, only write up until we reach wal
-         * segment size.
-         */
-        if xlogoff + bytes_left > wal_seg_size {
-            bytes_to_write = wal_seg_size - xlogoff;
-        } else {
-            bytes_to_write = bytes_left;
-        }
-
-        /* Open file */
-        let segno = start_pos.segment_number(wal_seg_size);
-        let wal_file_name = XLogFileName(
-            1, // FIXME: always use Postgres timeline 1
-            segno,
-            wal_seg_size,
-        );
-        let wal_file_path = wal_dir.join(wal_file_name.clone());
-        let wal_file_partial_path = wal_dir.join(wal_file_name.clone() + ".partial");
-
-        {
-            let mut wal_file: File;
-            /* Try to open already completed segment */
-            if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_path) {
-                wal_file = file;
-                partial = false;
-            } else if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_partial_path) {
-                /* Try to open existed partial file */
-                wal_file = file;
-                partial = true;
-            } else {
-                /* Create and fill new partial file */
-                partial = true;
-                match OpenOptions::new()
-                    .create(true)
-                    .write(true)
-                    .open(&wal_file_partial_path)
-                {
-                    Ok(mut file) => {
-                        for _ in 0..(wal_seg_size / XLOG_BLCKSZ) {
-                            file.write_all(&ZERO_BLOCK)?;
-                        }
-                        wal_file = file;
-                    }
-                    Err(e) => {
-                        error!("Failed to open log file {:?}: {}", &wal_file_path, e);
-                        return Err(e.into());
-                    }
-                }
-            }
-            wal_file.seek(SeekFrom::Start(xlogoff as u64))?;
-            wal_file.write_all(&buf[bytes_written..(bytes_written + bytes_to_write)])?;
-
-            // FIXME: Flush the file
-            //wal_file.sync_all()?;
-        }
-        /* Write was successful, advance our position */
-        bytes_written += bytes_to_write;
-        bytes_left -= bytes_to_write;
-        start_pos += bytes_to_write as u64;
-        xlogoff += bytes_to_write;
-
-        /* Did we reach the end of a WAL segment? */
-        if start_pos.segment_offset(wal_seg_size) == 0 {
-            xlogoff = 0;
-            if partial {
-                fs::rename(&wal_file_partial_path, &wal_file_path)?;
-            }
-        }
-    }
-    Ok(())
-}

pageserver/src/walredo.rs

@@ -1,9 +1,11 @@
 //!
-//! WAL redo
+//! WAL redo. This service runs PostgreSQL in a special wal_redo mode
+//! to apply given WAL records over an old page image and return new
+//! page image.
 //!
 //! We rely on Postgres to perform WAL redo for us. We launch a
 //! postgres process in special "wal redo" mode that's similar to
-//! single-user mode. We then pass the the previous page image, if any,
+//! single-user mode. We then pass the previous page image, if any,
 //! and all the WAL records we want to apply, to the postgres
 //! process. Then we get the page image back. Communication with the
 //! postgres process happens via stdin/stdout
@@ -11,21 +13,23 @@
 //! See src/backend/tcop/zenith_wal_redo.c for the other side of
 //! this communication.
 //!
-//! TODO: Even though the postgres code runs in a separate process,
-//! it's not a secure sandbox.
+//! The Postgres process is assumed to be secure against malicious WAL
+//! records. It achieves it by dropping privileges before replaying
+//! any WAL records, so that even if an attacker hijacks the Postgres
+//! process, he cannot escape out of it.
 //!
 use byteorder::{ByteOrder, LittleEndian};
 use bytes::{Buf, BufMut, Bytes, BytesMut};
+use lazy_static::lazy_static;
 use log::*;
-use std::assert;
+use serde::{Deserialize, Serialize};
 use std::cell::RefCell;
 use std::fs;
 use std::fs::OpenOptions;
 use std::io::prelude::*;
 use std::io::Error;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::process::Stdio;
-use std::sync::mpsc;
 use std::sync::Mutex;
 use std::time::Duration;
 use std::time::Instant;
@@ -33,15 +37,31 @@ use tokio::io::AsyncBufReadExt;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::process::{ChildStdin, ChildStdout, Command};
 use tokio::time::timeout;
+use zenith_metrics::{register_histogram, register_int_counter, Histogram, IntCounter};
+use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;
+use zenith_utils::zid::ZTenantId;
 
-use crate::repository::BufferTag;
+use crate::relish::*;
 use crate::repository::WALRecord;
+use crate::waldecoder::XlXactParsedRecord;
 use crate::waldecoder::{MultiXactId, XlMultiXactCreate};
 use crate::PageServerConf;
 use postgres_ffi::nonrelfile_utils::transaction_id_set_status;
 use postgres_ffi::pg_constants;
-use postgres_ffi::xlog_utils::XLogRecord;
+use postgres_ffi::XLogRecord;
+
+///
+/// `RelTag` + block number (`blknum`) gives us a unique id of the page in the cluster.
+///
+/// In Postgres `BufferTag` structure is used for exactly the same purpose.
+/// [See more related comments here](https://github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/buf_internals.h#L91).
+///
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Serialize, Deserialize)]
+pub struct BufferTag {
+    pub rel: RelTag,
+    pub blknum: u32,
+}
 
 ///
 /// WAL Redo Manager is responsible for replaying WAL records.
@@ -56,40 +76,79 @@ pub trait WalRedoManager: Send + Sync {
     /// the reords.
     fn request_redo(
         &self,
-        tag: BufferTag,
+        rel: RelishTag,
+        blknum: u32,
         lsn: Lsn,
         base_img: Option<Bytes>,
         records: Vec<WALRecord>,
     ) -> Result<Bytes, WalRedoError>;
 }
 
-static TIMEOUT: Duration = Duration::from_secs(20);
-
 ///
-/// The implementation consists of two parts: PostgresRedoManager, and
-/// PostgresRedoManagerInternal. PostgresRedoManager is the public struct
-/// that can be used to send redo requests to the manager.
-/// PostgresRedoManagerInternal is used by the manager thread itself.
+/// A dummy WAL Redo Manager implementation that doesn't allow replaying
+/// anything. Currently used during bootstrapping (zenith init), to create
+/// a Repository object without launching the real WAL redo process.
 ///
-pub struct PostgresRedoManager {
-    request_tx: Mutex<mpsc::Sender<WalRedoRequest>>,
+pub struct DummyRedoManager {}
+impl crate::walredo::WalRedoManager for DummyRedoManager {
+    fn request_redo(
+        &self,
+        _rel: RelishTag,
+        _blknum: u32,
+        _lsn: Lsn,
+        _base_img: Option<Bytes>,
+        _records: Vec<WALRecord>,
+    ) -> Result<Bytes, WalRedoError> {
+        Err(WalRedoError::InvalidState)
+    }
 }
 
-struct PostgresRedoManagerInternal {
+static TIMEOUT: Duration = Duration::from_secs(20);
+
+// Metrics collected on WAL redo operations
+//
+// We collect the time spent in actual WAL redo ('redo'), and time waiting
+// for access to the postgres process ('wait') since there is only one for
+// each tenant.
+lazy_static! {
+    static ref WAL_REDO_TIME: Histogram =
+        register_histogram!("pageserver_wal_redo_time", "Time spent on WAL redo")
+            .expect("failed to define a metric");
+    static ref WAL_REDO_WAIT_TIME: Histogram = register_histogram!(
+        "pageserver_wal_redo_wait_time",
+        "Time spent waiting for access to the WAL redo process"
+    )
+    .expect("failed to define a metric");
+    static ref WAL_REDO_RECORD_COUNTER: IntCounter = register_int_counter!(
+        "pageserver_wal_records_replayed",
+        "Number of WAL records replayed"
+    )
+    .unwrap();
+}
+
+///
+/// This is the real implementation that uses a Postgres process to
+/// perform WAL replay. Only one thread can use the processs at a time,
+/// that is controlled by the Mutex. In the future, we might want to
+/// launch a pool of processes to allow concurrent replay of multiple
+/// records.
+///
+pub struct PostgresRedoManager {
+    tenantid: ZTenantId,
     conf: &'static PageServerConf,
 
-    request_rx: mpsc::Receiver<WalRedoRequest>,
+    runtime: tokio::runtime::Runtime,
+    process: Mutex<Option<PostgresRedoProcess>>,
 }
 
 #[derive(Debug)]
 struct WalRedoRequest {
-    tag: BufferTag,
+    rel: RelishTag,
+    blknum: u32,
     lsn: Lsn,
 
     base_img: Option<Bytes>,
     records: Vec<WALRecord>,
-
-    response_channel: mpsc::Sender<Result<Bytes, WalRedoError>>,
 }
 
 /// An error happened in WAL redo
@@ -97,42 +156,14 @@ struct WalRedoRequest {
 pub enum WalRedoError {
     #[error(transparent)]
     IoError(#[from] std::io::Error),
+
+    #[error("cannot perform WAL redo now")]
+    InvalidState,
 }
 
 ///
 /// Public interface of WAL redo manager
 ///
-impl PostgresRedoManager {
-    ///
-    /// Create a new PostgresRedoManager.
-    ///
-    /// This launches a new thread to handle the requests.
-    pub fn new(conf: &'static PageServerConf) -> PostgresRedoManager {
-        let (tx, rx) = mpsc::channel();
-
-        //
-        // Launch the WAL redo thread
-        //
-        // Get mutable references to the values that we need to pass to the
-        // thread.
-        let request_rx = rx;
-
-        // Currently, the join handle is not saved anywhere and we
-        // won't try restart the thread if it dies.
-        let _walredo_thread = std::thread::Builder::new()
-            .name("WAL redo thread".into())
-            .spawn(move || {
-                let mut internal = PostgresRedoManagerInternal { conf, request_rx };
-                internal.wal_redo_main();
-            })
-            .unwrap();
-
-        PostgresRedoManager {
-            request_tx: Mutex::new(tx),
-        }
-    }
-}
-
 impl WalRedoManager for PostgresRedoManager {
     ///
     /// Request the WAL redo manager to apply some WAL records
@@ -142,62 +173,73 @@ impl WalRedoManager for PostgresRedoManager {
     ///
     fn request_redo(
         &self,
-        tag: BufferTag,
+        rel: RelishTag,
+        blknum: u32,
         lsn: Lsn,
         base_img: Option<Bytes>,
         records: Vec<WALRecord>,
     ) -> Result<Bytes, WalRedoError> {
-        // Create a channel where to receive the response
-        let (tx, rx) = mpsc::channel::<Result<Bytes, WalRedoError>>();
+        let start_time;
+        let lock_time;
+        let end_time;
 
         let request = WalRedoRequest {
-            tag,
+            rel,
+            blknum,
             lsn,
             base_img,
             records,
-            response_channel: tx,
         };
 
-        self.request_tx
-            .lock()
-            .unwrap()
-            .send(request)
-            .expect("could not send WAL redo request");
+        start_time = Instant::now();
+        let result = {
+            let mut process_guard = self.process.lock().unwrap();
+            lock_time = Instant::now();
 
-        rx.recv()
-            .expect("could not receive response to WAL redo request")
+            // launch the WAL redo process on first use
+            if process_guard.is_none() {
+                let p = self
+                    .runtime
+                    .block_on(PostgresRedoProcess::launch(self.conf, &self.tenantid))?;
+                *process_guard = Some(p);
+            }
+            let process = (*process_guard).as_ref().unwrap();
+
+            self.runtime
+                .block_on(self.handle_apply_request(&process, &request))
+        };
+        end_time = Instant::now();
+
+        WAL_REDO_WAIT_TIME.observe(lock_time.duration_since(start_time).as_secs_f64());
+        WAL_REDO_TIME.observe(end_time.duration_since(lock_time).as_secs_f64());
+
+        result
     }
 }
 
 fn mx_offset_to_flags_offset(xid: MultiXactId) -> usize {
-    return ((xid / pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP as u32) as u16
+    ((xid / pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP as u32) as u16
         % pg_constants::MULTIXACT_MEMBERGROUPS_PER_PAGE
-        * pg_constants::MULTIXACT_MEMBERGROUP_SIZE) as usize;
+        * pg_constants::MULTIXACT_MEMBERGROUP_SIZE) as usize
 }
 
 fn mx_offset_to_flags_bitshift(xid: MultiXactId) -> u16 {
-    return (xid as u16) % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP
-        * pg_constants::MXACT_MEMBER_BITS_PER_XACT;
+    (xid as u16) % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP
+        * pg_constants::MXACT_MEMBER_BITS_PER_XACT
 }
 
 /* Location (byte offset within page) of TransactionId of given member */
 fn mx_offset_to_member_offset(xid: MultiXactId) -> usize {
-    return mx_offset_to_flags_offset(xid)
+    mx_offset_to_flags_offset(xid)
         + (pg_constants::MULTIXACT_FLAGBYTES_PER_GROUP
-            + (xid as u16 % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP) * 4)
-            as usize;
+            + (xid as u16 % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP) * 4) as usize
 }
 
-///
-/// WAL redo thread
-///
-impl PostgresRedoManagerInternal {
-    //
-    // Main entry point for the WAL applicator thread.
-    //
-    fn wal_redo_main(&mut self) {
-        info!("WAL redo thread started");
-
+impl PostgresRedoManager {
+    ///
+    /// Create a new PostgresRedoManager.
+    ///
+    pub fn new(conf: &'static PageServerConf, tenantid: ZTenantId) -> PostgresRedoManager {
         // We block on waiting for requests on the walredo request channel, but
         // use async I/O to communicate with the child process. Initialize the
         // runtime for the async part.
@@ -206,35 +248,12 @@ impl PostgresRedoManagerInternal {
             .build()
             .unwrap();
 
-        let process: PostgresRedoProcess;
-
-        // FIXME: We need a dummy Postgres cluster to run the process in. Currently, we
-        // just create one with constant name. That fails if you try to launch more than
-        // one WAL redo manager concurrently.
-        let datadir = self.conf.workdir.join("wal-redo-datadir");
-
-        info!("launching WAL redo postgres process");
-
-        process = runtime
-            .block_on(PostgresRedoProcess::launch(&datadir))
-            .unwrap();
-
-        // Loop forever, handling requests as they come.
-        loop {
-            let request = self
-                .request_rx
-                .recv()
-                .expect("WAL redo request channel was closed");
-
-            let result = runtime.block_on(self.handle_apply_request(&process, &request));
-            let result_ok = result.is_ok();
-
-            // Send the result to the requester
-            let _ = request.response_channel.send(result);
-
-            if !result_ok {
-                error!("wal-redo-postgres failed to apply request {:?}", request);
-            }
+        // The actual process is launched lazily, on first request.
+        PostgresRedoManager {
+            runtime,
+            tenantid,
+            conf,
+            process: Mutex::new(None),
         }
     }
 
@@ -246,7 +265,8 @@ impl PostgresRedoManagerInternal {
         process: &PostgresRedoProcess,
         request: &WalRedoRequest,
     ) -> Result<Bytes, WalRedoError> {
-        let tag = request.tag;
+        let rel = request.rel;
+        let blknum = request.blknum;
         let lsn = request.lsn;
         let base_img = request.base_img.clone();
         let records = &request.records;
@@ -256,17 +276,28 @@ impl PostgresRedoManagerInternal {
         let start = Instant::now();
 
         let apply_result: Result<Bytes, Error>;
-        if tag.rel.forknum > pg_constants::INIT_FORKNUM {
+        if let RelishTag::Relation(rel) = rel {
+            // Relational WAL records are applied using wal-redo-postgres
+            let buf_tag = BufferTag { rel, blknum };
+            apply_result = process.apply_wal_records(buf_tag, base_img, records).await;
+        } else {
+            // Non-relational WAL records are handled here, with custom code that has the
+            // same effects as the corresponding Postgres WAL redo function.
             const ZERO_PAGE: [u8; 8192] = [0u8; 8192];
             let mut page = BytesMut::new();
             if let Some(fpi) = base_img {
+                // If full-page image is provided, then use it...
                 page.extend_from_slice(&fpi[..]);
             } else {
+                // otherwise initialize page with zeros
                 page.extend_from_slice(&ZERO_PAGE);
             }
+            // Apply all collected WAL records
             for record in records {
                 let mut buf = record.rec.clone();
 
+                WAL_REDO_RECORD_COUNTER.inc();
+
                 // 1. Parse XLogRecord struct
                 // FIXME: refactor to avoid code duplication.
                 let xlogrec = XLogRecord::from_bytes(&mut buf);
@@ -279,180 +310,116 @@ impl PostgresRedoManagerInternal {
                     buf.advance(skip);
                 }
 
-                if xlogrec.xl_rmid == pg_constants::RM_CLOG_ID {
-                    let info = xlogrec.xl_info & !pg_constants::XLR_INFO_MASK;
-                    if info == pg_constants::CLOG_ZEROPAGE {
-                        page.copy_from_slice(&ZERO_PAGE);
-                    }
-                } else if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
-                    let info = xlogrec.xl_info & pg_constants::XLOG_XACT_OPMASK;
-                    let mut status = 0;
-                    if info == pg_constants::XLOG_XACT_COMMIT || info == pg_constants::XLOG_XACT_COMMIT_PREPARED {
-                        status = pg_constants::TRANSACTION_STATUS_COMMITTED;
-						if info == pg_constants::XLOG_XACT_COMMIT {
-							transaction_id_set_status(xlogrec.xl_xid, status, &mut page);
-						}
-                        //handle subtrans
-                        let _xact_time = buf.get_i64_le();
-                        let mut xinfo = 0;
-                        if xlogrec.xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
-                            xinfo = buf.get_u32_le();
-                            if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
-                                let _dbid = buf.get_u32_le();
-                                let _tsid = buf.get_u32_le();
+                if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
+                    // Transaction manager stuff
+                    let rec_segno = match rel {
+                        RelishTag::Slru { slru, segno } => {
+                            if slru != SlruKind::Clog {
+                                panic!("Not valid XACT relish tag {:?}", rel);
+                            }
+                            segno
+                        }
+                        _ => panic!("Not valid XACT relish tag {:?}", rel),
+                    };
+                    let parsed_xact =
+                        XlXactParsedRecord::decode(&mut buf, xlogrec.xl_xid, xlogrec.xl_info);
+                    if parsed_xact.info == pg_constants::XLOG_XACT_COMMIT
+                        || parsed_xact.info == pg_constants::XLOG_XACT_COMMIT_PREPARED
+                    {
+                        transaction_id_set_status(
+                            parsed_xact.xid,
+                            pg_constants::TRANSACTION_STATUS_COMMITTED,
+                            &mut page,
+                        );
+                        for subxact in &parsed_xact.subxacts {
+                            let pageno = *subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
+                            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+                            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+                            // only update xids on the requested page
+                            if rec_segno == segno && blknum == rpageno {
+                                transaction_id_set_status(
+                                    *subxact,
+                                    pg_constants::TRANSACTION_STATUS_SUB_COMMITTED,
+                                    &mut page,
+                                );
                             }
                         }
-
-                        if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
-                            let nsubxacts = buf.get_i32_le();
-                            for _i in 0..nsubxacts {
-                                let subxact = buf.get_u32_le();
-                                let blkno = subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
-                                // only update xids on the requested page
-                                if tag.blknum == blkno {
-                                    status = pg_constants::TRANSACTION_STATUS_SUB_COMMITTED;
-                                    transaction_id_set_status(subxact, status, &mut page);
-                                }
+                    } else if parsed_xact.info == pg_constants::XLOG_XACT_ABORT
+                        || parsed_xact.info == pg_constants::XLOG_XACT_ABORT_PREPARED
+                    {
+                        transaction_id_set_status(
+                            parsed_xact.xid,
+                            pg_constants::TRANSACTION_STATUS_ABORTED,
+                            &mut page,
+                        );
+                        for subxact in &parsed_xact.subxacts {
+                            let pageno = *subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
+                            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+                            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+                            // only update xids on the requested page
+                            if rec_segno == segno && blknum == rpageno {
+                                transaction_id_set_status(
+                                    *subxact,
+                                    pg_constants::TRANSACTION_STATUS_ABORTED,
+                                    &mut page,
+                                );
                             }
                         }
-						if info == pg_constants::XLOG_XACT_COMMIT_PREPARED {
-							if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
-								let nrels = buf.get_i32_le();
-								for _i in 0..nrels {
-									let spcnode = buf.get_u32_le();
-									let dbnode = buf.get_u32_le();
-									let relnode = buf.get_u32_le();
-									//TODO handle this too?
-									trace!(
-										"XLOG_XACT_COMMIT relfilenode {}/{}/{}",
-										spcnode,
-										dbnode,
-										relnode
-									);
-								}
-							}
-							if xinfo & pg_constants::XACT_XINFO_HAS_INVALS != 0 {
-								let nmsgs = buf.get_i32_le();
-								for _i in 0..nmsgs {
-									let sizeof_shared_invalidation_message = 0;
-									buf.advance(sizeof_shared_invalidation_message);
-								}
-							}
-							assert!((xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE) != 0);
-							let xid = buf.get_u32_le();
-							transaction_id_set_status(xid, status, &mut page);
-						}
-                    } else if info == pg_constants::XLOG_XACT_ABORT || info == pg_constants::XLOG_XACT_ABORT_PREPARED {
-                        status = pg_constants::TRANSACTION_STATUS_ABORTED;
-						if info == pg_constants::XLOG_XACT_ABORT {
-							transaction_id_set_status(xlogrec.xl_xid, status, &mut page);
-						}
-                        //handle subtrans
-                        let _xact_time = buf.get_i64_le();
-                        let mut xinfo = 0;
-                        if xlogrec.xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
-                            xinfo = buf.get_u32_le();
-                            if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
-                                let _dbid = buf.get_u32_le();
-                                let _tsid = buf.get_u32_le();
-                            }
-                        }
-
-                        if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
-                            let nsubxacts = buf.get_i32_le();
-                            for _i in 0..nsubxacts {
-                                let subxact = buf.get_u32_le();
-                                let blkno = subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
-                                // only update xids on the requested page
-                                if tag.blknum == blkno {
-                                    status = pg_constants::TRANSACTION_STATUS_ABORTED;
-                                    transaction_id_set_status(subxact, status, &mut page);
-                                }
-                            }
-                        }
-						if info == pg_constants::XLOG_XACT_ABORT_PREPARED {
-							if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
-								let nrels = buf.get_i32_le();
-								for _i in 0..nrels {
-									let spcnode = buf.get_u32_le();
-									let dbnode = buf.get_u32_le();
-									let relnode = buf.get_u32_le();
-									//TODO handle this too?
-									trace!(
-										"XLOG_XACT_COMMIT relfilenode {}/{}/{}",
-										spcnode,
-										dbnode,
-										relnode
-									);
-								}
-							}
-							if xinfo & pg_constants::XACT_XINFO_HAS_INVALS != 0 {
-								let nmsgs = buf.get_i32_le();
-								for _i in 0..nmsgs {
-									let sizeof_shared_invalidation_message = 0;
-									buf.advance(sizeof_shared_invalidation_message);
-								}
-							}
-							assert!((xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE) != 0);
-							let xid = buf.get_u32_le();
-							transaction_id_set_status(xid, status, &mut page);
-						}
-                    } else if info == pg_constants::XLOG_XACT_PREPARE {
-						info!("Apply prepare {} record", xlogrec.xl_xid);
-						page.clear();
-						page.extend_from_slice(&buf[..]);
-					} else {
-                        error!("handle_apply_request for RM_XACT_ID-{} NOT SUPPORTED YET. RETURN. lsn {} main_data_offset {}, rec.len {}",
-                               status,
-                               record.lsn,
-                               record.main_data_offset, record.rec.len());
                     }
                 } else if xlogrec.xl_rmid == pg_constants::RM_MULTIXACT_ID {
+                    // Multixact operations
                     let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
-                    if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE {
-                        page.copy_from_slice(&ZERO_PAGE);
-                    } else if info == pg_constants::XLOG_MULTIXACT_ZERO_MEM_PAGE {
-                        page.copy_from_slice(&ZERO_PAGE);
-                    } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
+                    if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
                         let xlrec = XlMultiXactCreate::decode(&mut buf);
-                        if tag.rel.forknum == pg_constants::PG_MXACT_OFFSETS_FORKNUM {
-                            let offs = (xlrec.mid % pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32
-                                * 4) as usize;
-                            LittleEndian::write_u32(&mut page[offs..offs + 4], xlrec.moff);
-                        } else {
-                            assert!(tag.rel.forknum == pg_constants::PG_MXACT_MEMBERS_FORKNUM);
-                            for i in 0..xlrec.nmembers {
-                                let blkno = i / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-                                if blkno == tag.blknum {
-                                    // update only target block
-                                    let offset = xlrec.moff + i;
-                                    let memberoff = mx_offset_to_member_offset(offset);
-                                    let flagsoff = mx_offset_to_flags_offset(offset);
-                                    let bshift = mx_offset_to_flags_bitshift(offset);
-                                    let mut flagsval =
-                                        LittleEndian::read_u32(&page[flagsoff..flagsoff + 4]);
-                                    flagsval &=
-                                        !(((1 << pg_constants::MXACT_MEMBER_BITS_PER_XACT) - 1)
+                        if let RelishTag::Slru {
+                            slru,
+                            segno: rec_segno,
+                        } = rel
+                        {
+                            if slru == SlruKind::MultiXactMembers {
+                                for i in 0..xlrec.nmembers {
+                                    let pageno =
+                                        i / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
+                                    let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+                                    let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+                                    if segno == rec_segno && rpageno == blknum {
+                                        // update only target block
+                                        let offset = xlrec.moff + i;
+                                        let memberoff = mx_offset_to_member_offset(offset);
+                                        let flagsoff = mx_offset_to_flags_offset(offset);
+                                        let bshift = mx_offset_to_flags_bitshift(offset);
+                                        let mut flagsval =
+                                            LittleEndian::read_u32(&page[flagsoff..flagsoff + 4]);
+                                        flagsval &= !(((1
+                                            << pg_constants::MXACT_MEMBER_BITS_PER_XACT)
+                                            - 1)
                                             << bshift);
-                                    flagsval |= xlrec.members[i as usize].status << bshift;
-                                    LittleEndian::write_u32(
-                                        &mut page[flagsoff..flagsoff + 4],
-                                        flagsval,
-                                    );
-                                    LittleEndian::write_u32(
-                                        &mut page[memberoff..memberoff + 4],
-                                        xlrec.members[i as usize].xid,
-                                    );
+                                        flagsval |= xlrec.members[i as usize].status << bshift;
+                                        LittleEndian::write_u32(
+                                            &mut page[flagsoff..flagsoff + 4],
+                                            flagsval,
+                                        );
+                                        LittleEndian::write_u32(
+                                            &mut page[memberoff..memberoff + 4],
+                                            xlrec.members[i as usize].xid,
+                                        );
+                                    }
                                 }
+                            } else {
+                                // Multixact offsets SLRU
+                                let offs = (xlrec.mid
+                                    % pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32
+                                    * 4) as usize;
+                                LittleEndian::write_u32(&mut page[offs..offs + 4], xlrec.moff);
                             }
+                        } else {
+                            panic!();
                         }
-                    } else if info == pg_constants::XLOG_MULTIXACT_TRUNCATE_ID {
-                        // empty page image indicates that this SLRU page is truncated and can be removed by GC
-                        page.clear();
                     } else {
-                        assert!(false);
+                        panic!();
                     }
                 } else if xlogrec.xl_rmid == pg_constants::RM_RELMAP_ID {
+                    // Relation map file has size 512 bytes
                     page.clear();
                     page.extend_from_slice(&buf[12..]); // skip xl_relmap_update
                     assert!(page.len() == 512); // size of pg_filenode.map
@@ -460,15 +427,13 @@ impl PostgresRedoManagerInternal {
             }
 
             apply_result = Ok::<Bytes, Error>(page.freeze());
-        } else {
-            apply_result = process.apply_wal_records(tag, base_img, records).await;
         }
 
         let duration = start.elapsed();
 
         let result: Result<Bytes, WalRedoError>;
 
-        trace!(
+        debug!(
             "applied {} WAL records in {} ms to reconstruct page image at LSN {}",
             nrecords,
             duration.as_millis(),
@@ -489,6 +454,9 @@ impl PostgresRedoManagerInternal {
     }
 }
 
+///
+/// Handle to the Postgres WAL redo process
+///
 struct PostgresRedoProcess {
     stdin: RefCell<ChildStdin>,
     stdout: RefCell<ChildStdout>,
@@ -498,12 +466,15 @@ impl PostgresRedoProcess {
     //
     // Start postgres binary in special WAL redo mode.
     //
-    // Tests who run pageserver binary are setting proper PG_BIN_DIR
-    // and PG_LIB_DIR so that WalRedo would start right postgres.
+    async fn launch(
+        conf: &PageServerConf,
+        tenantid: &ZTenantId,
+    ) -> Result<PostgresRedoProcess, Error> {
+        // FIXME: We need a dummy Postgres cluster to run the process in. Currently, we
+        // just create one with constant name. That fails if you try to launch more than
+        // one WAL redo manager concurrently.
+        let datadir = conf.tenant_path(&tenantid).join("wal-redo-datadir");
 
-    // do that: We may later
-    // switch to setting same things in pageserver config file.
-    async fn launch(datadir: &Path) -> Result<PostgresRedoProcess, Error> {
         // Create empty data directory for wal-redo postgres, deleting old one first.
         if datadir.exists() {
             info!("directory {:?} exists, removing", &datadir);
@@ -512,9 +483,12 @@ impl PostgresRedoProcess {
             }
         }
         info!("running initdb in {:?}", datadir.display());
-        let initdb = Command::new("initdb")
+        let initdb = Command::new(conf.pg_bin_dir().join("initdb"))
             .args(&["-D", datadir.to_str().unwrap()])
             .arg("-N")
+            .env_clear()
+            .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
+            .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
             .output()
             .await
             .expect("failed to execute initdb");
@@ -536,12 +510,15 @@ impl PostgresRedoProcess {
             config.write_all(b"zenith.wal_redo=on\n")?;
         }
         // Start postgres itself
-        let mut child = Command::new("postgres")
+        let mut child = Command::new(conf.pg_bin_dir().join("postgres"))
             .arg("--wal-redo")
             .stdin(Stdio::piped())
             .stderr(Stdio::piped())
             .stdout(Stdio::piped())
-            .env("PGDATA", datadir)
+            .env_clear()
+            .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
+            .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
+            .env("PGDATA", &datadir)
             .spawn()
             .expect("postgres --wal-redo command failed to start");
 
@@ -621,6 +598,8 @@ impl PostgresRedoProcess {
             for rec in records.iter() {
                 let r = rec.clone();
 
+                WAL_REDO_RECORD_COUNTER.inc();
+
                 stdin
                     .write_all(&build_apply_record_msg(r.lsn, r.rec))
                     .await?;
@@ -665,9 +644,26 @@ fn build_begin_redo_for_block_msg(tag: BufferTag) -> Bytes {
 
     buf.put_u8(b'B');
     buf.put_u32(len as u32);
-    tag.pack(&mut buf);
 
-    assert!(buf.len() == 1 + len);
+    // FIXME: this is a temporary hack that should go away when we refactor
+    // the postgres protocol serialization + handlers.
+    //
+    // BytesMut is a dynamic growable buffer, used a lot in tokio code but
+    // not in the std library. To write to a BytesMut from a serde serializer,
+    // we need to either:
+    // - pre-allocate the required buffer space. This is annoying because we
+    //   shouldn't care what the exact serialized size is-- that's the
+    //   serializer's job.
+    // - Or, we need to create a temporary "writer" (which implements the
+    //   `Write` trait). It's a bit awkward, because the writer consumes the
+    //   underlying BytesMut, and we need to extract it later with
+    //   `into_inner`.
+    let mut writer = buf.writer();
+    tag.ser_into(&mut writer)
+        .expect("serialize BufferTag should always succeed");
+    let buf = writer.into_inner();
+
+    debug_assert!(buf.len() == 1 + len);
 
     buf.freeze()
 }
@@ -680,10 +676,13 @@ fn build_push_page_msg(tag: BufferTag, base_img: Bytes) -> Bytes {
 
     buf.put_u8(b'P');
     buf.put_u32(len as u32);
-    tag.pack(&mut buf);
+    let mut writer = buf.writer();
+    tag.ser_into(&mut writer)
+        .expect("serialize BufferTag should always succeed");
+    let mut buf = writer.into_inner();
     buf.put(base_img);
 
-    assert!(buf.len() == 1 + len);
+    debug_assert!(buf.len() == 1 + len);
 
     buf.freeze()
 }
@@ -697,7 +696,7 @@ fn build_apply_record_msg(endlsn: Lsn, rec: Bytes) -> Bytes {
     buf.put_u64(endlsn.0);
     buf.put(rec);
 
-    assert!(buf.len() == 1 + len);
+    debug_assert!(buf.len() == 1 + len);
 
     buf.freeze()
 }
@@ -708,9 +707,12 @@ fn build_get_page_msg(tag: BufferTag) -> Bytes {
 
     buf.put_u8(b'G');
     buf.put_u32(len as u32);
-    tag.pack(&mut buf);
+    let mut writer = buf.writer();
+    tag.ser_into(&mut writer)
+        .expect("serialize BufferTag should always succeed");
+    let buf = writer.into_inner();
 
-    assert!(buf.len() == 1 + len);
+    debug_assert!(buf.len() == 1 + len);
 
     buf.freeze()
 }

postgres_ffi/Cargo.toml

@@ -17,8 +17,10 @@ crc32c = "0.6.0"
 hex = "0.4.3"
 lazy_static = "1.4"
 log = "0.4.14"
+memoffset = "0.6.2"
 thiserror = "1.0"
 workspace_hack = { path = "../workspace_hack" }
+zenith_utils = { path = "../zenith_utils" }
 
 [build-dependencies]
 bindgen = "0.57"

postgres_ffi/README

@@ -1,3 +1,25 @@
-This module contains utility functions for interacting with PostgreSQL
-file formats.
+This module contains utilities for working with PostgreSQL file
+formats. It's a collection of structs that are auto-generated from the
+PostgreSQL header files using bindgen, and Rust functions to read and
+manipulate them.
 
+There are also a bunch of constants in `pg_constants.rs` that are copied
+from various PostgreSQL headers, rather than auto-generated. They mostly
+should be auto-generated too, but that's a TODO.
+
+The PostgreSQL on-disk file format is not portable across different
+CPU architectures and operating systems. It is also subject to change
+in each major PostgreSQL version. Currently, this module is based on
+PostgreSQL v14, but in the future we will probably need a separate
+copy for each PostgreSQL version.
+
+To interact with the C structs, there is some unsafe code in this
+module. Do not copy-paste that to the rest of the codebase! Keep the
+amount of unsafe code to a minimum, and limited to this module only,
+and only where it's truly needed.
+
+TODO: Currently, there is also some code that deals with WAL records
+in pageserver/src/waldecoder.rs.  That should be moved into this
+module. The rest of the codebase should not have intimate knowledge of
+PostgreSQL file formats or WAL layout, that knowledge should be
+encapsulated in this module.

postgres_ffi/build.rs

@@ -11,29 +11,42 @@ fn main() {
     // to bindgen, and lets you build up options for
     // the resulting bindings.
     let bindings = bindgen::Builder::default()
-        // The input header we would like to generate
-        // bindings for.
+        //
+        // All the needed PostgreSQL headers are included from 'pg_control_ffi.h'
+        //
         .header("pg_control_ffi.h")
+        //
         // Tell cargo to invalidate the built crate whenever any of the
         // included header files changed.
+        //
         .parse_callbacks(Box::new(bindgen::CargoCallbacks))
+        //
+        // These are the types and constants that we want to generate bindings for
+        //
         .whitelist_type("ControlFileData")
         .whitelist_type("CheckPoint")
         .whitelist_type("FullTransactionId")
+        .whitelist_type("XLogRecord")
+        .whitelist_type("XLogPageHeaderData")
+        .whitelist_type("XLogLongPageHeaderData")
+        .whitelist_var("XLOG_PAGE_MAGIC")
         .whitelist_var("PG_CONTROL_FILE_SIZE")
         .whitelist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
         .whitelist_type("DBState")
+        //
         // Path the server include dir. It is in tmp_install/include/server, if you did
         // "configure --prefix=<path to tmp_install>". But if you used "configure --prefix=/",
         // and used DESTDIR to move it into tmp_install, then it's in
         // tmp_install/include/postgres/server
         // 'pg_config --includedir-server' would perhaps be the more proper way to find it,
         // but this will do for now.
+        //
         .clang_arg("-I../tmp_install/include/server")
         .clang_arg("-I../tmp_install/include/postgresql/server")
+        //
         // Finish the builder and generate the bindings.
+        //
         .generate()
-        // Unwrap the Result and panic on failure.
         .expect("Unable to generate bindings");
 
     // Write the bindings to the $OUT_DIR/bindings.rs file.

postgres_ffi/pg_control_ffi.h

@@ -1,4 +1,10 @@
+/*
+ * This header file is the input to bindgen. It includes all the
+ * PostgreSQL headers that we need to auto-generate Rust structs
+ * from. If you need to expose a new struct to Rust code, add the
+ * header here, and whitelist the struct in the build.rs file.
+ */
 #include "c.h"
 #include "catalog/pg_control.h"
+#include "access/xlog_internal.h"
 
-const uint32 PG_CONTROLFILEDATA_OFFSETOF_CRC = offsetof(ControlFileData, crc);

postgres_ffi/samples/pg_hba.conf

@@ -0,0 +1,98 @@
+# PostgreSQL Client Authentication Configuration File
+# ===================================================
+#
+# Refer to the "Client Authentication" section in the PostgreSQL
+# documentation for a complete description of this file.  A short
+# synopsis follows.
+#
+# This file controls: which hosts are allowed to connect, how clients
+# are authenticated, which PostgreSQL user names they can use, which
+# databases they can access.  Records take one of these forms:
+#
+# local         DATABASE  USER  METHOD  [OPTIONS]
+# host          DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+# hostssl       DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+# hostnossl     DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+# hostgssenc    DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+# hostnogssenc  DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+#
+# (The uppercase items must be replaced by actual values.)
+#
+# The first field is the connection type:
+# - "local" is a Unix-domain socket
+# - "host" is a TCP/IP socket (encrypted or not)
+# - "hostssl" is a TCP/IP socket that is SSL-encrypted
+# - "hostnossl" is a TCP/IP socket that is not SSL-encrypted
+# - "hostgssenc" is a TCP/IP socket that is GSSAPI-encrypted
+# - "hostnogssenc" is a TCP/IP socket that is not GSSAPI-encrypted
+#
+# DATABASE can be "all", "sameuser", "samerole", "replication", a
+# database name, or a comma-separated list thereof. The "all"
+# keyword does not match "replication". Access to replication
+# must be enabled in a separate record (see example below).
+#
+# USER can be "all", a user name, a group name prefixed with "+", or a
+# comma-separated list thereof.  In both the DATABASE and USER fields
+# you can also write a file name prefixed with "@" to include names
+# from a separate file.
+#
+# ADDRESS specifies the set of hosts the record matches.  It can be a
+# host name, or it is made up of an IP address and a CIDR mask that is
+# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that
+# specifies the number of significant bits in the mask.  A host name
+# that starts with a dot (.) matches a suffix of the actual host name.
+# Alternatively, you can write an IP address and netmask in separate
+# columns to specify the set of hosts.  Instead of a CIDR-address, you
+# can write "samehost" to match any of the server's own IP addresses,
+# or "samenet" to match any address in any subnet that the server is
+# directly connected to.
+#
+# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256",
+# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert".
+# Note that "password" sends passwords in clear text; "md5" or
+# "scram-sha-256" are preferred since they send encrypted passwords.
+#
+# OPTIONS are a set of options for the authentication in the format
+# NAME=VALUE.  The available options depend on the different
+# authentication methods -- refer to the "Client Authentication"
+# section in the documentation for a list of which options are
+# available for which authentication methods.
+#
+# Database and user names containing spaces, commas, quotes and other
+# special characters must be quoted.  Quoting one of the keywords
+# "all", "sameuser", "samerole" or "replication" makes the name lose
+# its special character, and just match a database or username with
+# that name.
+#
+# This file is read on server startup and when the server receives a
+# SIGHUP signal.  If you edit the file on a running system, you have to
+# SIGHUP the server for the changes to take effect, run "pg_ctl reload",
+# or execute "SELECT pg_reload_conf()".
+#
+# Put your actual configuration here
+# ----------------------------------
+#
+# If you want to allow non-local connections, you need to add more
+# "host" records.  In that case you will also need to make PostgreSQL
+# listen on a non-local interface via the listen_addresses
+# configuration parameter, or via the -i or -h command line switches.
+
+# CAUTION: Configuring the system for local "trust" authentication
+# allows any local user to connect as any PostgreSQL user, including
+# the database superuser.  If you do not trust all your local users,
+# use another authentication method.
+
+
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+
+# "local" is for Unix domain socket connections only
+local   all             all                                     trust
+# IPv4 local connections:
+host    all             all             127.0.0.1/32            trust
+# IPv6 local connections:
+host    all             all             ::1/128                 trust
+# Allow replication connections from localhost, by a user with the
+# replication privilege.
+local   replication     all                                     trust
+host    replication     all             127.0.0.1/32            trust
+host    replication     all             ::1/128                 trust

postgres_ffi/src/controlfile_utils.rs

@@ -0,0 +1,124 @@
+//!
+//! Utilities for reading and writing the PostgreSQL control file.
+//!
+//! The PostgreSQL control file is one the first things that the PostgreSQL
+//! server reads when it starts up. It indicates whether the server was shut
+//! down cleanly, or if it crashed or was restored from online backup so that
+//! WAL recovery needs to be performed. It also contains a copy of the latest
+//! checkpoint record and its location in the WAL.
+//!
+//! The control file also contains fields for detecting whether the
+//! data directory is compatible with a postgres binary. That includes
+//! a version number, configuration options that can be set at
+//! compilation time like the block size, and the platform's alignment
+//! and endianess information. (The PostgreSQL on-disk file format is
+//! not portable across platforms.)
+//!
+//! The control file is stored in the PostgreSQL data directory, as
+//! `global/pg_control`. The data stored in it is designed to be smaller than
+//! 512 bytes, on the assumption that it can be updated atomically. The actual
+//! file is larger, 8192 bytes, but the rest of it is just filled with zeros.
+//!
+//! See src/include/catalog/pg_control.h in the PostgreSQL sources for more
+//! information. You can use PostgreSQL's pg_controldata utility to view its
+//! contents.
+//!
+use crate::{ControlFileData, PG_CONTROL_FILE_SIZE};
+
+use anyhow::{bail, Result};
+use bytes::{Bytes, BytesMut};
+
+/// Equivalent to sizeof(ControlFileData) in C
+const SIZEOF_CONTROLDATA: usize = std::mem::size_of::<ControlFileData>();
+
+impl ControlFileData {
+    /// Compute the offset of the `crc` field within the `ControlFileData` struct.
+    /// Equivalent to offsetof(ControlFileData, crc) in C.
+    // Someday this can be const when the right compiler features land.
+    fn pg_control_crc_offset() -> usize {
+        memoffset::offset_of!(ControlFileData, crc)
+    }
+
+    ///
+    /// Interpret a slice of bytes as a Postgres control file.
+    ///
+    pub fn decode(buf: &[u8]) -> Result<ControlFileData> {
+        // Check that the slice has the expected size. The control file is
+        // padded with zeros up to a 512 byte sector size, so accept a
+        // larger size too, so that the caller can just the whole file
+        // contents without knowing the exact size of the struct.
+        if buf.len() < SIZEOF_CONTROLDATA {
+            bail!("control file is too short");
+        }
+
+        // Compute the expected CRC of the content.
+        let OFFSETOF_CRC = Self::pg_control_crc_offset();
+        let expectedcrc = crc32c::crc32c(&buf[0..OFFSETOF_CRC]);
+
+        // Convert the slice into an array of the right size, and use `transmute` to
+        // reinterpret the raw bytes as a ControlFileData struct.
+        //
+        // NB: Ideally we would use 'zerocopy::FromBytes' for this, but bindgen doesn't
+        // derive FromBytes for us. The safety of this depends on the same constraints
+        // as for FromBytes, namely, all of its fields must implement FromBytes. That
+        // includes the primitive integer types, like `u8`, `u16`, `u32`, `u64` and their
+        // signed variants. But `bool` is not safe, because the contents of the high bits
+        // in a rust bool are undefined. In practice, PostgreSQL uses 1 to represent
+        // true and 0 for false, which is compatible with Rust bool, but let's try not to
+        // depend on it.
+        //
+        // FIXME: ControlFileData does contain 'bool's at the moment.
+        //
+        // See https://github.com/zenithdb/zenith/issues/207 for discussion on the safety
+        // of this.
+        let mut b: [u8; SIZEOF_CONTROLDATA] = [0u8; SIZEOF_CONTROLDATA];
+        b.copy_from_slice(&buf[0..SIZEOF_CONTROLDATA]);
+        let controlfile: ControlFileData =
+            unsafe { std::mem::transmute::<[u8; SIZEOF_CONTROLDATA], ControlFileData>(b) };
+
+        // Check the CRC
+        if expectedcrc != controlfile.crc {
+            bail!(
+                "invalid CRC in control file: expected {:08X}, was {:08X}",
+                expectedcrc,
+                controlfile.crc
+            );
+        }
+
+        Ok(controlfile)
+    }
+
+    ///
+    /// Convert a struct representing a Postgres control file into raw bytes.
+    ///
+    /// The CRC is recomputed to match the contents of the fields.
+    pub fn encode(&self) -> Bytes {
+        //
+        // Use `transmute` to reinterpret struct as raw bytes.
+        //
+        // FIXME: This triggers undefined behavior, because the contents
+        // of the padding bytes are undefined, and this leaks those
+        // undefined bytes into the resulting array. The Rust code won't
+        // care what's in those bytes, and PostgreSQL doesn't care
+        // either. HOWEVER, it is a potential security issue, because the
+        // bytes can contain arbitrary pieces of memory from the page
+        // server. In the worst case, that could be private keys or
+        // another tenant's data.
+        //
+        // See https://github.com/zenithdb/zenith/issues/207 for discussion.
+        let b: [u8; SIZEOF_CONTROLDATA] =
+            unsafe { std::mem::transmute::<ControlFileData, [u8; SIZEOF_CONTROLDATA]>(*self) };
+
+        // Recompute the CRC
+        let OFFSETOF_CRC = Self::pg_control_crc_offset();
+        let newcrc = crc32c::crc32c(&b[0..OFFSETOF_CRC]);
+
+        let mut buf = BytesMut::with_capacity(PG_CONTROL_FILE_SIZE as usize);
+        buf.extend_from_slice(&b[0..OFFSETOF_CRC]);
+        buf.extend_from_slice(&newcrc.to_ne_bytes());
+        // Fill the rest of the control file with zeros.
+        buf.resize(PG_CONTROL_FILE_SIZE as usize, 0);
+
+        buf.into()
+    }
+}

postgres_ffi/src/lib.rs

@@ -1,112 +1,13 @@
 #![allow(non_upper_case_globals)]
 #![allow(non_camel_case_types)]
 #![allow(non_snake_case)]
+// suppress warnings on rust 1.53 due to bindgen unit tests.
+// https://github.com/rust-lang/rust-bindgen/issues/1651
+#![allow(deref_nullptr)]
 include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
 
+pub mod controlfile_utils;
 pub mod nonrelfile_utils;
 pub mod pg_constants;
 pub mod relfile_utils;
 pub mod xlog_utils;
-
-use bytes::{Buf, Bytes, BytesMut};
-
-// sizeof(ControlFileData)
-const SIZEOF_CONTROLDATA: usize = std::mem::size_of::<ControlFileData>();
-const SIZEOF_CHECKPOINT: usize = std::mem::size_of::<CheckPoint>();
-const OFFSETOF_CRC: usize = PG_CONTROLFILEDATA_OFFSETOF_CRC as usize;
-
-impl ControlFileData {
-    // Initialize an all-zeros ControlFileData struct
-    pub fn new() -> ControlFileData {
-        let controlfile: ControlFileData;
-
-        let b = [0u8; SIZEOF_CONTROLDATA];
-        controlfile =
-            unsafe { std::mem::transmute::<[u8; SIZEOF_CONTROLDATA], ControlFileData>(b) };
-
-        controlfile
-    }
-}
-
-pub fn decode_pg_control(mut buf: Bytes) -> Result<ControlFileData, anyhow::Error> {
-    let mut b: [u8; SIZEOF_CONTROLDATA] = [0u8; SIZEOF_CONTROLDATA];
-    buf.copy_to_slice(&mut b);
-
-    let controlfile: ControlFileData;
-
-    // TODO: verify CRC
-    let mut data_without_crc: [u8; OFFSETOF_CRC] = [0u8; OFFSETOF_CRC];
-    data_without_crc.copy_from_slice(&b[0..OFFSETOF_CRC]);
-    let expectedcrc = crc32c::crc32c(&data_without_crc);
-
-    controlfile = unsafe { std::mem::transmute::<[u8; SIZEOF_CONTROLDATA], ControlFileData>(b) };
-
-    if expectedcrc != controlfile.crc {
-        anyhow::bail!(
-            "invalid CRC in control file: expected {:08X}, was {:08X}",
-            expectedcrc,
-            controlfile.crc
-        );
-    }
-
-    Ok(controlfile)
-}
-
-pub fn encode_pg_control(controlfile: ControlFileData) -> Bytes {
-    let b: [u8; SIZEOF_CONTROLDATA];
-
-    b = unsafe { std::mem::transmute::<ControlFileData, [u8; SIZEOF_CONTROLDATA]>(controlfile) };
-
-    // Recompute the CRC
-    let mut data_without_crc: [u8; OFFSETOF_CRC] = [0u8; OFFSETOF_CRC];
-    data_without_crc.copy_from_slice(&b[0..OFFSETOF_CRC]);
-    let newcrc = crc32c::crc32c(&data_without_crc);
-
-    let mut buf = BytesMut::with_capacity(PG_CONTROL_FILE_SIZE as usize);
-
-    buf.extend_from_slice(&b[0..OFFSETOF_CRC]);
-    buf.extend_from_slice(&newcrc.to_ne_bytes());
-    // Fill the rest of the control file with zeros.
-    buf.resize(PG_CONTROL_FILE_SIZE as usize, 0);
-
-    buf.into()
-}
-
-pub fn encode_checkpoint(checkpoint: CheckPoint) -> Bytes {
-    let b: [u8; SIZEOF_CHECKPOINT];
-    b = unsafe { std::mem::transmute::<CheckPoint, [u8; SIZEOF_CHECKPOINT]>(checkpoint) };
-    return Bytes::copy_from_slice(&b[..]);
-}
-
-pub fn decode_checkpoint(mut buf: Bytes) -> Result<CheckPoint, anyhow::Error> {
-    let mut b = [0u8; SIZEOF_CHECKPOINT];
-    buf.copy_to_slice(&mut b);
-    let checkpoint: CheckPoint;
-    checkpoint = unsafe { std::mem::transmute::<[u8; SIZEOF_CHECKPOINT], CheckPoint>(b) };
-    Ok(checkpoint)
-}
-
-impl CheckPoint {
-    pub fn new(lsn: u64, timeline: u32) -> CheckPoint {
-        CheckPoint {
-            redo: lsn,
-            ThisTimeLineID: timeline,
-            PrevTimeLineID: timeline,
-            fullPageWrites: true, // TODO: get actual value of full_page_writes
-            nextXid: FullTransactionId {
-                value: pg_constants::FIRST_NORMAL_TRANSACTION_ID as u64,
-            }, // TODO: handle epoch?
-            nextOid: pg_constants::FIRST_BOOTSTRAP_OBJECT_ID,
-            nextMulti: 1,
-            nextMultiOffset: 0,
-            oldestXid: pg_constants::FIRST_NORMAL_TRANSACTION_ID,
-            oldestXidDB: 0,
-            oldestMulti: 1,
-            oldestMultiDB: 0,
-            time: 0,
-            oldestCommitTsXid: 0,
-            newestCommitTsXid: 0,
-            oldestActiveXid: pg_constants::INVALID_TRANSACTION_ID,
-        }
-    }
-}

postgres_ffi/src/nonrelfile_utils.rs

@@ -28,5 +28,5 @@ pub fn transaction_id_get_status(xid: u32, page: &[u8]) -> u8 {
     let bshift: u8 =
         ((xid % pg_constants::CLOG_XACTS_PER_BYTE) * pg_constants::CLOG_BITS_PER_XACT as u32) as u8;
 
-    return ((page[byteno] >> bshift) & pg_constants::CLOG_XACT_BITMASK) as u8;
+    ((page[byteno] >> bshift) & pg_constants::CLOG_XACT_BITMASK) as u8
 }

postgres_ffi/src/pg_constants.rs

@@ -1,6 +1,11 @@
 //!
 //! Misc constants, copied from PostgreSQL headers.
 //!
+//! TODO: These probably should be auto-generated using bindgen,
+//! rather than copied by hand. Although on the other hand, it's nice
+//! to have them all here in one place, and have the ability to add
+//! comments on them.
+//!
 
 //
 // From pg_tablespace_d.h
@@ -15,18 +20,11 @@ pub const MAIN_FORKNUM: u8 = 0;
 pub const FSM_FORKNUM: u8 = 1;
 pub const VISIBILITYMAP_FORKNUM: u8 = 2;
 pub const INIT_FORKNUM: u8 = 3;
-// Special values for non-rel files' tags (Zenith-specific)
-//Special values for non-rel files' tags
-pub const PG_CONTROLFILE_FORKNUM: u8 = 42;
-pub const PG_FILENODEMAP_FORKNUM: u8 = 43;
-pub const PG_XACT_FORKNUM: u8 = 44;
-pub const PG_MXACT_OFFSETS_FORKNUM: u8 = 45;
-pub const PG_MXACT_MEMBERS_FORKNUM: u8 = 46;
-pub const PG_TWOPHASE_FORKNUM: u8 = 47;
-pub const PG_CHECKPOINT_FORKNUM: u8 = 48;
 
 // From storage_xlog.h
 pub const SMGR_TRUNCATE_HEAP: u32 = 0x0001;
+pub const SMGR_TRUNCATE_VM: u32 = 0x0002;
+pub const SMGR_TRUNCATE_FSM: u32 = 0x0004;
 
 // from pg_config.h. These can be changed with configure options --with-blocksize=BLOCKSIZE and
 // --with-segsize=SEGSIZE, but assume the defaults for now.
@@ -48,7 +46,6 @@ pub const SIZE_OF_PAGE_HEADER: u16 = 24;
 pub const BITS_PER_HEAPBLOCK: u16 = 2;
 pub const HEAPBLOCKS_PER_PAGE: u16 = (BLCKSZ - SIZE_OF_PAGE_HEADER) * 8 / BITS_PER_HEAPBLOCK;
 
-pub const TRANSACTION_STATUS_IN_PROGRESS: u8 = 0x00;
 pub const TRANSACTION_STATUS_COMMITTED: u8 = 0x01;
 pub const TRANSACTION_STATUS_ABORTED: u8 = 0x02;
 pub const TRANSACTION_STATUS_SUB_COMMITTED: u8 = 0x03;
@@ -65,9 +62,11 @@ pub const XLOG_XACT_ABORT_PREPARED: u8 = 0x40;
 
 // From srlu.h
 pub const SLRU_PAGES_PER_SEGMENT: u32 = 32;
+pub const SLRU_SEG_SIZE: usize = BLCKSZ as usize * SLRU_PAGES_PER_SEGMENT as usize;
 
 /* mask for filtering opcodes out of xl_info */
 pub const XLOG_XACT_OPMASK: u8 = 0x70;
+pub const XLOG_HEAP_OPMASK: u8 = 0x70;
 /* does this record have a 'xinfo' field or not */
 pub const XLOG_XACT_HAS_INFO: u8 = 0x80;
 
@@ -88,8 +87,12 @@ pub const XACT_XINFO_HAS_TWOPHASE: u32 = 1u32 << 4;
 pub const XLOG_NEXTOID: u8 = 0x30;
 pub const XLOG_SWITCH: u8 = 0x40;
 pub const XLOG_SMGR_TRUNCATE: u8 = 0x20;
+pub const DB_SHUTDOWNED: u32 = 1;
 
 // From multixact.h
+pub const FIRST_MULTIXACT_ID: u32 = 1;
+pub const MAX_MULTIXACT_ID: u32 = 0xFFFFFFFF;
+
 pub const XLOG_MULTIXACT_ZERO_OFF_PAGE: u8 = 0x00;
 pub const XLOG_MULTIXACT_ZERO_MEM_PAGE: u8 = 0x10;
 pub const XLOG_MULTIXACT_CREATE_ID: u8 = 0x20;
@@ -176,3 +179,46 @@ pub const FIRST_NORMAL_OBJECT_ID: u32 = 16384;
 
 /* FIXME: pageserver should request wal_seg_size from compute node */
 pub const WAL_SEGMENT_SIZE: usize = 16 * 1024 * 1024;
+
+pub const XLOG_BLCKSZ: usize = 8192;
+pub const XLOG_CHECKPOINT_SHUTDOWN: u8 = 0x00;
+pub const XLOG_CHECKPOINT_ONLINE: u8 = 0x10;
+pub const XLP_LONG_HEADER: u16 = 0x0002;
+
+pub const PG_MAJORVERSION: &'static str = "14";
+
+// List of subdirectories inside pgdata.
+// Copied from src/bin/initdb/initdb.c
+pub const PGDATA_SUBDIRS: [&'static str; 22] = [
+    "global",
+    "pg_wal/archive_status",
+    "pg_commit_ts",
+    "pg_dynshmem",
+    "pg_notify",
+    "pg_serial",
+    "pg_snapshots",
+    "pg_subtrans",
+    "pg_twophase",
+    "pg_multixact",
+    "pg_multixact/members",
+    "pg_multixact/offsets",
+    "base",
+    "base/1",
+    "pg_replslot",
+    "pg_tblspc",
+    "pg_stat",
+    "pg_stat_tmp",
+    "pg_xact",
+    "pg_logical",
+    "pg_logical/snapshots",
+    "pg_logical/mappings",
+];
+
+pub const PGDATA_SPECIAL_FILES: [&'static str; 4] = [
+    "pg_hba.conf",
+    "pg_ident.conf",
+    "postgresql.conf",
+    "postgresql.auto.conf",
+];
+
+pub static PG_HBA: &'static str = include_str!("../samples/pg_hba.conf");

postgres_ffi/src/relfile_utils.rs

@@ -38,16 +38,6 @@ pub fn forknumber_to_name(forknum: u8) -> Option<&'static str> {
         pg_constants::FSM_FORKNUM => Some("fsm"),
         pg_constants::VISIBILITYMAP_FORKNUM => Some("vm"),
         pg_constants::INIT_FORKNUM => Some("init"),
-
-        // These should not appear in WAL records, but we use them internally,
-        // and need to be prepared to print them out in log messages and such
-        pg_constants::PG_CONTROLFILE_FORKNUM => Some("controlfile"),
-        pg_constants::PG_FILENODEMAP_FORKNUM => Some("filenodemap"),
-        pg_constants::PG_XACT_FORKNUM => Some("xact"),
-        pg_constants::PG_MXACT_OFFSETS_FORKNUM => Some("mxact_offsets"),
-        pg_constants::PG_MXACT_MEMBERS_FORKNUM => Some("mxact_members"),
-        pg_constants::PG_TWOPHASE_FORKNUM => Some("twophase"),
-
         _ => Some("UNKNOWN FORKNUM"),
     }
 }

postgres_ffi/src/xlog_utils.rs

@@ -8,8 +8,17 @@
 //
 
 use crate::pg_constants;
+use crate::CheckPoint;
+use crate::ControlFileData;
+use crate::FullTransactionId;
+use crate::XLogLongPageHeaderData;
+use crate::XLogPageHeaderData;
+use crate::XLogRecord;
+use crate::XLOG_PAGE_MAGIC;
+
 use byteorder::{ByteOrder, LittleEndian};
 use bytes::{Buf, Bytes};
+use bytes::{BufMut, BytesMut};
 use crc32c::*;
 use log::*;
 use std::cmp::min;
@@ -21,19 +30,22 @@ use std::time::SystemTime;
 pub const XLOG_FNAME_LEN: usize = 24;
 pub const XLOG_BLCKSZ: usize = 8192;
 pub const XLP_FIRST_IS_CONTRECORD: u16 = 0x0001;
-pub const XLOG_PAGE_MAGIC: u16 = 0xD109;
 pub const XLP_REM_LEN_OFFS: usize = 2 + 2 + 4 + 8;
-pub const XLOG_SIZE_OF_XLOG_SHORT_PHD: usize = XLP_REM_LEN_OFFS + 4 + 4;
-pub const XLOG_SIZE_OF_XLOG_LONG_PHD: usize = XLOG_SIZE_OF_XLOG_SHORT_PHD + 8 + 4 + 4;
 pub const XLOG_RECORD_CRC_OFFS: usize = 4 + 4 + 8 + 1 + 1 + 2;
-pub const XLOG_SIZE_OF_XLOG_RECORD: usize = XLOG_RECORD_CRC_OFFS + 4;
 pub const MAX_SEND_SIZE: usize = XLOG_BLCKSZ * 16;
 
+pub const XLOG_SIZE_OF_XLOG_SHORT_PHD: usize = std::mem::size_of::<XLogPageHeaderData>();
+pub const XLOG_SIZE_OF_XLOG_LONG_PHD: usize = std::mem::size_of::<XLogLongPageHeaderData>();
+pub const XLOG_SIZE_OF_XLOG_RECORD: usize = std::mem::size_of::<XLogRecord>();
+pub const SIZE_OF_XLOG_RECORD_DATA_HEADER_SHORT: usize = 1 * 2;
+
 pub type XLogRecPtr = u64;
 pub type TimeLineID = u32;
-pub type TimestampTz = u64;
+pub type TimestampTz = i64;
 pub type XLogSegNo = u64;
 
+const XID_CHECKPOINT_INTERVAL: u32 = 1024;
+
 #[allow(non_snake_case)]
 pub fn XLogSegmentsPerXLogId(wal_segsz_bytes: usize) -> XLogSegNo {
     (0x100000000u64 / wal_segsz_bytes as u64) as XLogSegNo
@@ -83,9 +95,9 @@ pub fn get_current_timestamp() -> TimestampTz {
     const USECS_PER_SEC: u64 = 1000000;
     match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) {
         Ok(n) => {
-            (n.as_secs() - ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY))
+            ((n.as_secs() - ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY))
                 * USECS_PER_SEC
-                + n.subsec_micros() as u64
+                + n.subsec_micros() as u64) as i64
         }
         Err(_) => panic!("SystemTime before UNIX EPOCH!"),
     }
@@ -96,17 +108,23 @@ fn find_end_of_wal_segment(
     segno: XLogSegNo,
     tli: TimeLineID,
     wal_seg_size: usize,
+    is_partial: bool,
+    rec_offs: &mut usize,
+    rec_hdr: &mut [u8; XLOG_SIZE_OF_XLOG_RECORD],
+    crc: &mut u32,
+    check_contrec: bool,
 ) -> u32 {
     let mut offs: usize = 0;
     let mut contlen: usize = 0;
-    let mut wal_crc: u32 = 0;
-    let mut crc: u32 = 0;
-    let mut rec_offs: usize = 0;
     let mut buf = [0u8; XLOG_BLCKSZ];
     let file_name = XLogFileName(tli, segno, wal_seg_size);
     let mut last_valid_rec_pos: usize = 0;
-    let mut file = File::open(data_dir.join(file_name.clone() + ".partial")).unwrap();
-    let mut rec_hdr = [0u8; XLOG_RECORD_CRC_OFFS];
+    let file_path = data_dir.join(if is_partial {
+        file_name.clone() + ".partial"
+    } else {
+        file_name
+    });
+    let mut file = File::open(&file_path).unwrap();
 
     while offs < wal_seg_size {
         if offs % XLOG_BLCKSZ == 0 {
@@ -120,14 +138,34 @@ fn find_end_of_wal_segment(
             let xlp_magic = LittleEndian::read_u16(&buf[0..2]);
             let xlp_info = LittleEndian::read_u16(&buf[2..4]);
             let xlp_rem_len = LittleEndian::read_u32(&buf[XLP_REM_LEN_OFFS..XLP_REM_LEN_OFFS + 4]);
-            if xlp_magic != XLOG_PAGE_MAGIC {
-                info!("Invalid WAL file {}.partial magic {}", file_name, xlp_magic);
+            if xlp_magic != XLOG_PAGE_MAGIC as u16 {
+                info!("Invalid WAL file {:?} magic {}", &file_path, xlp_magic);
                 break;
             }
             if offs == 0 {
                 offs = XLOG_SIZE_OF_XLOG_LONG_PHD;
                 if (xlp_info & XLP_FIRST_IS_CONTRECORD) != 0 {
-                    offs += ((xlp_rem_len + 7) & !7) as usize;
+                    if check_contrec {
+                        let xl_tot_len = LittleEndian::read_u32(&rec_hdr[0..4]) as usize;
+                        contlen = xlp_rem_len as usize;
+                        if *rec_offs + contlen < xl_tot_len
+                            || (*rec_offs + contlen != xl_tot_len
+                                && contlen != XLOG_BLCKSZ - XLOG_SIZE_OF_XLOG_LONG_PHD)
+                        {
+                            info!(
+                                "Corrupted continuation record: offs={}, contlen={}, xl_tot_len={}",
+                                *rec_offs, contlen, xl_tot_len
+                            );
+                            return 0;
+                        }
+                    } else {
+                        offs += ((xlp_rem_len + 7) & !7) as usize;
+                    }
+                } else if *rec_offs != 0 {
+                    // There is incompleted page at previous segment but no cont record:
+                    // it means that current segment is not valid and we have to return back.
+                    info!("CONTRECORD flag is missed in page header");
+                    return 0;
                 }
             } else {
                 offs += XLOG_SIZE_OF_XLOG_SHORT_PHD;
@@ -138,9 +176,8 @@ fn find_end_of_wal_segment(
             if xl_tot_len == 0 {
                 break;
             }
-            last_valid_rec_pos = offs;
             offs += 4;
-            rec_offs = 4;
+            *rec_offs = 4;
             contlen = xl_tot_len - 4;
             rec_hdr[0..4].copy_from_slice(&buf[page_offs..page_offs + 4]);
         } else {
@@ -150,34 +187,33 @@ fn find_end_of_wal_segment(
 
             // read the rest of the record, or as much as fits on this page.
             let n = min(contlen, pageleft);
-            if rec_offs < XLOG_RECORD_CRC_OFFS {
-                let len = min(XLOG_RECORD_CRC_OFFS - rec_offs, n);
-                rec_hdr[rec_offs..rec_offs + len].copy_from_slice(&buf[page_offs..page_offs + len]);
+            let mut hdr_len: usize = 0;
+            if *rec_offs < XLOG_SIZE_OF_XLOG_RECORD {
+                // copy header
+                hdr_len = min(XLOG_SIZE_OF_XLOG_RECORD - *rec_offs, n);
+                rec_hdr[*rec_offs..*rec_offs + hdr_len]
+                    .copy_from_slice(&buf[page_offs..page_offs + hdr_len]);
             }
-            if rec_offs <= XLOG_RECORD_CRC_OFFS && rec_offs + n >= XLOG_SIZE_OF_XLOG_RECORD {
-                let crc_offs = page_offs - rec_offs + XLOG_RECORD_CRC_OFFS;
-                wal_crc = LittleEndian::read_u32(&buf[crc_offs..crc_offs + 4]);
-                crc = crc32c_append(0, &buf[crc_offs + 4..page_offs + n]);
-                crc = !crc;
-            } else {
-                crc ^= 0xFFFFFFFFu32;
-                crc = crc32c_append(crc, &buf[page_offs..page_offs + n]);
-                crc = !crc;
-            }
-            rec_offs += n;
+            *crc = crc32c_append(*crc, &buf[page_offs + hdr_len..page_offs + n]);
+            *rec_offs += n;
             offs += n;
             contlen -= n;
 
             if contlen == 0 {
-                crc = !crc;
-                crc = crc32c_append(crc, &rec_hdr);
+                *crc = crc32c_append(*crc, &rec_hdr[0..XLOG_RECORD_CRC_OFFS]);
                 offs = (offs + 7) & !7; // pad on 8 bytes boundary */
-                if crc == wal_crc {
+                let wal_crc = LittleEndian::read_u32(
+                    &rec_hdr[XLOG_RECORD_CRC_OFFS..XLOG_RECORD_CRC_OFFS + 4],
+                );
+                if *crc == wal_crc {
                     last_valid_rec_pos = offs;
+                    // Reset rec_offs and crc for start of new record
+                    *rec_offs = 0;
+                    *crc = 0;
                 } else {
                     info!(
-                        "CRC mismatch {} vs {} at {}",
-                        crc, wal_crc, last_valid_rec_pos
+                        "CRC mismatch {} vs {} at offset {} lsn {}",
+                        *crc, wal_crc, offs, last_valid_rec_pos
                     );
                     break;
                 }
@@ -199,51 +235,171 @@ pub fn find_end_of_wal(
     let mut high_tli: TimeLineID = 0;
     let mut high_ispartial = false;
 
-    for entry in fs::read_dir(data_dir).unwrap() {
-        if let Ok(entry) = entry {
-            let ispartial: bool;
-            let entry_name = entry.file_name();
-            let fname = entry_name.to_str().unwrap();
-            /*
-             * Check if the filename looks like an xlog file, or a .partial file.
-             */
-            if IsXLogFileName(fname) {
-                ispartial = false;
-            } else if IsPartialXLogFileName(fname) {
-                ispartial = true;
-            } else {
-                continue;
-            }
-            let (segno, tli) = XLogFromFileName(fname, wal_seg_size);
-            if !ispartial && entry.metadata().unwrap().len() != wal_seg_size as u64 {
-                continue;
-            }
-            if segno > high_segno
-                || (segno == high_segno && tli > high_tli)
-                || (segno == high_segno && tli == high_tli && high_ispartial && !ispartial)
-            {
-                high_segno = segno;
-                high_tli = tli;
-                high_ispartial = ispartial;
-            }
+    for entry in fs::read_dir(data_dir).unwrap().flatten() {
+        let ispartial: bool;
+        let entry_name = entry.file_name();
+        let fname = entry_name.to_str().unwrap();
+        /*
+         * Check if the filename looks like an xlog file, or a .partial file.
+         */
+        if IsXLogFileName(fname) {
+            ispartial = false;
+        } else if IsPartialXLogFileName(fname) {
+            ispartial = true;
+        } else {
+            continue;
+        }
+        let (segno, tli) = XLogFromFileName(fname, wal_seg_size);
+        if !ispartial && entry.metadata().unwrap().len() != wal_seg_size as u64 {
+            continue;
+        }
+        if segno > high_segno
+            || (segno == high_segno && tli > high_tli)
+            || (segno == high_segno && tli == high_tli && high_ispartial && !ispartial)
+        {
+            high_segno = segno;
+            high_tli = tli;
+            high_ispartial = ispartial;
         }
     }
     if high_segno > 0 {
         let mut high_offs = 0;
-        /*
-         * Move the starting pointer to the start of the next segment, if the
-         * highest one we saw was completed.
-         */
-        if !high_ispartial {
-            high_segno += 1;
-        } else if precise {
-            /* otherwise locate last record in last partial segment */
-            high_offs = find_end_of_wal_segment(data_dir, high_segno, high_tli, wal_seg_size);
+        if precise {
+            let mut crc: u32 = 0;
+            let mut rec_offs: usize = 0;
+            let mut rec_hdr = [0u8; XLOG_SIZE_OF_XLOG_RECORD];
+            let wal_dir = data_dir.join("pg_wal");
+
+            /*
+             * To be able to calculate CRC of records crossing segment boundary,
+             * we need to parse previous segments.
+             * So first traverse segments in backward direction to locate record start
+             * and then traverse forward, accumulating CRC.
+             */
+            let mut prev_segno = high_segno - 1;
+            let mut prev_offs: u32 = 0;
+            while prev_segno > 1 {
+                // TOFO: first segment constains dummy checkpoint record at the beginning
+                prev_offs = find_end_of_wal_segment(
+                    data_dir,
+                    prev_segno,
+                    high_tli,
+                    wal_seg_size,
+                    false,
+                    &mut rec_offs,
+                    &mut rec_hdr,
+                    &mut crc,
+                    false,
+                );
+                if prev_offs != 0 {
+                    break;
+                }
+                prev_segno -= 1;
+            }
+            if prev_offs != 0 {
+                // found start of WAL record
+                let first_segno = prev_segno;
+                let first_offs = prev_offs;
+                while prev_segno + 1 < high_segno {
+                    // now traverse record in forward direction, accumulating CRC
+                    prev_segno += 1;
+                    prev_offs = find_end_of_wal_segment(
+                        data_dir,
+                        prev_segno,
+                        high_tli,
+                        wal_seg_size,
+                        false,
+                        &mut rec_offs,
+                        &mut rec_hdr,
+                        &mut crc,
+                        true,
+                    );
+                    if prev_offs == 0 {
+                        info!("Segment {} is corrupted", prev_segno,);
+                        break;
+                    }
+                }
+                if prev_offs != 0 {
+                    high_offs = find_end_of_wal_segment(
+                        data_dir,
+                        high_segno,
+                        high_tli,
+                        wal_seg_size,
+                        high_ispartial,
+                        &mut rec_offs,
+                        &mut rec_hdr,
+                        &mut crc,
+                        true,
+                    );
+                }
+                if high_offs == 0 {
+                    // If last segment contais no valid records, then return back
+                    info!("Last WAL segment {} contains no valid record, truncate WAL till {} segment",
+						  high_segno, first_segno);
+                    // Remove last segments containing corrupted WAL record
+                    for segno in first_segno + 1..high_segno {
+                        let file_name = XLogFileName(high_tli, segno, wal_seg_size);
+                        let file_path = wal_dir.join(file_name);
+                        if let Err(e) = fs::remove_file(&file_path) {
+                            info!("Failed to remove file {:?}: {}", &file_path, e);
+                        }
+                    }
+                    let file_name = XLogFileName(high_tli, high_segno, wal_seg_size);
+                    let file_path = if high_ispartial {
+                        wal_dir.join(file_name.clone() + ".partial")
+                    } else {
+                        wal_dir.join(file_name.clone())
+                    };
+                    if let Err(e) = fs::remove_file(&file_path) {
+                        info!("Failed to remove file {:?}: {}", &file_path, e);
+                    }
+                    high_ispartial = false; // previous segment should not be partial
+                    high_segno = first_segno;
+                    high_offs = first_offs;
+                }
+            } else {
+                // failed to locate previous segment
+                assert!(prev_segno <= 1);
+                high_offs = find_end_of_wal_segment(
+                    data_dir,
+                    high_segno,
+                    high_tli,
+                    wal_seg_size,
+                    high_ispartial,
+                    &mut rec_offs,
+                    &mut rec_hdr,
+                    &mut crc,
+                    false,
+                );
+            }
+
+            // If last segment is not marked as partial, it means that next segment
+            // was not written. Let's make this segment partial once again.
+            if !high_ispartial {
+                let file_name = XLogFileName(high_tli, high_segno, wal_seg_size);
+                if let Err(e) = fs::rename(
+                    wal_dir.join(file_name.clone()),
+                    wal_dir.join(file_name.clone() + ".partial"),
+                ) {
+                    info!(
+                        "Failed to rename {} to {}.partial: {}",
+                        &file_name, &file_name, e
+                    );
+                }
+            }
+        } else {
+            /*
+             * Move the starting pointer to the start of the next segment, if the
+             * highest one we saw was completed.
+             */
+            if !high_ispartial {
+                high_segno += 1;
+            }
         }
         let high_ptr = XLogSegNoOffsetToRecPtr(high_segno, high_offs, wal_seg_size);
         return (high_ptr, high_tli);
     }
-    (0, 0)
+    (0, 1) // First timeline is 1
 }
 
 pub fn main() {
@@ -259,21 +415,6 @@ pub fn main() {
     );
 }
 
-//
-// Xlog record parsing routines
-// TODO move here other related code from waldecoder.rs
-//
-#[repr(C)]
-#[derive(Debug)]
-pub struct XLogRecord {
-    pub xl_tot_len: u32,
-    pub xl_xid: u32,
-    pub xl_prev: u64,
-    pub xl_info: u8,
-    pub xl_rmid: u8,
-    pub xl_crc: u32,
-}
-
 impl XLogRecord {
     pub fn from_bytes(buf: &mut Bytes) -> XLogRecord {
         XLogRecord {
@@ -289,8 +430,217 @@ impl XLogRecord {
         }
     }
 
+    pub fn encode(&self) -> Bytes {
+        let b: [u8; XLOG_SIZE_OF_XLOG_RECORD];
+        b = unsafe { std::mem::transmute::<XLogRecord, [u8; XLOG_SIZE_OF_XLOG_RECORD]>(*self) };
+        Bytes::copy_from_slice(&b[..])
+    }
+
     // Is this record an XLOG_SWITCH record? They need some special processing,
     pub fn is_xlog_switch_record(&self) -> bool {
         self.xl_info == pg_constants::XLOG_SWITCH && self.xl_rmid == pg_constants::RM_XLOG_ID
     }
 }
+
+impl XLogPageHeaderData {
+    pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogPageHeaderData {
+        let hdr: XLogPageHeaderData = XLogPageHeaderData {
+            xlp_magic: buf.get_u16_le(),
+            xlp_info: buf.get_u16_le(),
+            xlp_tli: buf.get_u32_le(),
+            xlp_pageaddr: buf.get_u64_le(),
+            xlp_rem_len: buf.get_u32_le(),
+        };
+        buf.get_u32_le(); //padding
+        hdr
+    }
+}
+
+impl XLogLongPageHeaderData {
+    pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogLongPageHeaderData {
+        XLogLongPageHeaderData {
+            std: XLogPageHeaderData::from_bytes(buf),
+            xlp_sysid: buf.get_u64_le(),
+            xlp_seg_size: buf.get_u32_le(),
+            xlp_xlog_blcksz: buf.get_u32_le(),
+        }
+    }
+
+    pub fn encode(&self) -> Bytes {
+        let b: [u8; XLOG_SIZE_OF_XLOG_LONG_PHD];
+        b = unsafe {
+            std::mem::transmute::<XLogLongPageHeaderData, [u8; XLOG_SIZE_OF_XLOG_LONG_PHD]>(*self)
+        };
+        Bytes::copy_from_slice(&b[..])
+    }
+}
+
+pub const SIZEOF_CHECKPOINT: usize = std::mem::size_of::<CheckPoint>();
+
+impl CheckPoint {
+    pub fn encode(&self) -> Bytes {
+        let b: [u8; SIZEOF_CHECKPOINT];
+        b = unsafe { std::mem::transmute::<CheckPoint, [u8; SIZEOF_CHECKPOINT]>(*self) };
+        Bytes::copy_from_slice(&b[..])
+    }
+
+    pub fn decode(buf: &[u8]) -> Result<CheckPoint, anyhow::Error> {
+        let mut b = [0u8; SIZEOF_CHECKPOINT];
+        b.copy_from_slice(&buf[0..SIZEOF_CHECKPOINT]);
+        let checkpoint: CheckPoint;
+        checkpoint = unsafe { std::mem::transmute::<[u8; SIZEOF_CHECKPOINT], CheckPoint>(b) };
+        Ok(checkpoint)
+    }
+
+    // Update next XID based on provided new_xid and stored epoch.
+    // Next XID should be greater than new_xid.
+    // Also take in account 32-bit wrap-around.
+    pub fn update_next_xid(&mut self, xid: u32) {
+        let xid = xid.wrapping_add(XID_CHECKPOINT_INTERVAL - 1) & !(XID_CHECKPOINT_INTERVAL - 1);
+        let full_xid = self.nextXid.value;
+        let new_xid = std::cmp::max(xid + 1, pg_constants::FIRST_NORMAL_TRANSACTION_ID);
+        let old_xid = full_xid as u32;
+        if new_xid.wrapping_sub(old_xid) as i32 > 0 {
+            let mut epoch = full_xid >> 32;
+            if new_xid < old_xid {
+                // wrap-around
+                epoch += 1;
+            }
+            self.nextXid = FullTransactionId {
+                value: (epoch << 32) | new_xid as u64,
+            };
+        }
+    }
+}
+
+//
+// Generate new WAL segment with single XLOG_CHECKPOINT_SHUTDOWN record.
+// We need this segment to start compute node.
+// In order to minimize changes in Postgres core, we prefer to
+// provide WAL segment from which is can extract checkpoint record in standard way,
+// rather then implement some alternative mechanism.
+//
+pub fn generate_wal_segment(pg_control: &ControlFileData) -> Bytes {
+    let mut seg_buf = BytesMut::with_capacity(pg_constants::WAL_SEGMENT_SIZE as usize);
+
+    let hdr = XLogLongPageHeaderData {
+        std: {
+            XLogPageHeaderData {
+                xlp_magic: XLOG_PAGE_MAGIC as u16,
+                xlp_info: pg_constants::XLP_LONG_HEADER,
+                xlp_tli: 1, // FIXME: always use Postgres timeline 1
+                xlp_pageaddr: pg_control.checkPoint - XLOG_SIZE_OF_XLOG_LONG_PHD as u64,
+                xlp_rem_len: 0,
+            }
+        },
+        xlp_sysid: pg_control.system_identifier,
+        xlp_seg_size: pg_constants::WAL_SEGMENT_SIZE as u32,
+        xlp_xlog_blcksz: XLOG_BLCKSZ as u32,
+    };
+
+    let hdr_bytes = hdr.encode();
+    seg_buf.extend_from_slice(&hdr_bytes);
+
+    let rec_hdr = XLogRecord {
+        xl_tot_len: (XLOG_SIZE_OF_XLOG_RECORD
+            + SIZE_OF_XLOG_RECORD_DATA_HEADER_SHORT
+            + SIZEOF_CHECKPOINT) as u32,
+        xl_xid: 0, //0 is for InvalidTransactionId
+        xl_prev: 0,
+        xl_info: pg_constants::XLOG_CHECKPOINT_SHUTDOWN,
+        xl_rmid: pg_constants::RM_XLOG_ID,
+        xl_crc: 0,
+    };
+
+    let mut rec_shord_hdr_bytes = BytesMut::new();
+    rec_shord_hdr_bytes.put_u8(pg_constants::XLR_BLOCK_ID_DATA_SHORT);
+    rec_shord_hdr_bytes.put_u8(SIZEOF_CHECKPOINT as u8);
+
+    let rec_bytes = rec_hdr.encode();
+    let checkpoint_bytes = pg_control.checkPointCopy.encode();
+
+    //calculate record checksum
+    let mut crc = 0;
+    crc = crc32c_append(crc, &rec_shord_hdr_bytes[..]);
+    crc = crc32c_append(crc, &checkpoint_bytes[..]);
+    crc = crc32c_append(crc, &rec_bytes[0..XLOG_RECORD_CRC_OFFS]);
+
+    seg_buf.extend_from_slice(&rec_bytes[0..XLOG_RECORD_CRC_OFFS]);
+    seg_buf.put_u32_le(crc);
+    seg_buf.extend_from_slice(&rec_shord_hdr_bytes);
+    seg_buf.extend_from_slice(&checkpoint_bytes);
+
+    //zero out the rest of the file
+    seg_buf.resize(pg_constants::WAL_SEGMENT_SIZE, 0);
+    seg_buf.freeze()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use regex::Regex;
+    use std::{env, process::Command, str::FromStr};
+    use zenith_utils::lsn::Lsn;
+
+    // Run find_end_of_wal against file in test_wal dir
+    // Ensure that it finds last record correctly
+    #[test]
+    pub fn test_find_end_of_wal() {
+        // 1. Run initdb to generate some WAL
+        let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
+        let data_dir = top_path.join("test_output/test_find_end_of_wal");
+        let initdb_path = top_path.join("tmp_install/bin/initdb");
+        let lib_path = top_path.join("tmp_install/lib");
+        if data_dir.exists() {
+            fs::remove_dir_all(&data_dir).unwrap();
+        }
+        println!("Using initdb from '{}'", initdb_path.display());
+        println!("Data directory '{}'", data_dir.display());
+        let initdb_output = Command::new(initdb_path)
+            .args(&["-D", data_dir.to_str().unwrap()])
+            .arg("--no-instructions")
+            .arg("--no-sync")
+            .env_clear()
+            .env("LD_LIBRARY_PATH", &lib_path)
+            .env("DYLD_LIBRARY_PATH", &lib_path)
+            .output()
+            .unwrap();
+        assert!(initdb_output.status.success());
+
+        // 2. Pick WAL generated by initdb
+        let wal_dir = data_dir.join("pg_wal");
+        let wal_seg_size = 16 * 1024 * 1024;
+
+        // 3. Check end_of_wal on non-partial WAL segment (we treat it as fully populated)
+        let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true);
+        let wal_end = Lsn(wal_end);
+        println!("wal_end={}, tli={}", wal_end, tli);
+        assert_eq!(wal_end, "0/1699D10".parse::<Lsn>().unwrap());
+
+        // 4. Get the actual end of WAL by pg_waldump
+        let waldump_path = top_path.join("tmp_install/bin/pg_waldump");
+        let waldump_output = Command::new(waldump_path)
+            .arg(wal_dir.join("000000010000000000000001"))
+            .env_clear()
+            .env("LD_LIBRARY_PATH", &lib_path)
+            .env("DYLD_LIBRARY_PATH", &lib_path)
+            .output()
+            .unwrap();
+        let waldump_output = std::str::from_utf8(&waldump_output.stderr).unwrap();
+        println!("waldump_output = '{}'", &waldump_output);
+        let re = Regex::new(r"invalid record length at (.+):").unwrap();
+        let caps = re.captures(&waldump_output).unwrap();
+        let waldump_wal_end = Lsn::from_str(caps.get(1).unwrap().as_str()).unwrap();
+
+        // 5. Rename file to partial to actually find last valid lsn
+        fs::rename(
+            wal_dir.join("000000010000000000000001"),
+            wal_dir.join("000000010000000000000001.partial"),
+        )
+        .unwrap();
+        let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true);
+        let wal_end = Lsn(wal_end);
+        println!("wal_end={}, tli={}", wal_end, tli);
+        assert_eq!(wal_end, waldump_wal_end);
+    }
+}

proxy/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "proxy"
+version = "0.1.0"
+authors = ["Stas Kelvich <stas.kelvich@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0"
+bytes = { version = "1.0.1", features = ['serde'] }
+md5 = "0.7.0"
+rand = "0.8.3"
+hex = "0.4.3"
+serde = "1"
+serde_json = "1"
+tokio = { version = "1.7.1", features = ["full"] }
+tokio-postgres = "0.7.2"
+clap = "2.33.0"
+
+zenith_utils = { path = "../zenith_utils" }

proxy/src/cplane_api.rs

@@ -0,0 +1,92 @@
+use anyhow::{bail, Result};
+use serde::{Deserialize, Serialize};
+use std::{
+    collections::HashMap,
+    net::{IpAddr, SocketAddr},
+};
+
+pub struct CPlaneApi {
+    // address: SocketAddr,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct DatabaseInfo {
+    pub host: IpAddr, // TODO: allow host name here too
+    pub port: u16,
+    pub dbname: String,
+    pub user: String,
+    pub password: String,
+}
+
+impl DatabaseInfo {
+    pub fn socket_addr(&self) -> SocketAddr {
+        SocketAddr::new(self.host, self.port)
+    }
+
+    pub fn conn_string(&self) -> String {
+        format!(
+            "dbname={} user={} password={}",
+            self.dbname, self.user, self.password
+        )
+    }
+}
+
+// mock cplane api
+impl CPlaneApi {
+    pub fn new(_address: &SocketAddr) -> CPlaneApi {
+        CPlaneApi {
+            // address: address.clone(),
+        }
+    }
+
+    pub fn check_auth(&self, user: &str, md5_response: &[u8], salt: &[u8; 4]) -> Result<()> {
+        // passwords for both is "mypass"
+        let auth_map: HashMap<_, &str> = vec![
+            ("stas@zenith", "716ee6e1c4a9364d66285452c47402b1"),
+            ("stas2@zenith", "3996f75df64c16a8bfaf01301b61d582"),
+        ]
+        .into_iter()
+        .collect();
+
+        let stored_hash = auth_map
+            .get(&user)
+            .ok_or_else(|| anyhow::Error::msg("user not found"))?;
+        let salted_stored_hash = format!(
+            "md5{:x}",
+            md5::compute([stored_hash.as_bytes(), salt].concat())
+        );
+
+        let received_hash = std::str::from_utf8(&md5_response)?;
+
+        println!(
+            "auth: {} rh={} sh={} ssh={} {:?}",
+            user, received_hash, stored_hash, salted_stored_hash, salt
+        );
+
+        if received_hash == salted_stored_hash {
+            Ok(())
+        } else {
+            bail!("Auth failed")
+        }
+    }
+
+    pub fn get_database_uri(&self, _user: &str, _database: &str) -> Result<DatabaseInfo> {
+        Ok(DatabaseInfo {
+            host: "127.0.0.1".parse()?,
+            port: 5432,
+            dbname: "stas".to_string(),
+            user: "stas".to_string(),
+            password: "mypass".to_string(),
+        })
+    }
+
+    // pub fn create_database(&self, _user: &String, _database: &String) -> Result<DatabaseInfo> {
+    //     Ok(DatabaseInfo {
+    //         host: "127.0.0.1".parse()?,
+    //         port: 5432,
+    //         dbname: "stas".to_string(),
+    //         user: "stas".to_string(),
+    //         password: "mypass".to_string(),
+    //     })
+    // }
+}

proxy/src/main.rs

@@ -0,0 +1,106 @@
+///
+/// Postgres protocol proxy/router.
+///
+/// This service listens psql port and can check auth via external service
+/// (control plane API in our case) and can create new databases and accounts
+/// in somewhat transparent manner (again via communication with control plane API).
+///
+use std::{
+    collections::HashMap,
+    net::{SocketAddr, TcpListener},
+    sync::{mpsc, Mutex},
+    thread,
+};
+
+use clap::{App, Arg};
+
+use cplane_api::DatabaseInfo;
+
+mod cplane_api;
+mod mgmt;
+mod proxy;
+
+pub struct ProxyConf {
+    /// main entrypoint for users to connect to
+    pub proxy_address: SocketAddr,
+
+    /// http management endpoint. Upon user account creation control plane
+    /// will notify us here, so that we can 'unfreeze' user session.
+    pub mgmt_address: SocketAddr,
+
+    /// send unauthenticated users to this URI
+    pub redirect_uri: String,
+
+    /// control plane address where we would check auth.
+    pub cplane_address: SocketAddr,
+}
+
+pub struct ProxyState {
+    pub conf: ProxyConf,
+    pub waiters: Mutex<HashMap<String, mpsc::Sender<anyhow::Result<DatabaseInfo>>>>,
+}
+
+fn main() -> anyhow::Result<()> {
+    let arg_matches = App::new("Zenith proxy/router")
+        .arg(
+            Arg::with_name("proxy")
+                .short("p")
+                .long("proxy")
+                .takes_value(true)
+                .help("listen for incoming client connections on ip:port")
+                .default_value("127.0.0.1:4432"),
+        )
+        .arg(
+            Arg::with_name("mgmt")
+                .short("m")
+                .long("mgmt")
+                .takes_value(true)
+                .help("listen for management callback connection on ip:port")
+                .default_value("127.0.0.1:7000"),
+        )
+        .arg(
+            Arg::with_name("uri")
+                .short("u")
+                .long("uri")
+                .takes_value(true)
+                .help("redirect unauthenticated users to given uri")
+                .default_value("http://localhost:3000/psql_session/"),
+        )
+        .get_matches();
+
+    let conf = ProxyConf {
+        proxy_address: arg_matches.value_of("proxy").unwrap().parse()?,
+        mgmt_address: arg_matches.value_of("mgmt").unwrap().parse()?,
+        redirect_uri: arg_matches.value_of("uri").unwrap().parse()?,
+        cplane_address: "127.0.0.1:3000".parse()?,
+    };
+    let state = ProxyState {
+        conf,
+        waiters: Mutex::new(HashMap::new()),
+    };
+    let state: &'static ProxyState = Box::leak(Box::new(state));
+
+    // Check that we can bind to address before further initialization
+    println!("Starting proxy on {}", state.conf.proxy_address);
+    let pageserver_listener = TcpListener::bind(state.conf.proxy_address)?;
+
+    println!("Starting mgmt on {}", state.conf.mgmt_address);
+    let mgmt_listener = TcpListener::bind(state.conf.mgmt_address)?;
+
+    let threads = vec![
+        // Spawn a thread to listen for connections. It will spawn further threads
+        // for each connection.
+        thread::Builder::new()
+            .name("Proxy thread".into())
+            .spawn(move || proxy::thread_main(&state, pageserver_listener))?,
+        thread::Builder::new()
+            .name("Mgmt thread".into())
+            .spawn(move || mgmt::thread_main(&state, mgmt_listener))?,
+    ];
+
+    for t in threads.into_iter() {
+        t.join().unwrap()?;
+    }
+
+    Ok(())
+}

proxy/src/mgmt.rs

@@ -0,0 +1,111 @@
+use std::{
+    net::{TcpListener, TcpStream},
+    thread,
+};
+
+use anyhow::bail;
+use bytes::Bytes;
+use serde::{Deserialize, Serialize};
+use zenith_utils::{
+    postgres_backend::{self, query_from_cstring, AuthType, PostgresBackend},
+    pq_proto::{BeMessage, SINGLE_COL_ROWDESC},
+};
+
+use crate::{cplane_api::DatabaseInfo, ProxyState};
+
+///
+/// Main proxy listener loop.
+///
+/// Listens for connections, and launches a new handler thread for each.
+///
+pub fn thread_main(state: &'static ProxyState, listener: TcpListener) -> anyhow::Result<()> {
+    loop {
+        let (socket, peer_addr) = listener.accept()?;
+        println!("accepted connection from {}", peer_addr);
+        socket.set_nodelay(true).unwrap();
+
+        thread::spawn(move || {
+            if let Err(err) = mgmt_conn_main(state, socket) {
+                println!("error: {}", err);
+            }
+        });
+    }
+}
+
+pub fn mgmt_conn_main(state: &'static ProxyState, socket: TcpStream) -> anyhow::Result<()> {
+    let mut conn_handler = MgmtHandler { state };
+    let pgbackend = PostgresBackend::new(socket, AuthType::Trust)?;
+    pgbackend.run(&mut conn_handler)
+}
+
+struct MgmtHandler {
+    state: &'static ProxyState,
+}
+/// Serialized examples:
+// {
+//     "session_id": "71d6d03e6d93d99a",
+//     "result": {
+//         "Success": {
+//             "host": "127.0.0.1",
+//             "port": 5432,
+//             "dbname": "stas",
+//             "user": "stas"
+//             "password": "mypass"
+//         }
+//     }
+// }
+// {
+//     "session_id": "71d6d03e6d93d99a",
+//     "result": {
+//         "Failure": "oops"
+//     }
+// }
+#[derive(Serialize, Deserialize)]
+pub struct PsqlSessionResponse {
+    session_id: String,
+    result: PsqlSessionResult,
+}
+
+#[derive(Serialize, Deserialize)]
+pub enum PsqlSessionResult {
+    Success(DatabaseInfo),
+    Failure(String),
+}
+
+impl postgres_backend::Handler for MgmtHandler {
+    fn process_query(
+        &mut self,
+        pgb: &mut PostgresBackend,
+        query_string: Bytes,
+    ) -> anyhow::Result<()> {
+        let query_string = query_from_cstring(query_string);
+
+        println!("Got mgmt query: '{}'", std::str::from_utf8(&query_string)?);
+
+        let resp: PsqlSessionResponse = serde_json::from_slice(&query_string)?;
+
+        let waiters = self.state.waiters.lock().unwrap();
+
+        let sender = waiters
+            .get(&resp.session_id)
+            .ok_or_else(|| anyhow::Error::msg("psql_session_id is not found"))?;
+
+        match resp.result {
+            PsqlSessionResult::Success(db_info) => {
+                sender.send(Ok(db_info))?;
+
+                pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
+                    .write_message_noflush(&BeMessage::DataRow(&[Some(b"ok")]))?
+                    .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
+                pgb.flush()?;
+                Ok(())
+            }
+
+            PsqlSessionResult::Failure(message) => {
+                sender.send(Err(anyhow::Error::msg(message.clone())))?;
+
+                bail!("psql session request failed: {}", message)
+            }
+        }
+    }
+}

proxy/src/proxy.rs

@@ -0,0 +1,256 @@
+use crate::cplane_api::CPlaneApi;
+use crate::cplane_api::DatabaseInfo;
+use crate::ProxyState;
+
+use anyhow::bail;
+use tokio_postgres::NoTls;
+
+use rand::Rng;
+use std::sync::mpsc::channel;
+use std::thread;
+use tokio::io::AsyncWriteExt;
+use zenith_utils::postgres_backend::{PostgresBackend, ProtoState};
+use zenith_utils::pq_proto::*;
+use zenith_utils::{postgres_backend, pq_proto::BeMessage};
+
+///
+/// Main proxy listener loop.
+///
+/// Listens for connections, and launches a new handler thread for each.
+///
+pub fn thread_main(
+    state: &'static ProxyState,
+    listener: std::net::TcpListener,
+) -> anyhow::Result<()> {
+    loop {
+        let (socket, peer_addr) = listener.accept()?;
+        println!("accepted connection from {}", peer_addr);
+        socket.set_nodelay(true).unwrap();
+
+        thread::spawn(move || {
+            if let Err(err) = proxy_conn_main(state, socket) {
+                println!("error: {}", err);
+            }
+        });
+    }
+}
+
+// XXX: clean up fields
+struct ProxyConnection {
+    state: &'static ProxyState,
+
+    cplane: CPlaneApi,
+
+    user: String,
+    database: String,
+
+    pgb: PostgresBackend,
+    md5_salt: [u8; 4],
+
+    psql_session_id: String,
+}
+
+pub fn proxy_conn_main(
+    state: &'static ProxyState,
+    socket: std::net::TcpStream,
+) -> anyhow::Result<()> {
+    let mut conn = ProxyConnection {
+        state,
+        cplane: CPlaneApi::new(&state.conf.cplane_address),
+        user: "".into(),
+        database: "".into(),
+        pgb: PostgresBackend::new(socket, postgres_backend::AuthType::MD5)?,
+        md5_salt: [0u8; 4],
+        psql_session_id: "".into(),
+    };
+
+    // Check StartupMessage
+    // This will set conn.existing_user and we can decide on next actions
+    conn.handle_startup()?;
+
+    // both scenarious here should end up producing database connection string
+    let db_info = if conn.is_existing_user() {
+        conn.handle_existing_user()?
+    } else {
+        conn.handle_new_user()?
+    };
+
+    // ok, proxy pass user connection to database_uri
+    let runtime = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+
+    let _ = runtime.block_on(proxy_pass(conn.pgb, db_info))?;
+
+    println!("proxy_conn_main done;");
+
+    Ok(())
+}
+
+impl ProxyConnection {
+    fn is_existing_user(&self) -> bool {
+        self.user.ends_with("@zenith")
+    }
+
+    fn handle_startup(&mut self) -> anyhow::Result<()> {
+        loop {
+            let msg = self.pgb.read_message()?;
+            println!("got message {:?}", msg);
+            match msg {
+                Some(FeMessage::StartupMessage(m)) => {
+                    println!("got startup message {:?}", m);
+
+                    match m.kind {
+                        StartupRequestCode::NegotiateGss | StartupRequestCode::NegotiateSsl => {
+                            println!("SSL requested");
+                            self.pgb.write_message(&BeMessage::Negotiate)?;
+                        }
+                        StartupRequestCode::Normal => {
+                            self.user = m
+                                .params
+                                .get("user")
+                                .ok_or_else(|| {
+                                    anyhow::Error::msg("user is required in startup packet")
+                                })?
+                                .into();
+                            self.database = m
+                                .params
+                                .get("database")
+                                .ok_or_else(|| {
+                                    anyhow::Error::msg("database is required in startup packet")
+                                })?
+                                .into();
+
+                            break;
+                        }
+                        StartupRequestCode::Cancel => break,
+                    }
+                }
+                None => {
+                    bail!("connection closed")
+                }
+                unexpected => {
+                    bail!("unexpected message type : {:?}", unexpected)
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn handle_existing_user(&mut self) -> anyhow::Result<DatabaseInfo> {
+        // ask password
+        rand::thread_rng().fill(&mut self.md5_salt);
+        self.pgb
+            .write_message(&BeMessage::AuthenticationMD5Password(&self.md5_salt))?;
+        self.pgb.state = ProtoState::Authentication; // XXX
+
+        // check password
+        println!("handle_existing_user");
+        let msg = self.pgb.read_message()?;
+        println!("got message {:?}", msg);
+        if let Some(FeMessage::PasswordMessage(m)) = msg {
+            println!("got password message '{:?}'", m);
+
+            assert!(self.is_existing_user());
+
+            let (_trailing_null, md5_response) = m
+                .split_last()
+                .ok_or_else(|| anyhow::Error::msg("unexpected password message"))?;
+
+            if let Err(e) = self.check_auth_md5(md5_response) {
+                self.pgb
+                    .write_message(&BeMessage::ErrorResponse(format!("{}", e)))?;
+                bail!("auth failed: {}", e);
+            } else {
+                self.pgb
+                    .write_message_noflush(&BeMessage::AuthenticationOk)?;
+                self.pgb
+                    .write_message_noflush(&BeMessage::ParameterStatus)?;
+                self.pgb.write_message(&BeMessage::ReadyForQuery)?;
+            }
+        }
+
+        // ok, we are authorized
+        self.cplane.get_database_uri(&self.user, &self.database)
+    }
+
+    fn handle_new_user(&mut self) -> anyhow::Result<DatabaseInfo> {
+        let mut psql_session_id_buf = [0u8; 8];
+        rand::thread_rng().fill(&mut psql_session_id_buf);
+        self.psql_session_id = hex::encode(psql_session_id_buf);
+
+        let hello_message = format!("☀️  Welcome to Zenith!
+
+To proceed with database creation open following link:
+
+    {}{}
+
+It needed to be done once and we will send you '.pgpass' file which will allow you to access or create
+databases without opening the browser.
+
+", self.state.conf.redirect_uri,self.psql_session_id);
+
+        self.pgb
+            .write_message_noflush(&BeMessage::AuthenticationOk)?;
+        self.pgb
+            .write_message_noflush(&BeMessage::ParameterStatus)?;
+        self.pgb
+            .write_message(&BeMessage::NoticeResponse(hello_message))?;
+
+        // await for database creation
+        let (tx, rx) = channel::<anyhow::Result<DatabaseInfo>>();
+        let _ = self
+            .state
+            .waiters
+            .lock()
+            .unwrap()
+            .insert(self.psql_session_id.clone(), tx);
+
+        // Wait for web console response
+        // XXX: respond with error to client
+        let dbinfo = rx.recv()??;
+
+        self.pgb.write_message_noflush(&BeMessage::NoticeResponse(
+            "Connecting to database.".to_string(),
+        ))?;
+        self.pgb.write_message(&BeMessage::ReadyForQuery)?;
+
+        Ok(dbinfo)
+    }
+
+    fn check_auth_md5(&self, md5_response: &[u8]) -> anyhow::Result<()> {
+        assert!(self.is_existing_user());
+        self.cplane
+            .check_auth(self.user.as_str(), md5_response, &self.md5_salt)
+    }
+}
+
+async fn proxy_pass(pgb: PostgresBackend, db_info: DatabaseInfo) -> anyhow::Result<()> {
+    let mut socket = tokio::net::TcpStream::connect(db_info.socket_addr()).await?;
+    let config = db_info.conn_string().parse::<tokio_postgres::Config>()?;
+    let _ = config.connect_raw(&mut socket, NoTls).await?;
+
+    println!("Connected to pg, proxying");
+
+    let incoming_std = pgb.into_stream();
+    incoming_std.set_nonblocking(true)?;
+    let mut incoming_conn = tokio::net::TcpStream::from_std(incoming_std)?;
+
+    let (mut ri, mut wi) = incoming_conn.split();
+    let (mut ro, mut wo) = socket.split();
+
+    let client_to_server = async {
+        tokio::io::copy(&mut ri, &mut wo).await?;
+        wo.shutdown().await
+    };
+
+    let server_to_client = async {
+        tokio::io::copy(&mut ro, &mut wi).await?;
+        wi.shutdown().await
+    };
+
+    tokio::try_join!(client_to_server, server_to_client)?;
+
+    Ok(())
+}

run_clippy.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# If you save this in your path under the name "cargo-zclippy" (or whatever
+# name you like), then you can run it as "cargo zclippy" from the shell prompt.
+#
+# If your text editor has rust-analyzer integration, you can also use this new
+# command as a replacement for "cargo check" or "cargo clippy" and see clippy
+# warnings and errors right in the editor.
+# In vscode, this setting is Rust-analyzer>Check On Save:Command
+
+cargo clippy "${@:2}" -- -A clippy::new_without_default -A clippy::manual_range_contains -A clippy::comparison_chain

test_runner/Pipfile

@@ -0,0 +1,19 @@
+[[source]]
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+pytest = ">=6.0.0"
+psycopg2 = "*"
+typing-extensions = "*"
+pyjwt = {extras = ["crypto"], version = "*"}
+
+[dev-packages]
+yapf = "*"
+flake8 = "*"
+mypy = "*"
+
+[requires]
+# we need at least 3.6, but pipenv doesn't allow to say this directly
+python_version = "3"

test_runner/Pipfile.lock

@@ -0,0 +1,287 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "f60a966726bcc19670402ad3fa57396b5dacf0a027544418ceb7cc0d42d94a52"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.python.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "attrs": {
+            "hashes": [
+                "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1",
+                "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==21.2.0"
+        },
+        "cffi": {
+            "hashes": [
+                "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d",
+                "sha256:0c0591bee64e438883b0c92a7bed78f6290d40bf02e54c5bf0978eaf36061771",
+                "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872",
+                "sha256:22b9c3c320171c108e903d61a3723b51e37aaa8c81255b5e7ce102775bd01e2c",
+                "sha256:26bb2549b72708c833f5abe62b756176022a7b9a7f689b571e74c8478ead51dc",
+                "sha256:33791e8a2dc2953f28b8d8d300dde42dd929ac28f974c4b4c6272cb2955cb762",
+                "sha256:3c8d896becff2fa653dc4438b54a5a25a971d1f4110b32bd3068db3722c80202",
+                "sha256:4373612d59c404baeb7cbd788a18b2b2a8331abcc84c3ba40051fcd18b17a4d5",
+                "sha256:487d63e1454627c8e47dd230025780e91869cfba4c753a74fda196a1f6ad6548",
+                "sha256:48916e459c54c4a70e52745639f1db524542140433599e13911b2f329834276a",
+                "sha256:4922cd707b25e623b902c86188aca466d3620892db76c0bdd7b99a3d5e61d35f",
+                "sha256:55af55e32ae468e9946f741a5d51f9896da6b9bf0bbdd326843fec05c730eb20",
+                "sha256:57e555a9feb4a8460415f1aac331a2dc833b1115284f7ded7278b54afc5bd218",
+                "sha256:5d4b68e216fc65e9fe4f524c177b54964af043dde734807586cf5435af84045c",
+                "sha256:64fda793737bc4037521d4899be780534b9aea552eb673b9833b01f945904c2e",
+                "sha256:6d6169cb3c6c2ad50db5b868db6491a790300ade1ed5d1da29289d73bbe40b56",
+                "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224",
+                "sha256:80b06212075346b5546b0417b9f2bf467fea3bfe7352f781ffc05a8ab24ba14a",
+                "sha256:818014c754cd3dba7229c0f5884396264d51ffb87ec86e927ef0be140bfdb0d2",
+                "sha256:8eb687582ed7cd8c4bdbff3df6c0da443eb89c3c72e6e5dcdd9c81729712791a",
+                "sha256:99f27fefe34c37ba9875f224a8f36e31d744d8083e00f520f133cab79ad5e819",
+                "sha256:9f3e33c28cd39d1b655ed1ba7247133b6f7fc16fa16887b120c0c670e35ce346",
+                "sha256:a8661b2ce9694ca01c529bfa204dbb144b275a31685a075ce123f12331be790b",
+                "sha256:a9da7010cec5a12193d1af9872a00888f396aba3dc79186604a09ea3ee7c029e",
+                "sha256:aedb15f0a5a5949ecb129a82b72b19df97bbbca024081ed2ef88bd5c0a610534",
+                "sha256:b315d709717a99f4b27b59b021e6207c64620790ca3e0bde636a6c7f14618abb",
+                "sha256:ba6f2b3f452e150945d58f4badd92310449876c4c954836cfb1803bdd7b422f0",
+                "sha256:c33d18eb6e6bc36f09d793c0dc58b0211fccc6ae5149b808da4a62660678b156",
+                "sha256:c9a875ce9d7fe32887784274dd533c57909b7b1dcadcc128a2ac21331a9765dd",
+                "sha256:c9e005e9bd57bc987764c32a1bee4364c44fdc11a3cc20a40b93b444984f2b87",
+                "sha256:d2ad4d668a5c0645d281dcd17aff2be3212bc109b33814bbb15c4939f44181cc",
+                "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195",
+                "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33",
+                "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f",
+                "sha256:e9dc245e3ac69c92ee4c167fbdd7428ec1956d4e754223124991ef29eb57a09d",
+                "sha256:eb687a11f0a7a1839719edd80f41e459cc5366857ecbed383ff376c4e3cc6afd",
+                "sha256:eb9e2a346c5238a30a746893f23a9535e700f8192a68c07c0258e7ece6ff3728",
+                "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7",
+                "sha256:f0010c6f9d1a4011e429109fda55a225921e3206e7f62a0c22a35344bfd13cca",
+                "sha256:f0c5d1acbfca6ebdd6b1e3eded8d261affb6ddcf2186205518f1428b8569bb99",
+                "sha256:f10afb1004f102c7868ebfe91c28f4a712227fe4cb24974350ace1f90e1febbf",
+                "sha256:f174135f5609428cc6e1b9090f9268f5c8935fddb1b25ccb8255a2d50de6789e",
+                "sha256:f3ebe6e73c319340830a9b2825d32eb6d8475c1dac020b4f0aa774ee3b898d1c",
+                "sha256:f627688813d0a4140153ff532537fbe4afea5a3dffce1f9deb7f91f848a832b5",
+                "sha256:fd4305f86f53dfd8cd3522269ed7fc34856a8ee3709a5e28b2836b2db9d4cd69"
+            ],
+            "version": "==1.14.6"
+        },
+        "cryptography": {
+            "hashes": [
+                "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d",
+                "sha256:1e056c28420c072c5e3cb36e2b23ee55e260cb04eee08f702e0edfec3fb51959",
+                "sha256:240f5c21aef0b73f40bb9f78d2caff73186700bf1bc6b94285699aff98cc16c6",
+                "sha256:26965837447f9c82f1855e0bc8bc4fb910240b6e0d16a664bb722df3b5b06873",
+                "sha256:37340614f8a5d2fb9aeea67fd159bfe4f5f4ed535b1090ce8ec428b2f15a11f2",
+                "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713",
+                "sha256:3d8427734c781ea5f1b41d6589c293089704d4759e34597dce91014ac125aad1",
+                "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177",
+                "sha256:8e56e16617872b0957d1c9742a3f94b43533447fd78321514abbe7db216aa250",
+                "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca",
+                "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d",
+                "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9"
+            ],
+            "version": "==3.4.7"
+        },
+        "iniconfig": {
+            "hashes": [
+                "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
+                "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
+            ],
+            "version": "==1.1.1"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7",
+                "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==21.0"
+        },
+        "pluggy": {
+            "hashes": [
+                "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
+                "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==0.13.1"
+        },
+        "psycopg2": {
+            "hashes": [
+                "sha256:079d97fc22de90da1d370c90583659a9f9a6ee4007355f5825e5f1c70dffc1fa",
+                "sha256:2087013c159a73e09713294a44d0c8008204d06326006b7f652bef5ace66eebb",
+                "sha256:2c992196719fadda59f72d44603ee1a2fdcc67de097eea38d41c7ad9ad246e62",
+                "sha256:7640e1e4d72444ef012e275e7b53204d7fab341fb22bc76057ede22fe6860b25",
+                "sha256:7f91312f065df517187134cce8e395ab37f5b601a42446bdc0f0d51773621854",
+                "sha256:830c8e8dddab6b6716a4bf73a09910c7954a92f40cf1d1e702fb93c8a919cc56",
+                "sha256:89409d369f4882c47f7ea20c42c5046879ce22c1e4ea20ef3b00a4dfc0a7f188",
+                "sha256:bf35a25f1aaa8a3781195595577fcbb59934856ee46b4f252f56ad12b8043bcf",
+                "sha256:de5303a6f1d0a7a34b9d40e4d3bef684ccc44a49bbe3eb85e3c0bffb4a131b7c"
+            ],
+            "index": "pypi",
+            "version": "==2.9.1"
+        },
+        "py": {
+            "hashes": [
+                "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3",
+                "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.10.0"
+        },
+        "pycparser": {
+            "hashes": [
+                "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",
+                "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.20"
+        },
+        "pyjwt": {
+            "extras": [
+                "crypto"
+            ],
+            "hashes": [
+                "sha256:934d73fbba91b0483d3857d1aff50e96b2a892384ee2c17417ed3203f173fca1",
+                "sha256:fba44e7898bbca160a2b2b501f492824fc8382485d3a6f11ba5d0c1937ce6130"
+            ],
+            "index": "pypi",
+            "version": "==2.1.0"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
+                "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
+            ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.4.7"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b",
+                "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890"
+            ],
+            "index": "pypi",
+            "version": "==6.2.4"
+        },
+        "toml": {
+            "hashes": [
+                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
+                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
+            ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==0.10.2"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497",
+                "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342",
+                "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"
+            ],
+            "index": "pypi",
+            "version": "==3.10.0.0"
+        }
+    },
+    "develop": {
+        "flake8": {
+            "hashes": [
+                "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b",
+                "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"
+            ],
+            "index": "pypi",
+            "version": "==3.9.2"
+        },
+        "mccabe": {
+            "hashes": [
+                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
+                "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
+            ],
+            "version": "==0.6.1"
+        },
+        "mypy": {
+            "hashes": [
+                "sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9",
+                "sha256:0aadfb2d3935988ec3815952e44058a3100499f5be5b28c34ac9d79f002a4a9a",
+                "sha256:119bed3832d961f3a880787bf621634ba042cb8dc850a7429f643508eeac97b9",
+                "sha256:1a85e280d4d217150ce8cb1a6dddffd14e753a4e0c3cf90baabb32cefa41b59e",
+                "sha256:3c4b8ca36877fc75339253721f69603a9c7fdb5d4d5a95a1a1b899d8b86a4de2",
+                "sha256:3e382b29f8e0ccf19a2df2b29a167591245df90c0b5a2542249873b5c1d78212",
+                "sha256:42c266ced41b65ed40a282c575705325fa7991af370036d3f134518336636f5b",
+                "sha256:53fd2eb27a8ee2892614370896956af2ff61254c275aaee4c230ae771cadd885",
+                "sha256:704098302473cb31a218f1775a873b376b30b4c18229421e9e9dc8916fd16150",
+                "sha256:7df1ead20c81371ccd6091fa3e2878559b5c4d4caadaf1a484cf88d93ca06703",
+                "sha256:866c41f28cee548475f146aa4d39a51cf3b6a84246969f3759cb3e9c742fc072",
+                "sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457",
+                "sha256:adaeee09bfde366d2c13fe6093a7df5df83c9a2ba98638c7d76b010694db760e",
+                "sha256:b6fb13123aeef4a3abbcfd7e71773ff3ff1526a7d3dc538f3929a49b42be03f0",
+                "sha256:b94e4b785e304a04ea0828759172a15add27088520dc7e49ceade7834275bedb",
+                "sha256:c0df2d30ed496a08de5daed2a9ea807d07c21ae0ab23acf541ab88c24b26ab97",
+                "sha256:c6c2602dffb74867498f86e6129fd52a2770c48b7cd3ece77ada4fa38f94eba8",
+                "sha256:ceb6e0a6e27fb364fb3853389607cf7eb3a126ad335790fa1e14ed02fba50811",
+                "sha256:d9dd839eb0dc1bbe866a288ba3c1afc33a202015d2ad83b31e875b5905a079b6",
+                "sha256:e4dab234478e3bd3ce83bac4193b2ecd9cf94e720ddd95ce69840273bf44f6de",
+                "sha256:ec4e0cd079db280b6bdabdc807047ff3e199f334050db5cbb91ba3e959a67504",
+                "sha256:ecd2c3fe726758037234c93df7e98deb257fd15c24c9180dacf1ef829da5f921",
+                "sha256:ef565033fa5a958e62796867b1df10c40263ea9ded87164d67572834e57a174d"
+            ],
+            "index": "pypi",
+            "version": "==0.910"
+        },
+        "mypy-extensions": {
+            "hashes": [
+                "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
+                "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
+            ],
+            "version": "==0.4.3"
+        },
+        "pycodestyle": {
+            "hashes": [
+                "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068",
+                "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.7.0"
+        },
+        "pyflakes": {
+            "hashes": [
+                "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3",
+                "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.3.1"
+        },
+        "toml": {
+            "hashes": [
+                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
+                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
+            ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==0.10.2"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497",
+                "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342",
+                "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"
+            ],
+            "index": "pypi",
+            "version": "==3.10.0.0"
+        },
+        "yapf": {
+            "hashes": [
+                "sha256:408fb9a2b254c302f49db83c59f9aa0b4b0fd0ec25be3a5c51181327922ff63d",
+                "sha256:e3a234ba8455fe201eaa649cdac872d590089a18b661e39bbac7020978dd9c2e"
+            ],
+            "index": "pypi",
+            "version": "==0.31.0"
+        }
+    }
+}

test_runner/README.md

@@ -4,13 +4,10 @@ This directory contains integration tests.
 
 Prerequisites:
 - Python 3.6 or later
-- Python packages: pytest, psycopg2
-    - pytest 6.0 is required.
-    - __NOTE: `apt install` on Debian/Ubuntu won't work.__
-      They ship a much older version of pytest (and sometimes rename it to
-      `pytest-3`.)
-    - Install using something like this:
-        - `pip3 install pytest psycopg2` (Debian or Ubuntu)
+- Dependencies: install them via `pipenv install`. Note that Debian/Ubuntu
+  packages are stale, as it commonly happens, so manual installation is not
+  recommended.
+  Run `pipenv shell` to activate the venv.
 - Zenith and Postgres binaries
     - See the root README.md for build directions
     - Tests can be run from the git tree; or see the environment variables
@@ -72,7 +69,8 @@ The tests make heavy use of pytest fixtures. You can read about how they work he
 Essentially, this means that each time you see a fixture named as an input parameter, the function with that name will be run and passed as a parameter to the function.
 
 So this code:
-```
+
+```python
 def test_something(zenith_cli, pg_bin):
     pass
 ```
@@ -80,9 +78,11 @@ def test_something(zenith_cli, pg_bin):
 ... will run the fixtures called `zenith_cli` and `pg_bin` and deliver those results to the test function.
 
 Fixtures can't be imported using the normal python syntax. Instead, use this:
-```
+
+```python
 pytest_plugins = ("fixtures.something")
 ```
+
 That will make all the fixtures in the `fixtures/something.py` file available.
 
 Anything that's likely to be used in multiple tests should be built into a fixture.
@@ -90,3 +90,15 @@ Anything that's likely to be used in multiple tests should be built into a fixtu
 Note that fixtures can clean up after themselves if they use the `yield` syntax.
 Cleanup will happen even if the test fails (raises an unhandled exception).
 Python destructors, e.g. `__del__()` aren't recommended for cleanup.
+
+
+### Code quality
+
+Before submitting a patch, please consider:
+
+* Writing a couple of docstrings to clarify the reasoning behind a new test.
+* Running `flake8` (or a linter of your choice, e.g. `pycodestyle`) and fixing possible defects, if any.
+* Formatting the code with `yapf -r -i .` (TODO: implement an opt-in pre-commit hook for that).
+* (Optional) Typechecking the code with `mypy .`. Currently this mostly affects `fixtures/zenith_fixtures.py`.
+
+The tools can be installed with `pipenv install --dev`.

test_runner/batch_others/test_auth.py

@@ -0,0 +1,117 @@
+
+from contextlib import closing
+from pathlib import Path
+from uuid import uuid4
+from dataclasses import dataclass
+import jwt
+import psycopg2
+from fixtures.zenith_fixtures import Postgres, ZenithCli, ZenithPageserver
+import pytest
+
+
+@pytest.fixture
+def pageserver_auth_enabled(zenith_cli: ZenithCli):
+    with ZenithPageserver(zenith_cli).init(enable_auth=True).start() as ps:
+        # For convenience in tests, create a branch from the freshly-initialized cluster.
+        zenith_cli.run(["branch", "empty", "main"])
+        yield ps
+
+
+@dataclass
+class AuthKeys:
+    pub: bytes
+    priv: bytes
+
+    def generate_management_token(self):
+        token = jwt.encode({"scope": "pageserverapi"}, self.priv, algorithm="RS256")
+
+        # jwt.encode can return 'bytes' or 'str', depending on Python version or type
+        # hinting or something (not sure what). If it returned 'bytes', convert it to 'str'
+        # explicitly.
+        if isinstance(token, bytes):
+            token = token.decode()
+
+        return token
+
+    def generate_tenant_token(self, tenant_id):
+        token = jwt.encode({"scope": "tenant", "tenant_id": tenant_id}, self.priv, algorithm="RS256")
+
+        if isinstance(token, bytes):
+            token = token.decode()
+
+        return token
+
+
+@pytest.fixture
+def auth_keys(repo_dir: str) -> AuthKeys:
+    # TODO probably this should be specified in cli config and used in tests for single source of truth
+    pub = (Path(repo_dir) / 'auth_public_key.pem').read_bytes()
+    priv = (Path(repo_dir) / 'auth_private_key.pem').read_bytes()
+    return AuthKeys(pub=pub, priv=priv)
+
+
+def test_pageserver_auth(pageserver_auth_enabled: ZenithPageserver, auth_keys: AuthKeys):
+    ps = pageserver_auth_enabled
+
+    tenant_token = auth_keys.generate_tenant_token(ps.initial_tenant)
+    invalid_tenant_token = auth_keys.generate_tenant_token(uuid4().hex)
+    management_token = auth_keys.generate_management_token()
+
+    # this does not invoke auth check and only decodes jwt and checks it for validity
+    # check both tokens
+    ps.safe_psql("status", password=tenant_token)
+    ps.safe_psql("status", password=management_token)
+
+    # tenant can create branches
+    ps.safe_psql(f"branch_create {ps.initial_tenant} new1 main", password=tenant_token)
+    # console can create branches for tenant
+    ps.safe_psql(f"branch_create {ps.initial_tenant} new2 main", password=management_token)
+
+    # fail to create branch using token with different tenantid
+    with pytest.raises(psycopg2.DatabaseError, match='Tenant id mismatch. Permission denied'):
+        ps.safe_psql(f"branch_create {ps.initial_tenant} new2 main", password=invalid_tenant_token)
+
+    # create tenant using management token
+    ps.safe_psql(f"tenant_create {uuid4().hex}", password=management_token)
+
+    # fail to create tenant using tenant token
+    with pytest.raises(psycopg2.DatabaseError, match='Attempt to access management api with tenant scope. Permission denied'):
+        ps.safe_psql(f"tenant_create {uuid4().hex}", password=tenant_token)
+
+
+@pytest.mark.parametrize('with_wal_acceptors', [False, True])
+def test_compute_auth_to_pageserver(
+    zenith_cli: ZenithCli,
+    wa_factory,
+    pageserver_auth_enabled: ZenithPageserver,
+    repo_dir: str,
+    with_wal_acceptors: bool,
+    auth_keys: AuthKeys,
+):
+    ps = pageserver_auth_enabled
+    # since we are in progress of refactoring protocols between compute safekeeper and page server
+    # use hardcoded management token in safekeeper
+    management_token = auth_keys.generate_management_token()
+
+    branch = f"test_compute_auth_to_pageserver{with_wal_acceptors}"
+    zenith_cli.run(["branch", branch, "empty"])
+    if with_wal_acceptors:
+        wa_factory.start_n_new(3, management_token)
+
+    with Postgres(
+        zenith_cli=zenith_cli,
+        repo_dir=repo_dir,
+        tenant_id=ps.initial_tenant,
+        port=55432, # FIXME port distribution is hardcoded in tests and in cli
+    ).create_start(
+        branch,
+        wal_acceptors=wa_factory.get_connstrs() if with_wal_acceptors else None,
+    ) as pg:
+        with closing(pg.connect()) as conn:
+            with conn.cursor() as cur:
+                # we rely upon autocommit after each statement
+                # as waiting for acceptors happens there
+                cur.execute('CREATE TABLE t(key int primary key, value text)')
+                cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
+                cur.execute('SELECT sum(key) FROM t')
+                assert cur.fetchone() == (5000050000, )

test_runner/batch_others/test_branch_behind.py

@@ -1,67 +1,76 @@
-import pytest
-import getpass
-import psycopg2
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
+
 #
 # Create a couple of branches off the main branch, at a historical point in time.
 #
-def test_branch_behind(zenith_cli, pageserver, postgres, pg_bin):
+def test_branch_behind(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
     # Branch at the point where only 100 rows were inserted
-    zenith_cli.run(["branch", "test_branch_behind", "empty"]);
+    zenith_cli.run(["branch", "test_branch_behind", "empty"])
 
     pgmain = postgres.create_start('test_branch_behind')
     print("postgres is running on 'test_branch_behind' branch")
 
-    main_pg_conn = psycopg2.connect(pgmain.connstr());
-    main_pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    main_pg_conn = pgmain.connect()
     main_cur = main_pg_conn.cursor()
 
     # Create table, and insert the first 100 rows
-    main_cur.execute('CREATE TABLE foo (t text)');
-    main_cur.execute("INSERT INTO foo SELECT 'long string to consume some space' || g FROM generate_series(1, 100) g");
-    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
+    main_cur.execute('CREATE TABLE foo (t text)')
+    main_cur.execute('''
+        INSERT INTO foo
+            SELECT 'long string to consume some space' || g
+            FROM generate_series(1, 100) g
+    ''')
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
     lsn_a = main_cur.fetchone()[0]
     print('LSN after 100 rows: ' + lsn_a)
 
     # Insert some more rows. (This generates enough WAL to fill a few segments.)
-    main_cur.execute("INSERT INTO foo SELECT 'long string to consume some space' || g FROM generate_series(1, 100000) g");
-    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
+    main_cur.execute('''
+        INSERT INTO foo
+            SELECT 'long string to consume some space' || g
+            FROM generate_series(1, 100000) g
+    ''')
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
     lsn_b = main_cur.fetchone()[0]
     print('LSN after 100100 rows: ' + lsn_b)
 
     # Branch at the point where only 100 rows were inserted
-    zenith_cli.run(["branch", "test_branch_behind_hundred", "test_branch_behind@"+lsn_a]);
+    zenith_cli.run(["branch", "test_branch_behind_hundred", "test_branch_behind@" + lsn_a])
 
     # Insert many more rows. This generates enough WAL to fill a few segments.
-    main_cur.execute("INSERT INTO foo SELECT 'long string to consume some space' || g FROM generate_series(1, 100000) g");
-    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
+    main_cur.execute('''
+        INSERT INTO foo
+            SELECT 'long string to consume some space' || g
+            FROM generate_series(1, 100000) g
+    ''')
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
 
-    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
     lsn_c = main_cur.fetchone()[0]
     print('LSN after 200100 rows: ' + lsn_c)
 
     # Branch at the point where only 200 rows were inserted
-    zenith_cli.run(["branch", "test_branch_behind_more", "test_branch_behind@"+lsn_b]);
+    zenith_cli.run(["branch", "test_branch_behind_more", "test_branch_behind@" + lsn_b])
 
     pg_hundred = postgres.create_start("test_branch_behind_hundred")
     pg_more = postgres.create_start("test_branch_behind_more")
 
     # On the 'hundred' branch, we should see only 100 rows
-    hundred_pg_conn = psycopg2.connect(pg_hundred.connstr())
-    hundred_pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    hundred_pg_conn = pg_hundred.connect()
     hundred_cur = hundred_pg_conn.cursor()
-    hundred_cur.execute('SELECT count(*) FROM foo');
-    assert(hundred_cur.fetchone()[0] == 100);
+    hundred_cur.execute('SELECT count(*) FROM foo')
+    assert hundred_cur.fetchone() == (100, )
 
     # On the 'more' branch, we should see 100200 rows
-    more_pg_conn = psycopg2.connect(pg_more.connstr())
-    more_pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    more_pg_conn = pg_more.connect()
     more_cur = more_pg_conn.cursor()
-    more_cur.execute('SELECT count(*) FROM foo');
-    assert(more_cur.fetchone()[0] == 100100);
+    more_cur.execute('SELECT count(*) FROM foo')
+    assert more_cur.fetchone() == (100100, )
 
     # All the rows are visible on the main branch
-    main_cur.execute('SELECT count(*) FROM foo');
-    assert(main_cur.fetchone()[0] == 200100);
+    main_cur.execute('SELECT count(*) FROM foo')
+    assert main_cur.fetchone() == (200100, )

test_runner/batch_others/test_clog_truncate.py

@@ -0,0 +1,72 @@
+import time
+import os
+
+from contextlib import closing
+
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+#
+# Test compute node start after clog truncation
+#
+def test_clog_truncate(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_clog_truncate", "empty"])
+
+    # set agressive autovacuum to make sure that truncation will happen
+    config = [
+        'autovacuum_max_workers=10', 'autovacuum_vacuum_threshold=0',
+        'autovacuum_vacuum_insert_threshold=0', 'autovacuum_vacuum_cost_delay=0',
+        'autovacuum_vacuum_cost_limit=10000', 'autovacuum_naptime =1s',
+        'autovacuum_freeze_max_age=100000'
+    ]
+
+    pg = postgres.create_start('test_clog_truncate', config_lines=config)
+    print('postgres is running on test_clog_truncate branch')
+
+    # Install extension containing function needed for test
+    pg.safe_psql('CREATE EXTENSION zenith_test_utils')
+
+    # Consume many xids to advance clog
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('select test_consume_xids(1000*1000*10);')
+            print('xids consumed')
+
+            # call a checkpoint to trigger TruncateSubtrans
+            cur.execute('CHECKPOINT;')
+
+            # ensure WAL flush
+            cur.execute('select txid_current()')
+            print(cur.fetchone())
+
+    # wait for autovacuum to truncate the pg_xact
+    # XXX Is it worth to add a timeout here?
+    pg_xact_0000_path = os.path.join(pg.pg_xact_dir_path(), '0000')
+    print("pg_xact_0000_path = " + pg_xact_0000_path)
+
+    while os.path.isfile(pg_xact_0000_path):
+        print("file exists. wait for truncation. " "pg_xact_0000_path = " + pg_xact_0000_path)
+        time.sleep(5)
+
+    # checkpoint to advance latest lsn
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('CHECKPOINT;')
+            cur.execute('select pg_current_wal_insert_lsn()')
+            lsn_after_truncation = cur.fetchone()[0]
+
+    # create new branch after clog truncation and start a compute node on it
+    print('create branch at lsn_after_truncation ' + lsn_after_truncation)
+    zenith_cli.run(
+        ["branch", "test_clog_truncate_new", "test_clog_truncate@" + lsn_after_truncation])
+
+    pg2 = postgres.create_start('test_clog_truncate_new')
+    print('postgres is running on test_clog_truncate_new branch')
+
+    # check that new node doesn't contain truncated segment
+    pg_xact_0000_path_new = os.path.join(pg2.pg_xact_dir_path(), '0000')
+    print("pg_xact_0000_path_new = " + pg_xact_0000_path_new)
+    assert os.path.isfile(pg_xact_0000_path_new) is False

test_runner/batch_others/test_config.py

@@ -1,7 +1,6 @@
-import pytest
-import os
-import getpass
-import psycopg2
+from contextlib import closing
+
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
@@ -9,22 +8,24 @@ pytest_plugins = ("fixtures.zenith_fixtures")
 #
 # Test starting Postgres with custom options
 #
-def test_config(zenith_cli, pageserver, postgres, pg_bin):
+def test_config(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
     # Create a branch for us
-    zenith_cli.run(["branch", "test_config", "empty"]);
+    zenith_cli.run(["branch", "test_config", "empty"])
 
     # change config
-    pg = postgres.create_start('test_config', ['log_min_messages=debug1'])
+    pg = postgres.create_start('test_config', config_lines=['log_min_messages=debug1'])
     print('postgres is running on test_config branch')
 
-    pg_conn = psycopg2.connect(pg.connstr())
-    pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
-    cur = pg_conn.cursor()
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('''
+                SELECT setting
+                FROM pg_settings
+                WHERE
+                    source != 'default'
+                    AND source != 'override'
+                    AND name = 'log_min_messages'
+            ''')
 
-    #check that config change was applied
-    cur.execute('SELECT name, setting from pg_settings WHERE source!=%s and source!=%s', ("default","override",))
-    for record in cur:
-        if record[0] == 'log_min_messages':
-            assert(record[1] == 'debug1')
-
-    pg_conn.close()
+            # check that config change was applied
+            assert cur.fetchone() == ('debug1', )

test_runner/batch_others/test_createdb.py

@@ -1,37 +1,38 @@
-import pytest
-import getpass
-import psycopg2
+from contextlib import closing
+from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory, ZenithCli
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
+
 #
 # Test CREATE DATABASE when there have been relmapper changes
 #
-def test_createdb(zenith_cli, pageserver, postgres, pg_bin):
-    zenith_cli.run(["branch", "test_createdb", "empty"]);
+def test_createdb(
+    zenith_cli: ZenithCli,
+    pageserver: ZenithPageserver,
+    postgres: PostgresFactory,
+    pg_bin,
+):
+    zenith_cli.run(["branch", "test_createdb", "empty"])
 
     pg = postgres.create_start('test_createdb')
     print("postgres is running on 'test_createdb' branch")
 
-    conn = psycopg2.connect(pg.connstr());
-    conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
-    cur = conn.cursor()
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            # Cause a 'relmapper' change in the original branch
+            cur.execute('VACUUM FULL pg_class')
 
-    # Cause a 'relmapper' change in the original branch
-    cur.execute('VACUUM FULL pg_class');
+            cur.execute('CREATE DATABASE foodb')
 
-    cur.execute('CREATE DATABASE foodb');
-
-    cur.execute('SELECT pg_current_wal_insert_lsn()');
-    lsn = cur.fetchone()[0]
-
-    conn.close();
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn = cur.fetchone()[0]
 
     # Create a branch
-    zenith_cli.run(["branch", "test_createdb2", "test_createdb@"+lsn]);
+    zenith_cli.run(["branch", "test_createdb2", "test_createdb@" + lsn])
 
     pg2 = postgres.create_start('test_createdb2')
 
     # Test that you can connect to the new database on both branches
-    conn = psycopg2.connect(pg.connstr('foodb'));
-    conn2 = psycopg2.connect(pg2.connstr('foodb'));
+    for db in (pg, pg2):
+        db.connect(dbname='foodb').close()

test_runner/batch_others/test_createuser.py

@@ -0,0 +1,33 @@
+from contextlib import closing
+
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+#
+# Test CREATE USER to check shared catalog restore
+#
+def test_createuser(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+    zenith_cli.run(["branch", "test_createuser", "empty"])
+
+    pg = postgres.create_start('test_createuser')
+    print("postgres is running on 'test_createuser' branch")
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            # Cause a 'relmapper' change in the original branch
+            cur.execute('CREATE USER testuser with password %s', ('testpwd', ))
+
+            cur.execute('CHECKPOINT')
+
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn = cur.fetchone()[0]
+
+    # Create a branch
+    zenith_cli.run(["branch", "test_createuser2", "test_createuser@" + lsn])
+
+    pg2 = postgres.create_start('test_createuser2')
+
+    # Test that you can connect to new branch as a new user
+    assert pg2.safe_psql('select current_user', username='testuser') == [('testuser', )]

test_runner/batch_others/test_gc.py

@@ -0,0 +1,105 @@
+import pytest
+
+from contextlib import closing
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+import psycopg2.extras
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+#
+# Test Garbage Collection of old page versions.
+#
+# This test is pretty tightly coupled with the current implementation of page version storage
+# and garbage collection in object_repository.rs.
+#
+@pytest.mark.skip(reason=""""
+    Current GC test is flaky and overly strict. Since we are migrating to the layered repo format
+    with different GC implementation let's just silence this test for now. This test only
+    works with the RocksDB implementation.
+""")
+def test_gc(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+    zenith_cli.run(["branch", "test_gc", "empty"])
+    pg = postgres.create_start('test_gc')
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            with closing(pageserver.connect()) as psconn:
+                with psconn.cursor(cursor_factory = psycopg2.extras.DictCursor) as pscur:
+
+                    # Get the timeline ID of our branch. We need it for the 'do_gc' command
+                    cur.execute("SHOW zenith.zenith_timeline")
+                    timeline = cur.fetchone()[0]
+
+                    # Create a test table
+                    cur.execute("CREATE TABLE foo(x integer)")
+
+                    # Run GC, to clear out any old page versions left behind in the catalogs by
+                    # the CREATE TABLE command. We want to have a clean slate with no garbage
+                    # before running the actual tests below, otherwise the counts won't match
+                    # what we expect.
+                    print("Running GC before test")
+                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+                    row = pscur.fetchone()
+                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
+                    # remember the number of relations
+                    n_relations = row['n_relations']
+                    assert n_relations > 0
+
+                    # Insert a row. The first insert will also create a metadata entry for the
+                    # relation, with size == 1 block. Hence, bump up the expected relation count.
+                    n_relations += 1
+                    print("Inserting one row and running GC")
+                    cur.execute("INSERT INTO foo VALUES (1)")
+                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+                    row = pscur.fetchone()
+                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
+                    assert row['n_relations'] == n_relations
+                    assert row['dropped'] == 0
+                    assert row['truncated'] == 31
+                    assert row['deleted'] == 4
+
+                    # Insert two more rows and run GC.
+                    print("Inserting two more rows and running GC")
+                    cur.execute("INSERT INTO foo VALUES (2)")
+                    cur.execute("INSERT INTO foo VALUES (3)")
+
+                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+                    row = pscur.fetchone()
+                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
+                    assert row['n_relations'] == n_relations
+                    assert row['dropped'] == 0
+                    assert row['truncated'] == 31
+                    assert row['deleted'] == 4
+
+                    # Insert one more row. It creates one more page version, but doesn't affect the
+                    # relation size.
+                    print("Inserting one more row")
+                    cur.execute("INSERT INTO foo VALUES (3)")
+
+                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+                    row = pscur.fetchone()
+                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
+                    assert row['n_relations'] == n_relations
+                    assert row['dropped'] == 0
+                    assert row['truncated'] == 31
+                    assert row['deleted'] == 2
+
+                    # Run GC again, with no changes in the database. Should not remove anything.
+                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+                    row = pscur.fetchone()
+                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
+                    assert row['n_relations'] == n_relations
+                    assert row['dropped'] == 0
+                    assert row['truncated'] == 31
+                    assert row['deleted'] == 0
+
+                    #
+                    # Test DROP TABLE checks that relation data and metadata was deleted by GC from object storage
+                    #
+                    cur.execute("DROP TABLE foo")
+
+                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+                    row = pscur.fetchone()
+                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
+                    # Each relation fork is counted separately, hence 3.
+                    assert row['dropped'] == 3

test_runner/batch_others/test_multixact.py

@@ -0,0 +1,65 @@
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+#
+# Test multixact state after branching
+# Now this test is very minimalistic -
+# it only checks next_multixact_id field in restored pg_control,
+# since we don't have functions to check multixact internals.
+#
+def test_multixact(pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin, zenith_cli, base_dir):
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_multixact", "empty"])
+    pg = postgres.create_start('test_multixact')
+
+    print("postgres is running on 'test_multixact' branch")
+    pg_conn = pg.connect()
+    cur = pg_conn.cursor()
+
+    cur.execute('''
+        CREATE TABLE t1(i int primary key);
+        INSERT INTO t1 select * from generate_series(1, 100);
+    ''')
+
+    cur.execute('SELECT next_multixact_id FROM pg_control_checkpoint()')
+    next_multixact_id_old = cur.fetchone()[0]
+
+    # Lock entries in parallel connections to set multixact
+    nclients = 3
+    connections = []
+    for i in range(nclients):
+        # Do not turn on autocommit. We want to hold the key-share locks.
+        conn = pg.connect(autocommit=False)
+        conn.cursor().execute('select * from t1 for key share')
+        connections.append(conn)
+
+    # We should have a multixact now. We can close the connections.
+    for c in connections:
+        c.close()
+
+    # force wal flush
+    cur.execute('checkpoint')
+
+    cur.execute('SELECT next_multixact_id, pg_current_wal_flush_lsn() FROM pg_control_checkpoint()')
+    res = cur.fetchone()
+    next_multixact_id = res[0]
+    lsn = res[1]
+
+    # Ensure that we did lock some tuples
+    assert int(next_multixact_id) > int(next_multixact_id_old)
+
+    # Branch at this point
+    zenith_cli.run(["branch", "test_multixact_new", "test_multixact@" + lsn])
+    pg_new = postgres.create_start('test_multixact_new')
+
+    print("postgres is running on 'test_multixact_new' branch")
+    pg_new_conn = pg_new.connect()
+    cur_new = pg_new_conn.cursor()
+
+    cur_new.execute('SELECT next_multixact_id FROM pg_control_checkpoint()')
+    next_multixact_id_new = cur_new.fetchone()[0]
+
+    # Check that we restored pg_controlfile correctly
+    assert next_multixact_id_new == next_multixact_id

test_runner/batch_others/test_pageserver_api.py

@@ -1,29 +1,27 @@
+import json
+import uuid
 import pytest
 import psycopg2
-import getpass
-import json
+from fixtures.zenith_fixtures import ZenithPageserver
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
+
 def test_status(pageserver):
-    pg_conn = psycopg2.connect(pageserver.connstr())
-    pg_conn.autocommit = True
-    cur = pg_conn.cursor()
-    cur.execute('status;')
-    assert cur.fetchone() == ('hello world',)
-    pg_conn.close()
+    assert pageserver.safe_psql('status') == [
+        ('hello world', ),
+    ]
 
-def test_branch_list(pageserver, zenith_cli):
 
+def test_branch_list(pageserver: ZenithPageserver, zenith_cli):
     # Create a branch for us
-    zenith_cli.run(["branch", "test_branch_list_main", "empty"]);
+    zenith_cli.run(["branch", "test_branch_list_main", "empty"])
 
-    page_server_conn = psycopg2.connect(pageserver.connstr())
-    page_server_conn.autocommit = True
-    page_server_cur = page_server_conn.cursor()
+    conn = pageserver.connect()
+    cur = conn.cursor()
 
-    page_server_cur.execute('branch_list;')
-    branches = json.loads(page_server_cur.fetchone()[0])
+    cur.execute(f'branch_list {pageserver.initial_tenant}')
+    branches = json.loads(cur.fetchone()[0])
     # Filter out branches created by other tests
     branches = [x for x in branches if x['name'].startswith('test_branch_list')]
 
@@ -38,8 +36,8 @@ def test_branch_list(pageserver, zenith_cli):
     zenith_cli.run(['branch', 'test_branch_list_experimental', 'test_branch_list_main'])
     zenith_cli.run(['pg', 'create', 'test_branch_list_experimental'])
 
-    page_server_cur.execute('branch_list;')
-    new_branches = json.loads(page_server_cur.fetchone()[0])
+    cur.execute(f'branch_list {pageserver.initial_tenant}')
+    new_branches = json.loads(cur.fetchone()[0])
     # Filter out branches created by other tests
     new_branches = [x for x in new_branches if x['name'].startswith('test_branch_list')]
     assert len(new_branches) == 2
@@ -51,4 +49,28 @@ def test_branch_list(pageserver, zenith_cli):
     # TODO: do the LSNs have to match here?
     assert new_branches[1] == branches[0]
 
-    page_server_conn.close()
+    conn.close()
+
+
+def test_tenant_list(pageserver: ZenithPageserver, zenith_cli):
+    res = zenith_cli.run(["tenant", "list"])
+    res.check_returncode()
+    tenants = res.stdout.splitlines()
+    assert tenants == [pageserver.initial_tenant]
+
+    conn = pageserver.connect()
+    cur = conn.cursor()
+
+    # check same tenant cannot be created twice
+    with pytest.raises(psycopg2.DatabaseError, match=f'tenant {pageserver.initial_tenant} already exists'):
+        cur.execute(f'tenant_create {pageserver.initial_tenant}')
+
+    # create one more tenant
+    tenant1 = uuid.uuid4().hex
+    cur.execute(f'tenant_create {tenant1}')
+
+    cur.execute('tenant_list')
+
+    # compare tenants list
+    new_tenants = sorted(json.loads(cur.fetchone()[0]))
+    assert sorted([pageserver.initial_tenant, tenant1]) == new_tenants

test_runner/batch_others/test_pageserver_restart.py

@@ -0,0 +1,66 @@
+import pytest
+import random
+import time
+
+from contextlib import closing
+from multiprocessing import Process, Value
+from fixtures.zenith_fixtures import WalAcceptorFactory, ZenithPageserver, PostgresFactory
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+# Check that dead minority doesn't prevent the commits: execute insert n_inserts
+# times, with fault_probability chance of getting a wal acceptor down or up
+# along the way. 2 of 3 are always alive, so the work keeps going.
+def test_pageserver_restart(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory: WalAcceptorFactory):
+
+    # One safekeeper is enough for this test.
+    wa_factory.start_n_new(1)
+
+    zenith_cli.run(["branch", "test_pageserver_restart", "empty"])
+    pg = postgres.create_start('test_pageserver_restart',
+                               wal_acceptors=wa_factory.get_connstrs())
+
+    pg_conn = pg.connect()
+    cur = pg_conn.cursor()
+
+    # Create table, and insert some rows. Make it big enough that it doesn't fit in
+    # shared_buffers, otherwise the SELECT after restart will just return answer
+    # from shared_buffers without hitting the page server, which defeats the point
+    # of this test.
+    cur.execute('CREATE TABLE foo (t text)')
+    cur.execute('''
+        INSERT INTO foo
+            SELECT 'long string to consume some space' || g
+            FROM generate_series(1, 100000) g
+    ''')
+
+    # Verify that the table is larger than shared_buffers
+    cur.execute('''
+        select setting::int * pg_size_bytes(unit) as shared_buffers, pg_relation_size('foo') as tbl_ize
+        from pg_settings where name = 'shared_buffers'
+    ''')
+    row = cur.fetchone()
+    print("shared_buffers is {}, table size {}", row[0], row[1]);
+    assert int(row[0]) < int(row[1])
+
+    # Stop and restart pageserver. This is a more or less graceful shutdown, although
+    # the page server doesn't currently have a shutdown routine so there's no difference
+    # between stopping and crashing.
+    pageserver.stop();
+    pageserver.start();
+
+    # Stopping the pageserver breaks the connection from the postgres backend to
+    # the page server, and causes the next query on the connection to fail. Start a new
+    # postgres connection too, to avoid that error. (Ideally, the compute node would
+    # handle that and retry internally, without propagating the error to the user, but
+    # currently it doesn't...)
+    pg_conn = pg.connect()
+    cur = pg_conn.cursor()
+
+    cur.execute("SELECT count(*) FROM foo")
+    assert cur.fetchone() == (100000, )
+
+    # Stop the page server by force, and restart it
+    pageserver.stop();
+    pageserver.start();
+

test_runner/batch_others/test_pgbench.py

@@ -1,17 +1,16 @@
-import pytest
+from fixtures.zenith_fixtures import PostgresFactory
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
 
-def test_pgbench(pageserver, postgres, pg_bin, zenith_cli):
-
+def test_pgbench(postgres: PostgresFactory, pg_bin, zenith_cli):
     # Create a branch for us
-    zenith_cli.run(["branch", "test_pgbench", "empty"]);
+    zenith_cli.run(["branch", "test_pgbench", "empty"])
 
     pg = postgres.create_start('test_pgbench')
     print("postgres is running on 'test_pgbench' branch")
 
-    connstr = pg.connstr();
+    connstr = pg.connstr()
 
     pg_bin.run_capture(['pgbench', '-i', connstr])
     pg_bin.run_capture(['pgbench'] + '-c 10 -T 5 -P 1 -M prepared'.split() + [connstr])

test_runner/batch_others/test_restart_compute.py

@@ -0,0 +1,86 @@
+import pytest
+
+from contextlib import closing
+from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+#
+# Test restarting and recreating a postgres instance
+#
+@pytest.mark.parametrize('with_wal_acceptors', [True, False])
+def test_restart_compute(
+        zenith_cli,
+        pageserver: ZenithPageserver,
+        postgres: PostgresFactory,
+        pg_bin,
+        wa_factory,
+        with_wal_acceptors: bool,
+    ):
+    wal_acceptor_connstrs = None
+    zenith_cli.run(["branch", "test_restart_compute", "empty"])
+
+    if with_wal_acceptors:
+        wa_factory.start_n_new(3)
+        wal_acceptor_connstrs = wa_factory.get_connstrs()
+
+    pg = postgres.create_start('test_restart_compute',
+                               wal_acceptors=wal_acceptor_connstrs)
+    print("postgres is running on 'test_restart_compute' branch")
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('CREATE TABLE t(key int primary key, value text)')
+            cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
+            cur.execute('SELECT sum(key) FROM t')
+            r = cur.fetchone()
+            assert r == (5000050000, )
+            print("res = ", r)
+
+    # Remove data directory and restart
+    pg.stop_and_destroy().create_start('test_restart_compute',
+                                       wal_acceptors=wal_acceptor_connstrs)
+
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            # We can still see the row
+            cur.execute('SELECT sum(key) FROM t')
+            r = cur.fetchone()
+            assert r == (5000050000, )
+            print("res = ", r)
+
+            # Insert another row
+            cur.execute("INSERT INTO t VALUES (100001, 'payload2')")
+            cur.execute('SELECT count(*) FROM t')
+
+            r = cur.fetchone()
+            assert r == (100001, )
+            print("res = ", r)
+
+    # Again remove data directory and restart
+    pg.stop_and_destroy().create_start('test_restart_compute',
+                                       wal_acceptors=wal_acceptor_connstrs)
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            # We can still see the rows
+            cur.execute('SELECT count(*) FROM t')
+
+            r = cur.fetchone()
+            assert r == (100001, )
+            print("res = ", r)
+
+    # And again remove data directory and restart
+    pg.stop_and_destroy().create_start('test_restart_compute',
+                                       wal_acceptors=wal_acceptor_connstrs)
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            # We can still see the rows
+            cur.execute('SELECT count(*) FROM t')
+
+            r = cur.fetchone()
+            assert r == (100001, )
+            print("res = ", r)

test_runner/batch_others/test_snapfiles_gc.py

@@ -0,0 +1,124 @@
+from contextlib import closing
+import psycopg2.extras
+import time;
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+def print_gc_result(row):
+    print("GC duration {elapsed} ms".format_map(row));
+    print("  REL    total: {snapshot_relfiles_total}, needed_by_cutoff {snapshot_relfiles_needed_by_cutoff}, needed_by_branches: {snapshot_relfiles_needed_by_branches}, not_updated: {snapshot_relfiles_not_updated}, removed: {snapshot_relfiles_removed}, dropped: {snapshot_relfiles_dropped}".format_map(row))
+    print("  NONREL total: {snapshot_nonrelfiles_total}, needed_by_cutoff {snapshot_nonrelfiles_needed_by_cutoff}, needed_by_branches: {snapshot_nonrelfiles_needed_by_branches}, not_updated: {snapshot_nonrelfiles_not_updated}, removed: {snapshot_nonrelfiles_removed}, dropped: {snapshot_nonrelfiles_dropped}".format_map(row))
+
+
+#
+# Test Garbage Collection of old snapshot files
+#
+# This test is pretty tightly coupled with the current implementation of layered
+# storage, in layered_repository.rs.
+#
+def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin):
+    zenith_cli.run(["branch", "test_snapfiles_gc", "empty"])
+    pg = postgres.create_start('test_snapfiles_gc')
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            with closing(pageserver.connect()) as psconn:
+                with psconn.cursor(cursor_factory = psycopg2.extras.DictCursor) as pscur:
+
+                    # Get the timeline ID of our branch. We need it for the 'do_gc' command
+                    cur.execute("SHOW zenith.zenith_timeline")
+                    timeline = cur.fetchone()[0]
+
+                    # Create a test table
+                    cur.execute("CREATE TABLE foo(x integer)")
+                    cur.execute("INSERT INTO foo VALUES (1)")
+
+                    cur.execute("select relfilenode from pg_class where oid = 'foo'::regclass");
+                    row = cur.fetchone();
+                    print("relfilenode is {}", row[0]);
+
+                    # Run GC, to clear out any garbage left behind in the catalogs by
+                    # the CREATE TABLE command. We want to have a clean slate with no garbage
+                    # before running the actual tests below, otherwise the counts won't match
+                    # what we expect.
+                    #
+                    # Also run vacuum first to make it less likely that autovacuum or pruning
+                    # kicks in and confuses our numbers.
+                    cur.execute("VACUUM")
+
+                    # delete the row, to update the Visibility Map. We don't want the VM
+                    # update to confuse our numbers either.
+                    cur.execute("DELETE FROM foo")
+
+                    print("Running GC before test")
+                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+                    row = pscur.fetchone()
+                    print_gc_result(row);
+                    # remember the number of files
+                    snapshot_relfiles_remain = row['snapshot_relfiles_total'] - row['snapshot_relfiles_removed']
+                    assert snapshot_relfiles_remain > 0
+
+                    # Insert a row.
+                    print("Inserting one row and running GC")
+                    cur.execute("INSERT INTO foo VALUES (1)")
+                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+                    row = pscur.fetchone()
+                    print_gc_result(row);
+                    assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain + 1
+                    assert row['snapshot_relfiles_removed'] == 1
+                    assert row['snapshot_relfiles_dropped'] == 0
+
+                    # Insert two more rows and run GC.
+                    # This should create a new snapshot file with the new contents, and
+                    # remove the old one.
+                    print("Inserting two more rows and running GC")
+                    cur.execute("INSERT INTO foo VALUES (2)")
+                    cur.execute("INSERT INTO foo VALUES (3)")
+
+                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+                    row = pscur.fetchone()
+                    print_gc_result(row);
+                    assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain + 1
+                    assert row['snapshot_relfiles_removed'] == 1
+                    assert row['snapshot_relfiles_dropped'] == 0
+
+                    # Do it again. Should again create a new snapshot file and remove old one.
+                    print("Inserting two more rows and running GC")
+                    cur.execute("INSERT INTO foo VALUES (2)")
+                    cur.execute("INSERT INTO foo VALUES (3)")
+
+                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+                    row = pscur.fetchone()
+                    print_gc_result(row);
+                    assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain + 1
+                    assert row['snapshot_relfiles_removed'] == 1
+                    assert row['snapshot_relfiles_dropped'] == 0
+
+                    # Run GC again, with no changes in the database. Should not remove anything.
+                    print("Run GC again, with nothing to do")
+                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+                    row = pscur.fetchone()
+                    print_gc_result(row);
+                    assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain
+                    assert row['snapshot_relfiles_removed'] == 0
+                    assert row['snapshot_relfiles_dropped'] == 0
+
+                    #
+                    # Test DROP TABLE checks that relation data and metadata was deleted by GC from object storage
+                    #
+                    print("Drop table and run GC again");
+                    cur.execute("DROP TABLE foo")
+
+                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
+                    row = pscur.fetchone()
+                    print_gc_result(row);
+
+                    # Each relation fork is counted separately, hence 3.
+                    assert row['snapshot_relfiles_dropped'] == 3
+
+                    # The catalog updates also create new snapshot files of the catalogs, which
+                    # are counted as 'removed'
+                    assert row['snapshot_relfiles_removed'] > 0
+
+                    # TODO: perhaps we should count catalog and user relations separately,
+                    # to make this kind of testing more robust

test_runner/batch_others/test_tenants.py

@@ -0,0 +1,48 @@
+from contextlib import closing
+
+import pytest
+
+from fixtures.zenith_fixtures import (
+    TenantFactory,
+    ZenithCli,
+    PostgresFactory,
+)
+
+
+@pytest.mark.parametrize('with_wal_acceptors', [False, True])
+def test_tenants_normal_work(
+    zenith_cli: ZenithCli,
+    tenant_factory: TenantFactory,
+    postgres: PostgresFactory,
+    wa_factory,
+    with_wal_acceptors: bool,
+):
+    """Tests tenants with and without wal acceptors"""
+    tenant_1 = tenant_factory.create()
+    tenant_2 = tenant_factory.create()
+
+    zenith_cli.run(["branch", f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", "main", f"--tenantid={tenant_1}"])
+    zenith_cli.run(["branch", f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", "main", f"--tenantid={tenant_2}"])
+    if with_wal_acceptors:
+        wa_factory.start_n_new(3)
+
+    pg_tenant1 = postgres.create_start(
+        f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}",
+        tenant_1,
+        wal_acceptors=wa_factory.get_connstrs() if with_wal_acceptors else None,
+    )
+    pg_tenant2 = postgres.create_start(
+        f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}",
+        tenant_2,
+        wal_acceptors=wa_factory.get_connstrs() if with_wal_acceptors else None,
+    )
+
+    for pg in [pg_tenant1, pg_tenant2]:
+        with closing(pg.connect()) as conn:
+            with conn.cursor() as cur:
+                # we rely upon autocommit after each statement
+                # as waiting for acceptors happens there
+                cur.execute("CREATE TABLE t(key int primary key, value text)")
+                cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
+                cur.execute("SELECT sum(key) FROM t")
+                assert cur.fetchone() == (5000050000,)

test_runner/batch_others/test_twophase.py

@@ -1,58 +1,88 @@
-#
-# Test branching, when a transaction is in prepared state
-#
-import pytest
-import getpass
-import psycopg2
+import os
+
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
-def test_twophase(zenith_cli, pageserver, postgres, pg_bin):
-    zenith_cli.run(["branch", "test_twophase", "empty"]);
 
-    pg = postgres.create_start('test_twophase', ['max_prepared_transactions=5'])
+#
+# Test branching, when a transaction is in prepared state
+#
+def test_twophase(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+    zenith_cli.run(["branch", "test_twophase", "empty"])
+
+    pg = postgres.create_start('test_twophase', config_lines=['max_prepared_transactions=5'])
     print("postgres is running on 'test_twophase' branch")
 
-    conn = psycopg2.connect(pg.connstr());
-    conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    conn = pg.connect()
     cur = conn.cursor()
 
-    cur.execute('CREATE TABLE foo (t text)');
+    cur.execute('CREATE TABLE foo (t text)')
 
     # Prepare a transaction that will insert a row
-    cur.execute('BEGIN');
-    cur.execute("INSERT INTO foo VALUES ('one')");
-    cur.execute("PREPARE TRANSACTION 'insert_one'");
+    cur.execute('BEGIN')
+    cur.execute("INSERT INTO foo VALUES ('one')")
+    cur.execute("PREPARE TRANSACTION 'insert_one'")
 
     # Prepare another transaction that will insert a row
-    cur.execute('BEGIN');
-    cur.execute("INSERT INTO foo VALUES ('two')");
-    cur.execute("PREPARE TRANSACTION 'insert_two'");
+    cur.execute('BEGIN')
+    cur.execute("INSERT INTO foo VALUES ('two')")
+    cur.execute("PREPARE TRANSACTION 'insert_two'")
 
-    cur.execute('BEGIN');
-    cur.execute("INSERT INTO foo VALUES ('three')");
-    cur.execute("PREPARE TRANSACTION 'insert_three'");
-    cur.execute("COMMIT PREPARED 'insert_three'");
+    # Prepare a transaction that will insert a row
+    cur.execute('BEGIN')
+    cur.execute("INSERT INTO foo VALUES ('three')")
+    cur.execute("PREPARE TRANSACTION 'insert_three'")
 
-    cur.execute('SELECT pg_current_wal_insert_lsn()');
-    lsn = cur.fetchone()[0]
+    # Prepare another transaction that will insert a row
+    cur.execute('BEGIN')
+    cur.execute("INSERT INTO foo VALUES ('four')")
+    cur.execute("PREPARE TRANSACTION 'insert_four'")
+
+    # On checkpoint state data copied to files in
+    # pg_twophase directory and fsynced
+    cur.execute('CHECKPOINT')
+
+    twophase_files = os.listdir(pg.pg_twophase_dir_path())
+    print(twophase_files)
+    assert len(twophase_files) == 4
+
+    cur.execute("COMMIT PREPARED 'insert_three'")
+    cur.execute("ROLLBACK PREPARED 'insert_four'")
+    cur.execute('CHECKPOINT')
+
+    twophase_files = os.listdir(pg.pg_twophase_dir_path())
+    print(twophase_files)
+    assert len(twophase_files) == 2
 
     # Create a branch with the transaction in prepared state
-    zenith_cli.run(["branch", "test_twophase_prepared", "test_twophase@"+lsn]);
+    zenith_cli.run(["branch", "test_twophase_prepared", "test_twophase"])
 
-    pg2 = postgres.create_start('test_twophase_prepared', ['max_prepared_transactions=5'])
-    conn2 = psycopg2.connect(pg2.connstr());
-    conn2.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    # Create compute node, but don't start.
+    # We want to observe pgdata before postgres starts
+    pg2 = postgres.create(
+        'test_twophase_prepared',
+        config_lines=['max_prepared_transactions=5'],
+    )
+
+    # Check that we restored only needed twophase files
+    twophase_files2 = os.listdir(pg2.pg_twophase_dir_path())
+    print(twophase_files2)
+    assert twophase_files2.sort() == twophase_files.sort()
+
+    pg2 = pg2.start()
+    conn2 = pg2.connect()
     cur2 = conn2.cursor()
 
-    # On the new branch, commit one of the prepared transactions, abort the other one.
-    cur2.execute("COMMIT PREPARED 'insert_one'");
-    cur2.execute("ROLLBACK PREPARED 'insert_two'");
+    # On the new branch, commit one of the prepared transactions,
+    # abort the other one.
+    cur2.execute("COMMIT PREPARED 'insert_one'")
+    cur2.execute("ROLLBACK PREPARED 'insert_two'")
 
-    cur2.execute('SELECT * FROM foo');
-    assert(cur2.fetchall() == [('one',),('three',)]);
+    cur2.execute('SELECT * FROM foo')
+    assert cur2.fetchall() == [('one',), ('three',)]
 
-    # Neither insert is visible on the original branch, the transactions are still
-    # in prepared state there.
-    cur.execute('SELECT * FROM foo');
-    assert(cur.fetchall() == [('three',)]);
+    # Only one committed insert is visible on the original branch
+    cur.execute('SELECT * FROM foo')
+    assert cur.fetchall() == [('three',)]

test_runner/batch_others/test_wal_acceptor.py

@@ -0,0 +1,200 @@
+import pytest
+import random
+import time
+
+from contextlib import closing
+from multiprocessing import Process, Value
+from fixtures.zenith_fixtures import WalAcceptorFactory, ZenithPageserver, PostgresFactory
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+# basic test, write something in setup with wal acceptors, ensure that commits
+# succeed and data is written
+def test_normal_work(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory):
+    zenith_cli.run(["branch", "test_wal_acceptors_normal_work", "empty"])
+    wa_factory.start_n_new(3)
+    pg = postgres.create_start('test_wal_acceptors_normal_work',
+                               wal_acceptors=wa_factory.get_connstrs())
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            # we rely upon autocommit after each statement
+            # as waiting for acceptors happens there
+            cur.execute('CREATE TABLE t(key int primary key, value text)')
+            cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
+            cur.execute('SELECT sum(key) FROM t')
+            assert cur.fetchone() == (5000050000, )
+
+
+# Run page server and multiple acceptors, and multiple compute nodes running
+# against different timelines.
+def test_many_timelines(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory):
+    n_timelines = 2
+
+    wa_factory.start_n_new(3)
+
+    branches = ["test_wal_acceptors_many_timelines_{}".format(tlin) for tlin in range(n_timelines)]
+
+    # start postgres on each timeline
+    pgs = []
+    for branch in branches:
+        zenith_cli.run(["branch", branch, "empty"])
+        pgs.append(postgres.create_start(branch, wal_acceptors=wa_factory.get_connstrs()))
+
+    # Do everything in different loops to have actions on different timelines
+    # interleaved.
+    # create schema
+    for pg in pgs:
+        pg.safe_psql("CREATE TABLE t(key int primary key, value text)")
+
+    # Populate data
+    for pg in pgs:
+        pg.safe_psql("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
+
+    # Check data
+    for pg in pgs:
+        res = pg.safe_psql("SELECT sum(key) FROM t")
+        assert res[0] == (5000050000, )
+
+
+# Check that dead minority doesn't prevent the commits: execute insert n_inserts
+# times, with fault_probability chance of getting a wal acceptor down or up
+# along the way. 2 of 3 are always alive, so the work keeps going.
+def test_restarts(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory: WalAcceptorFactory):
+    fault_probability = 0.01
+    n_inserts = 1000
+    n_acceptors = 3
+
+    wa_factory.start_n_new(n_acceptors)
+
+    zenith_cli.run(["branch", "test_wal_acceptors_restarts", "empty"])
+    pg = postgres.create_start('test_wal_acceptors_restarts',
+                               wal_acceptors=wa_factory.get_connstrs())
+
+    # we rely upon autocommit after each statement
+    # as waiting for acceptors happens there
+    pg_conn = pg.connect()
+    cur = pg_conn.cursor()
+
+    failed_node = None
+    cur.execute('CREATE TABLE t(key int primary key, value text)')
+    for i in range(n_inserts):
+        cur.execute("INSERT INTO t values (%s, 'payload');", (i + 1, ))
+
+        if random.random() <= fault_probability:
+            if failed_node is None:
+                failed_node = wa_factory.instances[random.randrange(0, n_acceptors)]
+                failed_node.stop()
+            else:
+                failed_node.start()
+                failed_node = None
+    cur.execute('SELECT sum(key) FROM t')
+    assert cur.fetchone() == (500500, )
+
+
+start_delay_sec = 2
+
+
+def delayed_wal_acceptor_start(wa):
+    time.sleep(start_delay_sec)
+    wa.start()
+
+
+# When majority of acceptors is offline, commits are expected to be frozen
+def test_unavailability(zenith_cli, postgres: PostgresFactory, wa_factory):
+    wa_factory.start_n_new(2)
+
+    zenith_cli.run(["branch", "test_wal_acceptors_unavailability", "empty"])
+    pg = postgres.create_start('test_wal_acceptors_unavailability',
+                               wal_acceptors=wa_factory.get_connstrs())
+
+    # we rely upon autocommit after each statement
+    # as waiting for acceptors happens there
+    pg_conn = pg.connect()
+    cur = pg_conn.cursor()
+
+    # check basic work with table
+    cur.execute('CREATE TABLE t(key int primary key, value text)')
+    cur.execute("INSERT INTO t values (1, 'payload')")
+
+    # shutdown one of two acceptors, that is, majority
+    wa_factory.instances[0].stop()
+
+    proc = Process(target=delayed_wal_acceptor_start, args=(wa_factory.instances[0], ))
+    proc.start()
+
+    start = time.time()
+    cur.execute("INSERT INTO t values (2, 'payload')")
+    # ensure that the query above was hanging while acceptor was down
+    assert (time.time() - start) >= start_delay_sec
+    proc.join()
+
+    # for the world's balance, do the same with second acceptor
+    wa_factory.instances[1].stop()
+
+    proc = Process(target=delayed_wal_acceptor_start, args=(wa_factory.instances[1], ))
+    proc.start()
+
+    start = time.time()
+    cur.execute("INSERT INTO t values (3, 'payload')")
+    # ensure that the query above was hanging while acceptor was down
+    assert (time.time() - start) >= start_delay_sec
+    proc.join()
+
+    cur.execute("INSERT INTO t values (4, 'payload')")
+
+    cur.execute('SELECT sum(key) FROM t')
+    assert cur.fetchone() == (10, )
+
+
+# shut down random subset of acceptors, sleep, wake them up, rinse, repeat
+def xmas_garland(acceptors, stop):
+    while not bool(stop.value):
+        victims = []
+        for wa in acceptors:
+            if random.random() >= 0.5:
+                victims.append(wa)
+        for v in victims:
+            v.stop()
+        time.sleep(1)
+        for v in victims:
+            v.start()
+        time.sleep(1)
+
+
+# value which gets unset on exit
+@pytest.fixture
+def stop_value():
+    stop = Value('i', 0)
+    yield stop
+    stop.value = 1
+
+
+# do inserts while concurrently getting up/down subsets of acceptors
+def test_race_conditions(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory, stop_value):
+
+    wa_factory.start_n_new(3)
+
+    zenith_cli.run(["branch", "test_wal_acceptors_race_conditions", "empty"])
+    pg = postgres.create_start('test_wal_acceptors_race_conditions',
+                               wal_acceptors=wa_factory.get_connstrs())
+
+    # we rely upon autocommit after each statement
+    # as waiting for acceptors happens there
+    pg_conn = pg.connect()
+    cur = pg_conn.cursor()
+
+    cur.execute('CREATE TABLE t(key int primary key, value text)')
+
+    proc = Process(target=xmas_garland, args=(wa_factory.instances, stop_value))
+    proc.start()
+
+    for i in range(1000):
+        cur.execute("INSERT INTO t values (%s, 'payload');", (i + 1, ))
+
+    cur.execute('SELECT sum(key) FROM t')
+    assert cur.fetchone() == (500500, )
+
+    stop_value.value = 1
+    proc.join()