Fix 2PC support

2026-08-13 09:41:27 +00:00 · 2021-05-21 20:01:44 +03:00
143 changed files with 7637 additions and 16732 deletions
@@ -7,7 +7,7 @@ executors:
  zenith-build-executor:
    resource_class: xlarge
    docker:
-      - image: cimg/rust:1.52.1
+      - image: cimg/rust:1.51.0
 jobs:
@@ -37,7 +37,7 @@ jobs:
          command: |
            if [ ! -e tmp_install/bin/postgres ]; then
              sudo apt update
-              sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev
+              sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libcurl4-openssl-dev
            fi
        # Build postgres if the restore_cache didn't find a build.
@@ -119,7 +119,8 @@ jobs:
            - target
        # Run rust unit tests
-      - run: cargo test
+        # FIXME: remove -p zenith_utils once integration tests are moved to python
      - run: cargo test -p zenith_utils
        # Install the rust binaries, for use by test jobs
        # `--locked` is required; otherwise, `cargo install` will ignore Cargo.lock.
@@ -191,12 +192,7 @@ jobs:
          condition: << parameters.needs_postgres_source >>
          steps:
            - run: git submodule update --init --depth 1
-      - run:
+      - run: pip install pytest psycopg2
          name: Install pipenv & deps
          working_directory: test_runner
          command: |
            pip install pipenv
            pipenv install
      - run:
          name: Run pytest
          working_directory: test_runner
@@ -215,21 +211,25 @@ jobs:
            #
            # The junit.xml file allows CircleCI to display more fine-grained test information
            # in its "Tests" tab in the results page.
-            # -s prevents pytest from capturing output, which helps to see
+            pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short $TEST_SELECTION $EXTRA_PARAMS
            # what's going on if the test hangs
            # --verbose prints name of each test (helpful when there are
            # multiple tests in one file)
            # -rA prints summary in the end
            pipenv run pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short -s --verbose -rA $TEST_SELECTION $EXTRA_PARAMS
      - run:
          # CircleCI artifacts are preserved one file at a time, so skipping
          # this step isn't a good idea. If you want to extract the
          # pageserver state, perhaps a tarball would be a better idea.
-          name: Delete all data but logs
+          name: Delete pageserver data
          when: always
          command: |
            du -sh /tmp/test_output/*
-            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "wal_acceptor.log" ! -name "regression.diffs" -delete
+            for DIR in /tmp/test_output/*; do
              mv $DIR/repo/pageserver.log $DIR/ || true # ignore errors
              for PGDIR in $DIR/repo/pgdatadirs/pg?; do
                echo "PGDIR: $PGDIR"
                NEW_LOG="${PGDIR##*/}_log"
                mv $PGDIR/log "$DIR/$NEW_LOG" || true # ignore errors
              done
              echo "rm $DIR/repo"
              rm -rf $DIR/repo
            done
            du -sh /tmp/test_output/*
      - store_artifacts:
          path: /tmp/test_output
@@ -237,23 +237,6 @@ jobs:
      - store_test_results:
          path: /tmp/test_output
  # Build zenithdb/zenith:latest image and push it to Docker hub
  docker-image:
    docker:
      - image: cimg/base:2021.04
    steps:
      - checkout
      - setup_remote_docker:
          docker_layer_caching: true
      - run:
          name: Init postgres submodule
          command: git submodule update --init --depth 1
      - run:
          name: Build and push Docker image
          command: |
            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
            docker build -t zenithdb/zenith:latest . && docker push zenithdb/zenith:latest
 workflows:
  build_and_test:
    jobs:
@@ -282,14 +265,3 @@ workflows:
          test_selection: batch_others
          requires:
            - build-zenith-<< matrix.build_type >>
      - docker-image:
          # Context gives an ability to login
          context: Docker Hub
          # Build image only for commits to main
          filters:
            branches:
              only:
                - main
          requires:
            - pg_regress tests release
            - other tests release
@@ -1,13 +0,0 @@
 **/.git/
 **/__pycache__
 **/.pytest_cache
 /target
 /tmp_check
 /tmp_install
 /tmp_check_cli
 /test_output
 /.vscode
 /.zenith
 /integration_tests/.zenith
 /Dockerfile
@@ -35,7 +35,7 @@ jobs:
      - name: Install postgres dependencies
        run: |
          sudo apt update
-          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev
+          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libcurl4-openssl-dev
      - name: Set pg revision for caching
        id: pg_ver
@@ -1,31 +0,0 @@
 # How to contribute
 Howdy! Usual good software engineering practices apply. Write
 tests. Write comments. Follow standard Rust coding practices where
 possible. Use 'cargo fmt' and 'clippy' to tidy up formatting.
 There are soft spots in the code, which could use cleanup,
 refactoring, additional comments, and so forth. Let's try to raise the
 bar, and clean things up as we go. Try to leave code in a better shape
 than it was before.
 ## Submitting changes
 1. Make a PR for every change.
   Even seemingly trivial patches can break things in surprising ways.
 Use of common sense is OK. If you're only fixing a typo in a comment,
 it's probably fine to just push it. But if in doubt, open a PR.
 2. Get at least one +1 on your PR before you push.
   For simple patches, it will only take a minute for someone to review
 it.
 3. Always keep the CI green.
   Do not push, if the CI failed on your PR. Even if you think it's not
 your patch's fault. Help to fix the root cause if something else has
 broken the CI, before pushing.
 *Happy Hacking!*
@@ -1,20 +0,0 @@
 This software is licensed under the Apache 2.0 License:
 ----------------------------------------------------------------------------
 Copyright 2021 Zenith Labs, Inc
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ----------------------------------------------------------------------------
 The PostgreSQL submodule in vendor/postgres is licensed under the
 PostgreSQL license. See vendor/postgres/COPYRIGHT.
@@ -1,17 +1,11 @@
 [workspace]
 members = [
-    "control_plane",
+    "integration_tests",
    "pageserver",
    "postgres_ffi",
    "proxy",
    "walkeeper",
    "workspace_hack",
    "zenith",
-    "zenith_metrics",
+    "control_plane",
    "postgres_ffi",
    "zenith_utils",
    "workspace_hack",
 ]
 [profile.release]
 # This is useful for profiling and, to some extent, debug.
 # Besides, debug info should not affect the performance.
 debug = true
@@ -1,78 +0,0 @@
 #
 # Docker image for console integration testing.
 #
 #
 # Build Postgres separately --- this layer will be rebuilt only if one of
 # mentioned paths will get any changes.
 #
 FROM zenithdb/build:buster AS pg-build
 WORKDIR /zenith
 COPY ./vendor/postgres vendor/postgres
 COPY ./Makefile Makefile
 RUN make -j $(getconf _NPROCESSORS_ONLN) -s postgres
 #
 # Calculate cargo dependencies.
 # This will always run, but only generate recipe.json with list of dependencies without
 # installing them.
 #
 FROM zenithdb/build:buster AS cargo-deps-inspect
 WORKDIR /zenith
 COPY . .
 RUN cargo chef prepare --recipe-path /zenith/recipe.json
 #
 # Build cargo dependencies.
 # This temp cantainner should be rebuilt only if recipe.json was changed.
 #
 FROM zenithdb/build:buster AS deps-build
 WORKDIR /zenith
 COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
 COPY --from=cargo-deps-inspect /usr/local/cargo/bin/cargo-chef /usr/local/cargo/bin/
 COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
 RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json
 #
 # Build zenith binaries
 #
 FROM zenithdb/build:buster AS build
 WORKDIR /zenith
 COPY . .
 # Copy cached dependencies
 COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
 COPY --from=deps-build /zenith/target target
 COPY --from=deps-build /usr/local/cargo/ /usr/local/cargo/
 RUN cargo build --release
 #
 # Copy binaries to resulting image.
 #
 FROM debian:buster-slim
 WORKDIR /data
 RUN apt-get update && apt-get -yq install librocksdb-dev libseccomp-dev openssl && \
    mkdir zenith_install
 COPY --from=build /zenith/target/release/pageserver /usr/local/bin
 COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin
 COPY --from=build /zenith/target/release/proxy /usr/local/bin
 COPY --from=pg-build /zenith/tmp_install postgres_install
 COPY docker-entrypoint.sh /docker-entrypoint.sh
 # Remove build artifacts (~ 500 MB)
 RUN rm -rf postgres_install/build && \
    # 'Install' Postgres binaries locally
    cp -r postgres_install/* /usr/local/ && \
    # Prepare an archive of Postgres binaries (should be around 11 MB)
    # and keep it inside container for an ease of deploy pipeline.
    cd postgres_install && tar -czf /data/postgres_install.tar.gz . && cd .. && \
    rm -rf postgres_install
 RUN useradd -m -d /data zenith
 VOLUME ["/data"]
 USER zenith
 EXPOSE 6400
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD ["pageserver"]
@@ -1,95 +0,0 @@
 #
 # Docker image for console integration testing.
 #
 # We may also reuse it in CI to unify installation process and as a general binaries building
 # tool for production servers.
 #
 # Dynamic linking is used for librocksdb and libstdc++ bacause librocksdb-sys calls
 # bindgen with "dynamic" feature flag. This also prevents usage of dockerhub alpine-rust
 # images which are statically linked and have guards against any dlopen. I would rather
 # prefer all static binaries so we may change the way librocksdb-sys builds or wait until
 # we will have our own storage and drop rockdb dependency.
 #
 # Cargo-chef is used to separate dependencies building from main binaries building. This
 # way `docker build` will download and install dependencies only of there are changes to
 # out Cargo.toml files.
 #
 #
 # build postgres separately -- this layer will be rebuilt only if one of
 # mentioned paths will get any changes
 #
 FROM alpine:3.13 as pg-build
 RUN apk add --update clang llvm compiler-rt compiler-rt-static lld musl-dev binutils \
                     make bison flex readline-dev zlib-dev perl linux-headers libseccomp-dev
 WORKDIR zenith
 COPY ./vendor/postgres vendor/postgres
 COPY ./Makefile Makefile
 # Build using clang and lld
 RUN CC='clang' LD='lld' CFLAGS='-fuse-ld=lld --rtlib=compiler-rt' make postgres -j4
 #
 # Calculate cargo dependencies.
 # This will always run, but only generate recipe.json with list of dependencies without
 # installing them.
 #
 FROM alpine:20210212 as cargo-deps-inspect
 RUN apk add --update rust cargo
 RUN cargo install cargo-chef
 WORKDIR zenith
 COPY . .
 RUN cargo chef prepare --recipe-path recipe.json
 #
 # Build cargo dependencies.
 # This temp cantainner would be build only if recipe.json was changed.
 #
 FROM alpine:20210212 as deps-build
 RUN apk add --update rust cargo openssl-dev clang build-base
 # rust-rocksdb can be built against system-wide rocksdb -- that saves about
 # 10 minutes during build. Rocksdb apk package is in testing now, but use it
 # anyway. In case of any troubles we can download and build rocksdb here manually
 # (to cache it as a docker layer).
 RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
 WORKDIR zenith
 COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
 COPY --from=cargo-deps-inspect /root/.cargo/bin/cargo-chef /root/.cargo/bin/
 COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
 RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json
 #
 # Build zenith binaries
 #
 FROM alpine:20210212 as build
 RUN apk add --update rust cargo openssl-dev clang build-base
 RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
 WORKDIR zenith
 COPY . .
 # Copy cached dependencies
 COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
 COPY --from=deps-build /zenith/target target
 COPY --from=deps-build /root/.cargo /root/.cargo
 RUN cargo build --release
 #
 # Copy binaries to resulting image.
 # build-base hare to provide libstdc++ (it will also bring gcc, but leave it this way until we figure
 # out how to statically link rocksdb or avoid it at all).
 #
 FROM alpine:3.13
 RUN apk add --update openssl build-base libseccomp-dev
 RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb
 COPY --from=build /zenith/target/release/pageserver /usr/local/bin
 COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin
 COPY --from=build /zenith/target/release/proxy /usr/local/bin
 COPY --from=pg-build /zenith/tmp_install /usr/local
 COPY docker-entrypoint.sh /docker-entrypoint.sh
 RUN addgroup zenith && adduser -h /data -D -G zenith zenith
 VOLUME ["/data"]
 WORKDIR /data
 USER zenith
 EXPOSE 6400
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD ["pageserver"]
@@ -1,15 +0,0 @@
 #
 # Image with all the required dependencies to build https://github.com/zenithdb/zenith
 # and Postgres from https://github.com/zenithdb/postgres
 # Also includes some rust development and build tools.
 #
 FROM rust:slim-buster
 WORKDIR /zenith
 # Install postgres and zenith build dependencies
 # clang is for rocksdb
 RUN apt-get update && apt-get -yq install automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \
                                          libseccomp-dev pkg-config libssl-dev librocksdb-dev clang
 # Install rust tools
 RUN rustup component add clippy && cargo install cargo-chef cargo-audit
@@ -1,202 +0,0 @@
                                 Apache License
                           Version 2.0, January 2004
                        http://www.apache.org/licenses/
   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
   1. Definitions.
      "License" shall mean the terms and conditions for use, reproduction,
      and distribution as defined by Sections 1 through 9 of this document.
      "Licensor" shall mean the copyright owner or entity authorized by
      the copyright owner that is granting the License.
      "Legal Entity" shall mean the union of the acting entity and all
      other entities that control, are controlled by, or are under common
      control with that entity. For the purposes of this definition,
      "control" means (i) the power, direct or indirect, to cause the
      direction or management of such entity, whether by contract or
      otherwise, or (ii) ownership of fifty percent (50%) or more of the
      outstanding shares, or (iii) beneficial ownership of such entity.
      "You" (or "Your") shall mean an individual or Legal Entity
      exercising permissions granted by this License.
      "Source" form shall mean the preferred form for making modifications,
      including but not limited to software source code, documentation
      source, and configuration files.
      "Object" form shall mean any form resulting from mechanical
      transformation or translation of a Source form, including but
      not limited to compiled object code, generated documentation,
      and conversions to other media types.
      "Work" shall mean the work of authorship, whether in Source or
      Object form, made available under the License, as indicated by a
      copyright notice that is included in or attached to the work
      (an example is provided in the Appendix below).
      "Derivative Works" shall mean any work, whether in Source or Object
      form, that is based on (or derived from) the Work and for which the
      editorial revisions, annotations, elaborations, or other modifications
      represent, as a whole, an original work of authorship. For the purposes
      of this License, Derivative Works shall not include works that remain
      separable from, or merely link (or bind by name) to the interfaces of,
      the Work and Derivative Works thereof.
      "Contribution" shall mean any work of authorship, including
      the original version of the Work and any modifications or additions
      to that Work or Derivative Works thereof, that is intentionally
      submitted to Licensor for inclusion in the Work by the copyright owner
      or by an individual or Legal Entity authorized to submit on behalf of
      the copyright owner. For the purposes of this definition, "submitted"
      means any form of electronic, verbal, or written communication sent
      to the Licensor or its representatives, including but not limited to
      communication on electronic mailing lists, source code control systems,
      and issue tracking systems that are managed by, or on behalf of, the
      Licensor for the purpose of discussing and improving the Work, but
      excluding communication that is conspicuously marked or otherwise
      designated in writing by the copyright owner as "Not a Contribution."
      "Contributor" shall mean Licensor and any individual or Legal Entity
      on behalf of whom a Contribution has been received by Licensor and
      subsequently incorporated within the Work.
   2. Grant of Copyright License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      copyright license to reproduce, prepare Derivative Works of,
      publicly display, publicly perform, sublicense, and distribute the
      Work and such Derivative Works in Source or Object form.
   3. Grant of Patent License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      (except as stated in this section) patent license to make, have made,
      use, offer to sell, sell, import, and otherwise transfer the Work,
      where such license applies only to those patent claims licensable
      by such Contributor that are necessarily infringed by their
      Contribution(s) alone or by combination of their Contribution(s)
      with the Work to which such Contribution(s) was submitted. If You
      institute patent litigation against any entity (including a
      cross-claim or counterclaim in a lawsuit) alleging that the Work
      or a Contribution incorporated within the Work constitutes direct
      or contributory patent infringement, then any patent licenses
      granted to You under this License for that Work shall terminate
      as of the date such litigation is filed.
   4. Redistribution. You may reproduce and distribute copies of the
      Work or Derivative Works thereof in any medium, with or without
      modifications, and in Source or Object form, provided that You
      meet the following conditions:
      (a) You must give any other recipients of the Work or
          Derivative Works a copy of this License; and
      (b) You must cause any modified files to carry prominent notices
          stating that You changed the files; and
      (c) You must retain, in the Source form of any Derivative Works
          that You distribute, all copyright, patent, trademark, and
          attribution notices from the Source form of the Work,
          excluding those notices that do not pertain to any part of
          the Derivative Works; and
      (d) If the Work includes a "NOTICE" text file as part of its
          distribution, then any Derivative Works that You distribute must
          include a readable copy of the attribution notices contained
          within such NOTICE file, excluding those notices that do not
          pertain to any part of the Derivative Works, in at least one
          of the following places: within a NOTICE text file distributed
          as part of the Derivative Works; within the Source form or
          documentation, if provided along with the Derivative Works; or,
          within a display generated by the Derivative Works, if and
          wherever such third-party notices normally appear. The contents
          of the NOTICE file are for informational purposes only and
          do not modify the License. You may add Your own attribution
          notices within Derivative Works that You distribute, alongside
          or as an addendum to the NOTICE text from the Work, provided
          that such additional attribution notices cannot be construed
          as modifying the License.
      You may add Your own copyright statement to Your modifications and
      may provide additional or different license terms and conditions
      for use, reproduction, or distribution of Your modifications, or
      for any such Derivative Works as a whole, provided Your use,
      reproduction, and distribution of the Work otherwise complies with
      the conditions stated in this License.
   5. Submission of Contributions. Unless You explicitly state otherwise,
      any Contribution intentionally submitted for inclusion in the Work
      by You to the Licensor shall be under the terms and conditions of
      this License, without any additional terms or conditions.
      Notwithstanding the above, nothing herein shall supersede or modify
      the terms of any separate license agreement you may have executed
      with Licensor regarding such Contributions.
   6. Trademarks. This License does not grant permission to use the trade
      names, trademarks, service marks, or product names of the Licensor,
      except as required for reasonable and customary use in describing the
      origin of the Work and reproducing the content of the NOTICE file.
   7. Disclaimer of Warranty. Unless required by applicable law or
      agreed to in writing, Licensor provides the Work (and each
      Contributor provides its Contributions) on an "AS IS" BASIS,
      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
      implied, including, without limitation, any warranties or conditions
      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
      PARTICULAR PURPOSE. You are solely responsible for determining the
      appropriateness of using or redistributing the Work and assume any
      risks associated with Your exercise of permissions under this License.
   8. Limitation of Liability. In no event and under no legal theory,
      whether in tort (including negligence), contract, or otherwise,
      unless required by applicable law (such as deliberate and grossly
      negligent acts) or agreed to in writing, shall any Contributor be
      liable to You for damages, including any direct, indirect, special,
      incidental, or consequential damages of any character arising as a
      result of this License or out of the use or inability to use the
      Work (including but not limited to damages for loss of goodwill,
      work stoppage, computer failure or malfunction, or any and all
      other commercial damages or losses), even if such Contributor
      has been advised of the possibility of such damages.
   9. Accepting Warranty or Additional Liability. While redistributing
      the Work or Derivative Works thereof, You may choose to offer,
      and charge a fee for, acceptance of support, warranty, indemnity,
      or other liability obligations and/or rights consistent with this
      License. However, in accepting such obligations, You may act only
      on Your own behalf and on Your sole responsibility, not on behalf
      of any other Contributor, and only if You agree to indemnify,
      defend, and hold each Contributor harmless for any liability
      incurred by, or claims asserted against, such Contributor by reason
      of your accepting any such warranty or additional liability.
   END OF TERMS AND CONDITIONS
   APPENDIX: How to apply the Apache License to your work.
      To apply the Apache License to your work, attach the following
      boilerplate notice, with the fields enclosed by brackets "[]"
      replaced with your own identifying information. (Don't include
      the brackets!)  The text should be enclosed in the appropriate
      comment syntax for the file format. We also recommend that a
      file or class name and description of purpose be included on the
      same "printed page" as the copyright notice for easier
      identification within third-party archives.
   Copyright [yyyy] [name of copyright owner]
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
       http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
@@ -1,11 +1,3 @@
 # Seccomp BPF is only available for Linux
 UNAME_S := $(shell uname -s)
 ifeq ($(UNAME_S),Linux)
 	SECCOMP = --with-libseccomp
 else
 	SECCOMP =
 endif
 #
 # Top level Makefile to build Zenith and PostgreSQL
 #
@@ -29,12 +21,8 @@ tmp_install/build/config.status:
 	+@echo "Configuring postgres build"
 	mkdir -p tmp_install/build
 	(cd tmp_install/build && \
-	../../vendor/postgres/configure CFLAGS='-O0 -g3 $(CFLAGS)' \
+	../../vendor/postgres/configure CFLAGS='-O0' --enable-debug --enable-cassert \
-		--enable-cassert \
+	    --enable-depend --with-libxml --prefix=$(abspath tmp_install) > configure.log)
 		--enable-debug \
 		--enable-depend \
 		$(SECCOMP) \
 		--prefix=$(abspath tmp_install) > configure.log)
 # nicer alias for running 'configure'
 postgres-configure: tmp_install/build/config.status
@@ -50,9 +38,8 @@ postgres: postgres-configure
 	+@echo "Compiling PostgreSQL"
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 install
 	+@echo "Compiling contrib/zenith"
-	$(MAKE) -C tmp_install/build/contrib/zenith install
+	(cd vendor/postgres/contrib/zenith && \
-	+@echo "Compiling contrib/zenith_test_utils"
+	$(MAKE) PG_CONFIG=$(abspath tmp_install)/bin/pg_config install USE_PGXS=1)
 	$(MAKE) -C tmp_install/build/contrib/zenith_test_utils install
 postgres-clean:
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 clean
@@ -1 +0,0 @@
 ./test_runner/Pipfile
@@ -1 +0,0 @@
 ./test_runner/Pipfile.lock
@@ -4,34 +4,20 @@ Zenith substitutes PostgreSQL storage layer and redistributes data across a clus
 ## Running local installation
-1. Install build dependencies and other useful packages
+1. Build zenith and patched postgres
 On Ubuntu or Debian this set of packages should be sufficient to build the code:
 ```text
 apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libseccomp-dev \
 libssl-dev clang
 ```
 [Rust] 1.52 or later is also required.
 To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively.
 To run the integration tests (not required to use the code), install
 Python (3.6 or higher), and install python3 packages with `pipenv` using `pipenv install` in the project directory.
 2. Build zenith and patched postgres
 ```sh
-git clone --recursive https://github.com/zenithdb/zenith.git
+git clone --recursive https://github.com/libzenith/zenith.git
 cd zenith
 make -j5
 ```
-3. Start pageserver and postgres on top of it (should be called from repo root):
+2. Start pageserver and postgres on top of it (should be called from repo root):
 ```sh
 # Create repository in .zenith with proper paths to binaries and data
 # Later that would be responsibility of a package install script
 > ./target/debug/zenith init
-pageserver init succeeded
+<...>
 new zenith repository was created in .zenith
 # start pageserver
 > ./target/debug/zenith start
@@ -49,8 +35,8 @@ BRANCH	ADDRESS		LSN		STATUS
 main	127.0.0.1:55432	0/1609610	running
 ```
-4. Now it is possible to connect to postgres and run some queries:
+3. Now it is possible to connect to postgres and run some queries:
-```text
+```sh
 > psql -p55432 -h 127.0.0.1 postgres
 postgres=# CREATE TABLE t(key int primary key, value text);
 CREATE TABLE
@@ -63,7 +49,7 @@ postgres=# select * from t;
 (1 row)
 ```
-5. And create branches and run postgres on them:
+4. And create branches and run postgres on them:
 ```sh
 # create branch named migration_check
 > ./target/debug/zenith branch migration_check main
@@ -83,7 +69,7 @@ waiting for server to start.... done
 # but all modifications would not affect data in original postgres
 > psql -p55433 -h 127.0.0.1 postgres
 postgres=# select * from t;
- key | value
+ key | value 
 -----+-------
   1 | 1
 (1 row)
@@ -97,60 +83,33 @@ INSERT 0 1
 ```sh
 git clone --recursive https://github.com/libzenith/zenith.git
 make # builds also postgres and installs it to ./tmp_install
 cd test_runner
 pytest
 ```
 ## Documentation
 Now we use README files to cover design ideas and overall architecture for each module and `rustdoc` style documentation comments. See also [/docs/](/docs/) a top-level overview of all available markdown documentation.
 To view your `rustdoc` documentation in a browser, try running `cargo doc --no-deps --open`
 ## Source tree layout
-`/control_plane`:
+/walkeeper:
-Local control plane.
+WAL safekeeper. Written in Rust.
 Functions to start, configure and stop pageserver and postgres instances running as a local processes.
 Intended to be used in integration tests and in CLI tools for local installations.
-`/zenith`
+/pageserver:
 Main entry point for the 'zenith' CLI utility.
 TODO: Doesn't it belong to control_plane?
 `/postgres_ffi`:
 Utility functions for interacting with PostgreSQL file formats.
 Misc constants, copied from PostgreSQL headers.
 `/zenith_utils`:
 Helpers that are shared between other crates in this repository.
 `/walkeeper`:
 WAL safekeeper (also known as WAL acceptor). Written in Rust.
 `/pageserver`:
 Page Server. Written in Rust.
 Depends on the modified 'postgres' binary for WAL redo.
-`/vendor/postgres`:
+/integration_tests:
 Tests with different combinations of a Postgres compute node, WAL safekeeper and Page Server.
 /vendor/postgres:
 PostgreSQL source tree, with the modifications needed for Zenith.
-`/vendor/postgres/contrib/zenith`:
+/vendor/postgres/src/bin/safekeeper:
-PostgreSQL extension that implements storage manager API and network communications with remote page server.
+Extension (safekeeper_proxy) that runs in the compute node, and connects to the WAL safekeepers
 and streams the WAL
 `/test_runner`:
 Integration tests, written in Python using the `pytest` framework.
 `test_runner/zenith_regress`:
 Quick way to add new SQL regression test to integration tests set.
@@ -16,10 +16,12 @@ toml = "0.5"
 lazy_static = "1.4"
 regex = "1"
 anyhow = "1.0"
 # hex = "0.4.3"
 bytes = "1.0.1"
 # fs_extra = "1.2.0"
 nix = "0.20"
 # thiserror = "1"
 url = "2.2.2"
 hex = { version = "0.4.3", features = ["serde"] }
 pageserver = { path = "../pageserver" }
 walkeeper = { path = "../walkeeper" }
@@ -7,19 +7,17 @@ use std::sync::Arc;
 use std::time::Duration;
 use std::{collections::BTreeMap, path::PathBuf};
 use std::{
-    fs::{self, File, OpenOptions},
+    fs::{self, OpenOptions},
    io::Read,
 };
 use anyhow::{Context, Result};
 use lazy_static::lazy_static;
 use regex::Regex;
 use zenith_utils::connstring::connection_host_port;
 use zenith_utils::postgres_backend::AuthType;
 use zenith_utils::zid::ZTenantId;
 use zenith_utils::zid::ZTimelineId;
 use crate::local_env::LocalEnv;
 use pageserver::ZTimelineId;
 use crate::storage::PageServerNode;
 //
@@ -28,36 +26,27 @@ use crate::storage::PageServerNode;
 pub struct ComputeControlPlane {
    base_port: u16,
    pageserver: Arc<PageServerNode>,
-    pub nodes: BTreeMap<(ZTenantId, String), Arc<PostgresNode>>,
+    pub nodes: BTreeMap<String, Arc<PostgresNode>>,
    env: LocalEnv,
 }
 impl ComputeControlPlane {
    // Load current nodes with ports from data directories on disk
    // Directory structure has the following layout:
    // pgdatadirs
    // |- tenants
    // |  |- <tenant_id>
    // |  |   |- <branch name>
    pub fn load(env: LocalEnv) -> Result<ComputeControlPlane> {
        // TODO: since pageserver do not have config file yet we believe here that
        // it is running on default port. Change that when pageserver will have config.
        let pageserver = Arc::new(PageServerNode::from_env(&env));
        let mut nodes = BTreeMap::default();
        let pgdatadirspath = &env.pg_data_dirs_path();
-
+        let nodes: Result<BTreeMap<_, _>> = fs::read_dir(&pgdatadirspath)
        for tenant_dir in fs::read_dir(&pgdatadirspath)
            .with_context(|| format!("failed to list {}", pgdatadirspath.display()))?
-        {
+            .into_iter()
-            let tenant_dir = tenant_dir?;
+            .map(|f| {
-            for timeline_dir in fs::read_dir(tenant_dir.path())
+                PostgresNode::from_dir_entry(f?, &env, &pageserver)
-                .with_context(|| format!("failed to list {}", tenant_dir.path().display()))?
+                    .map(|node| (node.name.clone(), Arc::new(node)))
-            {
+            })
-                let node = PostgresNode::from_dir_entry(timeline_dir?, &env, &pageserver)?;
+            .collect();
-                nodes.insert((node.tenantid, node.name.clone()), Arc::new(node));
+        let nodes = nodes?;
            }
        }
        Ok(ComputeControlPlane {
            base_port: 55431,
@@ -85,40 +74,86 @@ impl ComputeControlPlane {
        }
    }
-    pub fn new_node(
+    /// Connect to a page server, get base backup, and untar it to initialize a
    /// new data directory
    pub fn new_from_page_server(
        &mut self,
-        tenantid: ZTenantId,
+        is_test: bool,
-        branch_name: &str,
+        timelineid: ZTimelineId,
-        config_only: bool,
+        name: &str,
    ) -> Result<Arc<PostgresNode>> {
        let timeline_id = self
            .pageserver
            .branch_get_by_name(&tenantid, branch_name)?
            .timeline_id;
        let node = Arc::new(PostgresNode {
-            name: branch_name.to_owned(),
+            name: name.to_owned(),
            address: SocketAddr::new("127.0.0.1".parse().unwrap(), self.get_port()),
            env: self.env.clone(),
            pageserver: Arc::clone(&self.pageserver),
-            is_test: false,
+            is_test,
-            timelineid: timeline_id,
+            timelineid,
            tenantid,
        });
-        node.init_from_page_server(self.env.auth_type, config_only)?;
+        node.init_from_page_server()?;
-        self.nodes
+        self.nodes.insert(node.name.clone(), Arc::clone(&node));
-            .insert((tenantid, node.name.clone()), Arc::clone(&node));
+
        Ok(node)
    }
    pub fn new_test_node(&mut self, branch_name: &str) -> Arc<PostgresNode> {
        let timeline_id = self
            .pageserver
            .branch_get_by_name(branch_name)
            .expect("failed to get timeline_id")
            .timeline_id;
        let node = self.new_from_page_server(true, timeline_id, branch_name);
        let node = node.unwrap();
        // Configure the node to stream WAL directly to the pageserver
        node.append_conf(
            "postgresql.conf",
            format!(
-                concat!(
+                "shared_preload_libraries = zenith\n\
-                    "synchronous_standby_names = 'pageserver'\n", // TODO: add a new function arg?
+                zenith.callmemaybe_connstring = '{}'\n", // FIXME escaping
-                    "zenith.callmemaybe_connstring = '{}'\n",     // FIXME escaping
+                node.connstr()
-                ),
+            )
-                node.connstr(),
+            .as_str(),
        )
        .unwrap();
        node
    }
    pub fn new_test_master_node(&mut self, branch_name: &str) -> Arc<PostgresNode> {
        let timeline_id = self
            .pageserver
            .branch_get_by_name(branch_name)
            .expect("failed to get timeline_id")
            .timeline_id;
        let node = self
            .new_from_page_server(true, timeline_id, branch_name)
            .unwrap();
        node.append_conf(
            "postgresql.conf",
            "synchronous_standby_names = 'safekeeper_proxy'\n",
        )
        .unwrap();
        node
    }
    pub fn new_node(&mut self, branch_name: &str) -> Result<Arc<PostgresNode>> {
        let timeline_id = self.pageserver.branch_get_by_name(branch_name)?.timeline_id;
        let node = self.new_from_page_server(false, timeline_id, branch_name)?;
        // Configure the node to stream WAL directly to the pageserver
        node.append_conf(
            "postgresql.conf",
            format!(
                "shared_preload_libraries = zenith\n\
                zenith.callmemaybe_connstring = '{}'\n", // FIXME escaping
                node.connstr()
            )
            .as_str(),
        )?;
@@ -129,7 +164,6 @@ impl ComputeControlPlane {
 ///////////////////////////////////////////////////////////////////////////////
 #[derive(Debug)]
 pub struct PostgresNode {
    pub address: SocketAddr,
    name: String,
@@ -137,7 +171,6 @@ pub struct PostgresNode {
    pageserver: Arc<PageServerNode>,
    is_test: bool,
    pub timelineid: ZTimelineId,
    pub tenantid: ZTenantId,
 }
 impl PostgresNode {
@@ -157,8 +190,6 @@ impl PostgresNode {
            static ref CONF_PORT_RE: Regex = Regex::new(r"(?m)^\s*port\s*=\s*(\d+)\s*$").unwrap();
            static ref CONF_TIMELINE_RE: Regex =
                Regex::new(r"(?m)^\s*zenith.zenith_timeline\s*=\s*'(\w+)'\s*$").unwrap();
            static ref CONF_TENANT_RE: Regex =
                Regex::new(r"(?m)^\s*zenith.zenith_tenant\s*=\s*'(\w+)'\s*$").unwrap();
        }
        // parse data directory name
@@ -206,22 +237,6 @@ impl PostgresNode {
            .parse()
            .with_context(|| err_msg)?;
        // parse tenant
        let err_msg = format!(
            "failed to find tenant definition in config file {}",
            cfg_path.to_str().unwrap()
        );
        let tenantid = CONF_TENANT_RE
            .captures(config.as_str())
            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 1"))?
            .iter()
            .last()
            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 2"))?
            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 3"))?
            .as_str()
            .parse()
            .with_context(|| err_msg)?;
        // ok now
        Ok(PostgresNode {
            address: SocketAddr::new("127.0.0.1".parse().unwrap(), port),
@@ -230,19 +245,46 @@ impl PostgresNode {
            pageserver: Arc::clone(pageserver),
            is_test: false,
            timelineid,
            tenantid,
        })
    }
-    pub fn do_basebackup(&self) -> Result<()> {
+    // Connect to a page server, get base backup, and untar it to initialize a
    // new data directory
    pub fn init_from_page_server(&self) -> Result<()> {
        let pgdata = self.pgdata();
        println!(
            "Extracting base backup to create postgres instance: path={} port={}",
            pgdata.display(),
            self.address.port()
        );
-        let sql = format!("basebackup {} {}", self.tenantid, self.timelineid);
+        // initialize data directory
        if self.is_test {
            fs::remove_dir_all(&pgdata).ok();
        }
        let sql = format!("basebackup {}", self.timelineid);
        let mut client = self
            .pageserver
            .page_server_psql_client()
            .with_context(|| "connecting to page server failed")?;
        fs::create_dir_all(&pgdata)
            .with_context(|| format!("could not create data directory {}", pgdata.display()))?;
        fs::set_permissions(pgdata.as_path(), fs::Permissions::from_mode(0o700)).with_context(
            || {
                format!(
                    "could not set permissions in data directory {}",
                    pgdata.display()
                )
            },
        )?;
        // FIXME: The compute node should be able to stream the WAL it needs from the WAL safekeepers or archive.
        // But that's not implemented yet. For now, 'pg_wal' is included in the base backup tarball that
        // we receive from the Page Server, so we don't need to create the empty 'pg_wal' directory here.
        //fs::create_dir_all(pgdata.join("pg_wal"))?;
        let mut copyreader = client
            .copy_out(sql.as_str())
            .with_context(|| "page server 'basebackup' command failed")?;
@@ -258,52 +300,11 @@ impl PostgresNode {
        ar.unpack(&pgdata)
            .with_context(|| "extracting page backup failed")?;
-        Ok(())
+        // listen for selected port
    }
    // Connect to a page server, get base backup, and untar it to initialize a
    // new data directory
    pub fn init_from_page_server(&self, auth_type: AuthType, config_only: bool) -> Result<()> {
        let pgdata = self.pgdata();
        println!(
            "Extracting base backup to create postgres instance: path={} port={}",
            pgdata.display(),
            self.address.port()
        );
        // initialize data directory
        if self.is_test {
            fs::remove_dir_all(&pgdata).ok();
        }
        fs::create_dir_all(&pgdata)
            .with_context(|| format!("could not create data directory {}", pgdata.display()))?;
        fs::set_permissions(pgdata.as_path(), fs::Permissions::from_mode(0o700)).with_context(
            || {
                format!(
                    "could not set permissions in data directory {}",
                    pgdata.display()
                )
            },
        )?;
        if config_only {
            //Just create an empty config file
            File::create(self.pgdata().join("postgresql.conf").to_str().unwrap())?;
        } else {
            self.do_basebackup()?;
            fs::create_dir_all(self.pgdata().join("pg_wal"))?;
            fs::create_dir_all(self.pgdata().join("pg_wal").join("archive_status"))?;
        }
        // wal_log_hints is mandatory when running against pageserver (see gh issue#192)
        // TODO: is it possible to check wal_log_hints at pageserver side via XLOG_PARAMETER_CHANGE?
        self.append_conf(
            "postgresql.conf",
            &format!(
                "max_wal_senders = 10\n\
                 wal_log_hints = on\n\
                 max_replication_slots = 10\n\
                 hot_standby = on\n\
                 shared_buffers = 1MB\n\
@@ -320,40 +321,33 @@ impl PostgresNode {
        // Never clean up old WAL. TODO: We should use a replication
        // slot or something proper, to prevent the compute node
-        // from removing WAL that hasn't been streamed to the safekeeper or
+        // from removing WAL that hasn't been streamed to the safekeepr or
-        // page server yet. (gh issue #349)
+        // page server yet. But this will do for now.
        self.append_conf("postgresql.conf", "wal_keep_size='10TB'\n")?;
-        // set up authentication
+        // Connect it to the page server.
        let password = if let AuthType::ZenithJWT = auth_type {
            "$ZENITH_AUTH_TOKEN"
        } else {
            ""
        };
        // Configure that node to take pages from pageserver
        let (host, port) = connection_host_port(&self.pageserver.connection_config);
        self.append_conf(
            "postgresql.conf",
-            format!(
+            &format!(
-                concat!(
+                "shared_preload_libraries = zenith \n\
-                    "shared_preload_libraries = zenith\n",
+                 zenith.page_server_connstring = 'host={} port={}'\n\
-                    // $ZENITH_AUTH_TOKEN will be replaced with value from environment variable during compute pg startup
+                 zenith.zenith_timeline='{}'\n",
-                    // it is done this way because otherwise user will be able to retrieve the value using SHOW command or pg_settings
+                self.pageserver.address().ip(),
-                    "zenith.page_server_connstring = 'host={} port={} password={}'\n",
+                self.pageserver.address().port(),
-                    "zenith.zenith_timeline='{}'\n",
+                self.timelineid
-                    "zenith.zenith_tenant='{}'\n",
+            ),
                ),
                host, port, password, self.timelineid, self.tenantid,
            )
            .as_str(),
        )?;
        fs::create_dir_all(self.pgdata().join("pg_wal"))?;
        fs::create_dir_all(self.pgdata().join("pg_wal").join("archive_status"))?;
        self.pg_resetwal(&["-f"])?;
        Ok(())
    }
    pub fn pgdata(&self) -> PathBuf {
-        self.env.pg_data_dir(&self.tenantid, &self.name)
+        self.env.pg_data_dir(&self.name)
    }
    pub fn status(&self) -> &str {
@@ -377,88 +371,57 @@ impl PostgresNode {
        Ok(())
    }
-    fn pg_ctl(&self, args: &[&str], auth_token: &Option<String>) -> Result<()> {
+    fn pg_ctl(&self, args: &[&str]) -> Result<()> {
        let pg_ctl_path = self.env.pg_bin_dir().join("pg_ctl");
        let mut cmd = Command::new(pg_ctl_path);
        cmd.args(
            [
                &[
                    "-D",
                    self.pgdata().to_str().unwrap(),
                    "-l",
                    self.pgdata().join("pg.log").to_str().unwrap(),
                    "-w", //wait till pg_ctl actually does what was asked
                ],
                args,
            ]
            .concat(),
        )
        .env_clear()
        .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
        .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap());
        if let Some(token) = auth_token {
            cmd.env("ZENITH_AUTH_TOKEN", token);
        }
        let pg_ctl = cmd.status().with_context(|| "pg_ctl failed")?;
        let pg_ctl = Command::new(pg_ctl_path)
            .args(
                [
                    &[
                        "-D",
                        self.pgdata().to_str().unwrap(),
                        "-l",
                        self.pgdata().join("log").to_str().unwrap(),
                    ],
                    args,
                ]
                .concat(),
            )
            .env_clear()
            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
            .status()
            .with_context(|| "pg_ctl failed")?;
        if !pg_ctl.success() {
            anyhow::bail!("pg_ctl failed");
        }
        Ok(())
    }
-    pub fn start(&self, auth_token: &Option<String>) -> Result<()> {
+    fn pg_resetwal(&self, args: &[&str]) -> Result<()> {
-        // Bail if the node already running.
+        let pg_resetwal_path = self.env.pg_bin_dir().join("pg_resetwal");
-        if self.status() == "running" {
+
-            anyhow::bail!("The node is already running");
+        let pg_ctl = Command::new(pg_resetwal_path)
            .args([&["-D", self.pgdata().to_str().unwrap()], args].concat())
            .status()
            .with_context(|| "pg_resetwal failed")?;
        if !pg_ctl.success() {
            anyhow::bail!("pg_resetwal failed");
        }
-
+        Ok(())
        // 1. We always start compute node from scratch, so
        // if old dir exists, preserve config files and drop the directory
        // XXX Now we only use 'postgresql.conf'.
        // If we will need 'pg_hba.conf', support it here too
        let postgresql_conf_path = self.pgdata().join("postgresql.conf");
        let postgresql_conf = fs::read(postgresql_conf_path.clone()).with_context(|| {
            format!(
                "failed to read config file in {}",
                postgresql_conf_path.to_str().unwrap()
            )
        })?;
        println!(
            "Destroying postgres data directory '{}'",
            self.pgdata().to_str().unwrap()
        );
        fs::remove_dir_all(&self.pgdata())?;
        // 2. Create new node
        self.init_from_page_server(self.env.auth_type, false)?;
        // 3. Bring back config files
        if let Ok(mut file) = OpenOptions::new()
            .append(false)
            .write(true)
            .open(&postgresql_conf_path)
        {
            file.write_all(&postgresql_conf)?;
            file.sync_all()?;
        }
        // 4. Finally start the compute node postgres
        println!("Starting postgres node at '{}'", self.connstr());
        self.pg_ctl(&["start"], auth_token)
    }
-    pub fn restart(&self, auth_token: &Option<String>) -> Result<()> {
+    pub fn start(&self) -> Result<()> {
-        self.pg_ctl(&["restart"], auth_token)
+        println!("Starting postgres node at '{}'", self.connstr());
        self.pg_ctl(&["start"])
    }
    pub fn restart(&self) -> Result<()> {
        self.pg_ctl(&["restart"])
    }
    pub fn stop(&self, destroy: bool) -> Result<()> {
-        self.pg_ctl(&["-m", "immediate", "stop"], &None)?;
+        self.pg_ctl(&["-m", "immediate", "stop"])?;
        if destroy {
            println!(
                "Destroying postgres data directory '{}'",
@@ -471,11 +434,10 @@ impl PostgresNode {
    pub fn connstr(&self) -> String {
        format!(
-            "host={} port={} user={} dbname={}",
+            "host={} port={} user={}",
            self.address.ip(),
            self.address.port(),
-            "zenith_admin",
+            self.whoami()
            "postgres"
        )
    }
@@ -1,7 +1,7 @@
 //
 // Local control plane.
 //
-// Can start, configure and stop postgres instances running as a local processes.
+// Can start, cofigure and stop postgres instances running as a local processes.
 //
 // Intended to be used in integration tests and in CLI tools for
 // local installations.
@@ -4,24 +4,19 @@
 // Now it also provides init method which acts like a stub for proper installation
 // script which will use local paths.
 //
-use anyhow::{anyhow, Context, Result};
+use anyhow::{anyhow, Result};
 use hex;
 use serde::{Deserialize, Serialize};
 use std::fs;
 use std::path::PathBuf;
 use std::process::{Command, Stdio};
 use std::{collections::BTreeMap, env};
 use url::Url;
 use zenith_utils::auth::{encode_from_key_path, Claims, Scope};
 use zenith_utils::postgres_backend::AuthType;
 use zenith_utils::zid::ZTenantId;
 pub type Remotes = BTreeMap<String, String>;
 //
 // This data structures represent deserialized zenith CLI config
 //
-#[derive(Serialize, Deserialize, Clone, Debug)]
+#[derive(Serialize, Deserialize, Clone)]
 pub struct LocalEnv {
    // Pageserver connection strings
    pub pageserver_connstring: String,
@@ -38,22 +33,6 @@ pub struct LocalEnv {
    // Path to pageserver binary. Empty for remote pageserver.
    pub zenith_distrib_dir: Option<PathBuf>,
    // keeping tenant id in config to reduce copy paste when running zenith locally with single tenant
    #[serde(with = "hex")]
    pub tenantid: ZTenantId,
    // Repository format, 'rocksdb' or 'layered' or None for default
    pub repository_format: Option<String>,
    // jwt auth token used for communication with pageserver
    pub auth_token: String,
    // used to determine which auth type is used
    pub auth_type: AuthType,
    // used to issue tokens during e.g pg start
    pub private_key_path: PathBuf,
    pub remotes: Remotes,
 }
@@ -70,18 +49,16 @@ impl LocalEnv {
        Ok(self
            .zenith_distrib_dir
            .as_ref()
-            .ok_or_else(|| anyhow!("Can not manage remote pageserver"))?
+            .ok_or(anyhow!("Can not manage remote pageserver"))?
            .join("pageserver"))
    }
    pub fn pg_data_dirs_path(&self) -> PathBuf {
-        self.base_data_dir.join("pgdatadirs").join("tenants")
+        self.base_data_dir.join("pgdatadirs")
    }
-    pub fn pg_data_dir(&self, tenantid: &ZTenantId, branch_name: &str) -> PathBuf {
+    pub fn pg_data_dir(&self, name: &str) -> PathBuf {
-        self.pg_data_dirs_path()
+        self.pg_data_dirs_path().join(name)
            .join(tenantid.to_string())
            .join(branch_name)
    }
    // TODO: move pageserver files into ./pageserver
@@ -100,12 +77,7 @@ fn base_path() -> PathBuf {
 //
 // Initialize a new Zenith repository
 //
-pub fn init(
+pub fn init(remote_pageserver: Option<&str>) -> Result<()> {
    remote_pageserver: Option<&str>,
    tenantid: ZTenantId,
    auth_type: AuthType,
    repository_format: Option<&str>,
 ) -> Result<()> {
    // check if config already exists
    let base_path = base_path();
    if base_path.exists() {
@@ -114,7 +86,6 @@ pub fn init(
            base_path.to_str().unwrap()
        );
    }
    fs::create_dir(&base_path)?;
    // ok, now check that expected binaries are present
@@ -131,43 +102,8 @@ pub fn init(
        anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
    }
-    // generate keys for jwt
+    fs::create_dir(&base_path)?;
-    // openssl genrsa -out private_key.pem 2048
+    fs::create_dir(base_path.join("pgdatadirs"))?;
    let private_key_path = base_path.join("auth_private_key.pem");
    let keygen_output = Command::new("openssl")
        .arg("genrsa")
        .args(&["-out", private_key_path.to_str().unwrap()])
        .arg("2048")
        .stdout(Stdio::null())
        .output()
        .with_context(|| "failed to generate auth private key")?;
    if !keygen_output.status.success() {
        anyhow::bail!(
            "openssl failed: '{}'",
            String::from_utf8_lossy(&keygen_output.stderr)
        );
    }
    let public_key_path = base_path.join("auth_public_key.pem");
    // openssl rsa -in private_key.pem -pubout -outform PEM -out public_key.pem
    let keygen_output = Command::new("openssl")
        .arg("rsa")
        .args(&["-in", private_key_path.to_str().unwrap()])
        .arg("-pubout")
        .args(&["-outform", "PEM"])
        .args(&["-out", public_key_path.to_str().unwrap()])
        .stdout(Stdio::null())
        .output()
        .with_context(|| "failed to generate auth private key")?;
    if !keygen_output.status.success() {
        anyhow::bail!(
            "openssl failed: '{}'",
            String::from_utf8_lossy(&keygen_output.stderr)
        );
    }
    let auth_token =
        encode_from_key_path(&Claims::new(None, Scope::PageServerApi), &private_key_path)?;
    let conf = if let Some(addr) = remote_pageserver {
        // check that addr is parsable
@@ -179,11 +115,6 @@ pub fn init(
            zenith_distrib_dir: None,
            base_data_dir: base_path,
            remotes: BTreeMap::default(),
            tenantid,
            repository_format: repository_format.map(|x| x.into()),
            auth_token,
            auth_type,
            private_key_path,
        }
    } else {
        // Find zenith binaries.
@@ -198,16 +129,9 @@ pub fn init(
            zenith_distrib_dir: Some(zenith_distrib_dir),
            base_data_dir: base_path,
            remotes: BTreeMap::default(),
            tenantid,
            repository_format: repository_format.map(|x| x.into()),
            auth_token,
            auth_type,
            private_key_path,
        }
    };
    fs::create_dir_all(conf.pg_data_dirs_path())?;
    let toml = toml::to_string_pretty(&conf)?;
    fs::write(conf.base_data_dir.join("config"), toml)?;
@@ -1,5 +1,5 @@
 use std::collections::HashMap;
-use std::net::TcpStream;
+use std::net::{SocketAddr, TcpStream};
 use std::path::PathBuf;
 use std::process::Command;
 use std::thread;
@@ -8,77 +8,53 @@ use std::time::Duration;
 use anyhow::{anyhow, bail, Result};
 use nix::sys::signal::{kill, Signal};
 use nix::unistd::Pid;
-use postgres::{Config, NoTls};
+use postgres::{Client, NoTls};
 use zenith_utils::postgres_backend::AuthType;
 use zenith_utils::zid::ZTenantId;
 use crate::local_env::LocalEnv;
 use crate::read_pidfile;
 use pageserver::branches::BranchInfo;
 use zenith_utils::connstring::connection_address;
 //
 // Control routines for pageserver.
 //
 // Used in CLI and tests.
 //
 #[derive(Debug)]
 pub struct PageServerNode {
    pub kill_on_exit: bool,
-    pub connection_config: Config,
+    pub listen_address: Option<SocketAddr>,
    pub env: LocalEnv,
 }
 impl PageServerNode {
    pub fn from_env(env: &LocalEnv) -> PageServerNode {
        let password = if matches!(env.auth_type, AuthType::ZenithJWT) {
            &env.auth_token
        } else {
            ""
        };
        PageServerNode {
            kill_on_exit: false,
-            connection_config: Self::default_config(password), // default
+            listen_address: None, // default
            env: env.clone(),
        }
    }
-    fn default_config(password: &str) -> Config {
+    pub fn address(&self) -> SocketAddr {
-        format!("postgresql://no_user:{}@localhost:64000/no_db", password)
+        match self.listen_address {
-            .parse()
+            Some(addr) => addr,
-            .unwrap()
+            None => "127.0.0.1:64000".parse().unwrap(),
        }
    }
-    pub fn init(
+    pub fn init(&self) -> Result<()> {
        &self,
        create_tenant: Option<&str>,
        enable_auth: bool,
        repository_format: Option<&str>,
    ) -> Result<()> {
        let mut cmd = Command::new(self.env.pageserver_bin()?);
        let mut args = vec![
            "--init",
            "-D",
            self.env.base_data_dir.to_str().unwrap(),
            "--postgres-distrib",
            self.env.pg_distrib_dir.to_str().unwrap(),
        ];
        if enable_auth {
            args.extend(&["--auth-validation-public-key-path", "auth_public_key.pem"]);
            args.extend(&["--auth-type", "ZenithJWT"]);
        }
        if let Some(repo_format) = repository_format {
            args.extend(&["--repository-format", repo_format]);
        }
        create_tenant.map(|tenantid| args.extend(&["--create-tenant", tenantid]));
        let status = cmd
-            .args(args)
+            .args(&["--init", "-D", self.env.base_data_dir.to_str().unwrap()])
            .env_clear()
            .env("RUST_BACKTRACE", "1")
            .env(
                "POSTGRES_DISTRIB_DIR",
                self.env.pg_distrib_dir.to_str().unwrap(),
            )
            .env("ZENITH_REPO_DIR", self.repo_path())
            .env("PATH", self.env.pg_bin_dir().to_str().unwrap()) // needs postres-wal-redo binary
            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
            .status()
            .expect("pageserver init failed");
@@ -100,15 +76,28 @@ impl PageServerNode {
    pub fn start(&self) -> Result<()> {
        println!(
            "Starting pageserver at '{}' in {}",
-            connection_address(&self.connection_config),
+            self.address(),
            self.repo_path().display()
        );
        let mut cmd = Command::new(self.env.pageserver_bin()?);
-        cmd.args(&["-D", self.repo_path().to_str().unwrap()])
+        cmd.args(&[
-            .arg("-d")
+            "-l",
-            .env_clear()
+            self.address().to_string().as_str(),
-            .env("RUST_BACKTRACE", "1");
+            "-D",
            self.repo_path().to_str().unwrap(),
        ])
        .arg("-d")
        .env_clear()
        .env("RUST_BACKTRACE", "1")
        .env(
            "POSTGRES_DISTRIB_DIR",
            self.env.pg_distrib_dir.to_str().unwrap(),
        )
        .env("ZENITH_REPO_DIR", self.repo_path())
        .env("PATH", self.env.pg_bin_dir().to_str().unwrap()) // needs postres-wal-redo binary
        .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
        .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap());
        if !cmd.status()?.success() {
            bail!(
@@ -120,21 +109,18 @@ impl PageServerNode {
        // It takes a while for the page server to start up. Wait until it is
        // open for business.
        for retries in 1..15 {
-            match self.page_server_psql_client() {
+            let client = self.page_server_psql_client();
-                Ok(_) => {
+            if client.is_ok() {
-                    println!("Pageserver started");
+                break;
-                    return Ok(());
+            } else {
-                }
+                println!("Pageserver not responding yet, retrying ({})...", retries);
-                Err(err) => {
+                thread::sleep(Duration::from_secs(1));
                    println!(
                        "Pageserver not responding yet, err {} retrying ({})...",
                        err, retries
                    );
                    thread::sleep(Duration::from_secs(1));
                }
            }
        }
-        bail!("pageserver failed to start");
+
        println!("Pageserver started");
        Ok(())
    }
    pub fn stop(&self) -> Result<()> {
@@ -145,55 +131,47 @@ impl PageServerNode {
        }
        // wait for pageserver stop
        let address = connection_address(&self.connection_config);
        for _ in 0..5 {
-            let stream = TcpStream::connect(&address);
+            let stream = TcpStream::connect(self.address());
            thread::sleep(Duration::from_secs(1));
            if let Err(_e) = stream {
                println!("Pageserver stopped");
                return Ok(());
            }
-            println!("Stopping pageserver on {}", address);
+            println!("Stopping pageserver on {}", self.address());
        }
        bail!("Failed to stop pageserver with pid {}", pid);
    }
    pub fn page_server_psql(&self, sql: &str) -> Vec<postgres::SimpleQueryMessage> {
-        let mut client = self.connection_config.connect(NoTls).unwrap();
+        let connstring = format!(
            "host={} port={} dbname={} user={}",
            self.address().ip(),
            self.address().port(),
            "no_db",
            "no_user",
        );
        let mut client = Client::connect(connstring.as_str(), NoTls).unwrap();
        println!("Pageserver query: '{}'", sql);
        client.simple_query(sql).unwrap()
    }
    pub fn page_server_psql_client(&self) -> Result<postgres::Client, postgres::Error> {
-        self.connection_config.connect(NoTls)
+        let connstring = format!(
            "host={} port={} dbname={} user={}",
            self.address().ip(),
            self.address().port(),
            "no_db",
            "no_user",
        );
        Client::connect(connstring.as_str(), NoTls)
    }
-    pub fn tenants_list(&self) -> Result<Vec<String>> {
+    pub fn branches_list(&self) -> Result<Vec<BranchInfo>> {
        let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query("tenant_list")?;
+        let query_result = client.simple_query("branch_list")?;
        let tenants_json = query_result
            .first()
            .map(|msg| match msg {
                postgres::SimpleQueryMessage::Row(row) => row.get(0),
                _ => None,
            })
            .flatten()
            .ok_or_else(|| anyhow!("missing tenants"))?;
        Ok(serde_json::from_str(tenants_json)?)
    }
    pub fn tenant_create(&self, tenantid: &ZTenantId) -> Result<()> {
        let mut client = self.page_server_psql_client()?;
        client.simple_query(format!("tenant_create {}", tenantid).as_str())?;
        Ok(())
    }
    pub fn branches_list(&self, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
        let mut client = self.page_server_psql_client()?;
        let query_result = client.simple_query(&format!("branch_list {}", tenantid))?;
        let branches_json = query_result
            .first()
            .map(|msg| match msg {
@@ -203,19 +181,14 @@ impl PageServerNode {
            .flatten()
            .ok_or_else(|| anyhow!("missing branches"))?;
-        Ok(serde_json::from_str(branches_json)?)
+        let res: Vec<BranchInfo> = serde_json::from_str(branches_json)?;
        Ok(res)
    }
-    pub fn branch_create(
+    pub fn branch_create(&self, name: &str, startpoint: &str) -> Result<BranchInfo> {
        &self,
        branch_name: &str,
        startpoint: &str,
        tenantid: &ZTenantId,
    ) -> Result<BranchInfo> {
        let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query(
+        let query_result =
-            format!("branch_create {} {} {}", tenantid, branch_name, startpoint).as_str(),
+            client.simple_query(format!("branch_create {} {}", name, startpoint).as_str())?;
        )?;
        let branch_json = query_result
            .first()
@@ -238,12 +211,8 @@ impl PageServerNode {
    }
    // TODO: make this a separate request type and avoid loading all the branches
-    pub fn branch_get_by_name(
+    pub fn branch_get_by_name(&self, name: &str) -> Result<BranchInfo> {
-        &self,
+        let branch_infos = self.branches_list()?;
        tenantid: &ZTenantId,
        branch_name: &str,
    ) -> Result<BranchInfo> {
        let branch_infos = self.branches_list(tenantid)?;
        let branche_by_name: Result<HashMap<String, BranchInfo>> = branch_infos
            .into_iter()
            .map(|branch_info| Ok((branch_info.name.clone(), branch_info)))
@@ -251,11 +220,27 @@ impl PageServerNode {
        let branche_by_name = branche_by_name?;
        let branch = branche_by_name
-            .get(branch_name)
+            .get(name)
-            .ok_or_else(|| anyhow!("Branch {} not found", branch_name))?;
+            .ok_or_else(|| anyhow!("Branch {} not found", name))?;
        Ok(branch.clone())
    }
    pub fn system_id_get(&self) -> Result<u64> {
        let mut client = self.page_server_psql_client()?;
        let query_result = client
            .simple_query("identify_system")?
            .first()
            .map(|msg| match msg {
                postgres::SimpleQueryMessage::Row(row) => row.get(0),
                _ => None,
            })
            .flatten()
            .ok_or_else(|| anyhow!("failed to get system_id"))?
            .parse::<u64>()?;
        Ok(query_result)
    }
 }
 impl Drop for PageServerNode {
@@ -1,11 +0,0 @@
 #!/bin/sh
 if [ "$1" = 'pageserver' ]; then
    if [ ! -d "/data/tenants" ]; then
        echo "Initializing pageserver data directory"
        pageserver --init -D /data --postgres-distrib /usr/local
    fi
    echo "Staring pageserver at 0.0.0.0:6400"
    pageserver -l 0.0.0.0:6400 -D /data
 else
    "$@"
 fi
@@ -1,11 +0,0 @@
 # Zenith documentation
 ## Table of contents
 - [authentication.md](authentication.md) — pageserver JWT authentication.
 - [docker.md](docker.md) — Docker images and building pipeline.
 - [multitenancy.md](multitenancy.md) — how multitenancy is organized in the pageserver and Zenith CLI.
 - [pageserver/README](/pageserver/README) — pageserver overview.
 - [postgres_ffi/README](/postgres_ffi/README) — Postgres FFI overview.
 - [test_runner/README.md](/test_runner/README.md) — tests infrastructure overview.
 - [walkeeper/README](/walkeeper/README.md) — WAL service overview.
@@ -1,30 +0,0 @@
 ## Authentication
 ### Overview
 Current state of authentication includes usage of JWT tokens in communication between compute and pageserver and between CLI and pageserver. JWT token is signed using RSA keys. CLI generates a key pair during call to `zenith init`. Using following openssl commands:
 ```bash
 openssl genrsa -out private_key.pem 2048
 openssl rsa -in private_key.pem -pubout -outform PEM -out public_key.pem
 ```
 CLI also generates signed token and saves it in the config for later access to pageserver. Now authentication is optional. Pageserver has two variables in config: `auth_validation_public_key_path` and `auth_type`, so when auth type present and set to `ZenithJWT` pageserver will require authentication for connections. Actual JWT is passed in password field of connection string. There is a caveat for psql, it silently truncates passwords to 100 symbols, so to correctly pass JWT via psql you have to either use PGPASSWORD environment variable, or store password in psql config file.
 Currently there is no authentication between compute and safekeepers, because this communication layer is under heavy refactoring. After this refactoring support for authentication will be added there too. Now safekeeper supports "hardcoded" token passed via environment variable to be able to use callmemaybe command in pageserver.
 Compute uses token passed via environment variable to communicate to pageserver and in the future to the safekeeper too.
 JWT authentication now supports two scopes: tenant and pageserverapi. Tenant scope is intended for use in tenant related api calls, e.g. create_branch. Compute launched for particular tenant also uses this scope. Scope pageserver api is intended to be used by console to manage pageserver. For now we have only one management operation - create tenant.
 Examples for token generation in python:
 ```python
 # generate pageserverapi token
 management_token = jwt.encode({"scope": "pageserverapi"}, auth_keys.priv, algorithm="RS256")
 # generate tenant token
 tenant_token = jwt.encode({"scope": "tenant", "tenant_id": ps.initial_tenant}, auth_keys.priv, algorithm="RS256")
 ```
 Utility functions to work with jwts in rust are located in zenith_utils/src/auth.rs
@@ -1,38 +0,0 @@
 # Docker images of Zenith
 ## Images
 Currently we build two main images:
 - [zenithdb/zenith](https://hub.docker.com/repository/docker/zenithdb/zenith) — image with pre-built `pageserver`, `wal_acceptor` and `proxy` binaries and all the required runtime dependencies. Built from [/Dockerfile](/Dockerfile).
 - [zenithdb/compute-node](https://hub.docker.com/repository/docker/zenithdb/compute-node) — compute node image with pre-built Postgres binaries from [zenithdb/postgres](https://github.com/zenithdb/postgres).
 And two intermediate images used either to reduce build time or to deliver some additional binary tools from other repos:
 - [zenithdb/build](https://hub.docker.com/repository/docker/zenithdb/build) — image with all the dependencies required to build Zenith and compute node images. This image is based on `rust:slim-buster`, so it also has a proper `rust` environment. Built from [/Dockerfile.build](/Dockerfile.build).
 - [zenithdb/compute-tools](https://hub.docker.com/repository/docker/zenithdb/compute-tools) — compute node configuration management tools.
 ## Building pipeline
 1. Image `zenithdb/compute-tools` is re-built automatically.
 2. Image `zenithdb/build` is built manually. If you want to introduce any new compile time dependencies to Zenith or compute node you have to update this image as well, build it and push to Docker Hub.
 Build:
 ```sh
 docker build -t zenithdb/build:buster -f Dockerfile.build .
 ```
 Login:
 ```sh
 docker login
 ```
 Push to Docker Hub:
 ```sh
 docker push zenithdb/build:buster
 ```
 3. Image `zenithdb/compute-node` is built independently in the [zenithdb/postgres](https://github.com/zenithdb/postgres) repo.
 4. Image `zenithdb/zenith` is built in this repo after a successful `release` tests run and pushed to Docker Hub automatically.
@@ -1,59 +0,0 @@
 ## Multitenancy
 ### Overview
 Zenith supports multitenancy. One pageserver can serve multiple tenants at once. Tenants can be managed via zenith CLI. During page server setup tenant can be created using ```zenith init --create-tenant``` Also tenants can be added into the system on the fly without pageserver restart. This can be done using the following cli command: ```zenith tenant create``` Tenants use random identifiers which can be represented as a 32 symbols hexadecimal string. So zenith tenant create accepts desired tenant id as an optional argument. The concept of timelines/branches is working independently per tenant.
 ### Tenants in other commands
 By default during `zenith init` new tenant is created on the pageserver. Newly created tenant's id is saved to cli config, so other commands can use it automatically if no direct arugment `--tenantid=<tenantid>` is provided. So generally tenantid more frequently appears in internal pageserver interface. Its commands take tenantid argument to distinguish to which tenant operation should be applied. CLI support creation of new tenants.
 Examples for cli:
 ```sh
 zenith tenant list
 zenith tenant create // generates new id
 zenith tenant create ee6016ec31116c1b7c33dfdfca38892f
 zenith pg create main // default tenant from zenith init
 zenith pg create main --tenantid=ee6016ec31116c1b7c33dfdfca38892f
 zenith branch --tenantid=ee6016ec31116c1b7c33dfdfca38892f
 ```
 ### Data layout
 On the page server tenants introduce one level of indirection, so data directory structured the following way:
 ```
 <pageserver working directory>
 ├── pageserver.log
 ├── pageserver.pid
 ├── pageserver.toml
 └── tenants
   ├── 537cffa58a4fa557e49e19951b5a9d6b
   ├── de182bc61fb11a5a6b390a8aed3a804a
   └── ee6016ec31116c1b7c33dfdfca38891f
 ```
 Wal redo activity, timelines, snapshots are managed for each tenant independently.
 For local environment used for example in tests there also new level of indirection for tenants. It touches `pgdatadirs` directory. Now it contains `tenants` subdirectory so the structure looks the following way:
 ```
 pgdatadirs
 └── tenants
   ├── de182bc61fb11a5a6b390a8aed3a804a
   │  └── main
   └── ee6016ec31116c1b7c33dfdfca38892f
      └── main
 ```
 ### Changes to postgres
 Tenant id is passed to postgres via GUC the same way as the timeline. Tenant id is added to commands issued to pageserver, namely: pagestream, callmemaybe. Tenant id is also exists in ServerInfo structure, this is needed to pass the value to wal receiver to be able to forward it to the pageserver.
 ### Safety
 For now particular tenant can only appear on a particular pageserver. Set of WAL acceptors are also pinned to particular (tenantid, timeline) pair so there can only be one writer for particular (tenantid, timeline).
@@ -0,0 +1 @@
 tmp_check/
@@ -0,0 +1,18 @@
 [package]
 name = "integration_tests"
 version = "0.1.0"
 authors = ["Stas Kelvich <stas@zenith.tech>"]
 edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [dependencies]
 lazy_static = "1.4.0"
 rand = "0.8.3"
 anyhow = "1.0"
 nix = "0.20"
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 pageserver = { path = "../pageserver" }
 walkeeper = { path = "../walkeeper" }
 control_plane = { path = "../control_plane" }
@@ -0,0 +1,416 @@
 use std::collections::BTreeMap;
 use std::convert::TryInto;
 use std::fs::{self, File, OpenOptions};
 use std::io::Read;
 use std::net::SocketAddr;
 use std::path::{Path, PathBuf};
 use std::process::{Command, ExitStatus};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use anyhow::{bail, Result};
 use nix::sys::signal::{kill, Signal};
 use nix::unistd::Pid;
 use postgres;
 use control_plane::compute::PostgresNode;
 use control_plane::read_pidfile;
 use control_plane::{local_env::LocalEnv, storage::PageServerNode};
 // Find the directory where the binaries were put (i.e. target/debug/)
 fn cargo_bin_dir() -> PathBuf {
    let mut pathbuf = std::env::current_exe().unwrap();
    pathbuf.pop();
    if pathbuf.ends_with("deps") {
        pathbuf.pop();
    }
    pathbuf
 }
 // local compute env for tests
 pub fn create_test_env(testname: &str) -> LocalEnv {
    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../tmp_check/")
        .join(testname);
    let base_path_str = base_path.to_str().unwrap();
    // Remove remnants of old test repo
    let _ = fs::remove_dir_all(&base_path);
    fs::create_dir_all(&base_path)
        .expect(format!("could not create directory for {}", base_path_str).as_str());
    let pgdatadirs_path = base_path.join("pgdatadirs");
    fs::create_dir(&pgdatadirs_path)
        .expect(format!("could not create directory {:?}", pgdatadirs_path).as_str());
    LocalEnv {
        pageserver_connstring: "postgresql://127.0.0.1:64000".to_string(),
        pg_distrib_dir: Path::new(env!("CARGO_MANIFEST_DIR")).join("../tmp_install"),
        zenith_distrib_dir: Some(cargo_bin_dir()),
        base_data_dir: base_path,
        remotes: BTreeMap::default(),
    }
 }
 //
 // Collection of several example deployments useful for tests.
 //
 // I'm intendedly modelling storage and compute control planes as a separate entities
 // as it is closer to the actual setup.
 //
 pub struct TestStorageControlPlane {
    pub wal_acceptors: Vec<WalAcceptorNode>,
    pub pageserver: Arc<PageServerNode>,
    pub test_done: AtomicBool,
 }
 impl TestStorageControlPlane {
    // postgres <-> page_server
    //
    // Initialize a new repository and configure a page server to run in it
    //
    pub fn one_page_server(local_env: &LocalEnv) -> TestStorageControlPlane {
        let pserver = Arc::new(PageServerNode {
            env: local_env.clone(),
            kill_on_exit: true,
            listen_address: None,
        });
        pserver.init().unwrap();
        pserver.start().unwrap();
        TestStorageControlPlane {
            wal_acceptors: Vec::new(),
            pageserver: pserver,
            test_done: AtomicBool::new(false),
        }
    }
    // postgres <-> {wal_acceptor1, wal_acceptor2, ...}
    pub fn fault_tolerant(local_env: &LocalEnv, redundancy: usize) -> TestStorageControlPlane {
        let mut cplane = TestStorageControlPlane {
            wal_acceptors: Vec::new(),
            pageserver: Arc::new(PageServerNode {
                env: local_env.clone(),
                kill_on_exit: true,
                listen_address: None,
            }),
            test_done: AtomicBool::new(false),
            // repopath,
        };
        cplane.pageserver.init().unwrap();
        cplane.pageserver.start().unwrap();
        let systemid = cplane.pageserver.system_id_get().unwrap();
        const WAL_ACCEPTOR_PORT: usize = 54321;
        let datadir_base = local_env.base_data_dir.join("safekeepers");
        fs::create_dir_all(&datadir_base).unwrap();
        for i in 0..redundancy {
            let wal_acceptor = WalAcceptorNode {
                listen: format!("127.0.0.1:{}", WAL_ACCEPTOR_PORT + i)
                    .parse()
                    .unwrap(),
                data_dir: datadir_base.join(format!("wal_acceptor_{}", i)),
                systemid,
                env: local_env.clone(),
                pass_to_pageserver: true,
            };
            wal_acceptor.init();
            wal_acceptor.start();
            cplane.wal_acceptors.push(wal_acceptor);
        }
        cplane
    }
    pub fn stop(&self) {
        for wa in self.wal_acceptors.iter() {
            let _ = wa.stop();
        }
        self.test_done.store(true, Ordering::Relaxed);
    }
    pub fn get_wal_acceptor_conn_info(&self) -> String {
        self.wal_acceptors
            .iter()
            .map(|wa| wa.listen.to_string())
            .collect::<Vec<String>>()
            .join(",")
    }
    pub fn is_running(&self) -> bool {
        self.test_done.load(Ordering::Relaxed)
    }
 }
 impl Drop for TestStorageControlPlane {
    fn drop(&mut self) {
        self.stop();
    }
 }
 ///////////////////////////////////////////////////////////////////////////////
 //
 // PostgresNodeExt
 //
 ///////////////////////////////////////////////////////////////////////////////
 ///
 /// Testing utilities for PostgresNode type
 ///
 pub trait PostgresNodeExt {
    fn pg_regress(&self) -> ExitStatus;
    fn pg_bench(&self, clients: u32, seconds: u32) -> ExitStatus;
    fn start_proxy(&self, wal_acceptors: &str) -> WalProposerNode;
    fn open_psql(&self, db: &str) -> postgres::Client;
    fn dump_log_file(&self);
    fn safe_psql(&self, db: &str, sql: &str) -> Vec<postgres::Row>;
 }
 impl PostgresNodeExt for PostgresNode {
    fn pg_regress(&self) -> ExitStatus {
        self.safe_psql("postgres", "CREATE DATABASE regression");
        let regress_run_path = self.env.base_data_dir.join("regress");
        fs::create_dir_all(&regress_run_path).unwrap();
        fs::create_dir_all(regress_run_path.join("testtablespace")).unwrap();
        std::env::set_current_dir(regress_run_path).unwrap();
        let regress_build_path =
            Path::new(env!("CARGO_MANIFEST_DIR")).join("../tmp_install/build/src/test/regress");
        let regress_src_path =
            Path::new(env!("CARGO_MANIFEST_DIR")).join("../vendor/postgres/src/test/regress");
        let regress_check = Command::new(regress_build_path.join("pg_regress"))
            .args(&[
                "--bindir=''",
                "--use-existing",
                format!("--bindir={}", self.env.pg_bin_dir().to_str().unwrap()).as_str(),
                format!("--dlpath={}", regress_build_path.to_str().unwrap()).as_str(),
                format!(
                    "--schedule={}",
                    regress_src_path.join("parallel_schedule").to_str().unwrap()
                )
                .as_str(),
                format!("--inputdir={}", regress_src_path.to_str().unwrap()).as_str(),
            ])
            .env_clear()
            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
            .env("PGPORT", self.address.port().to_string())
            .env("PGUSER", self.whoami())
            .env("PGHOST", self.address.ip().to_string())
            .status()
            .expect("pg_regress failed");
        if !regress_check.success() {
            if let Ok(mut file) = File::open("regression.diffs") {
                let mut buffer = String::new();
                file.read_to_string(&mut buffer).unwrap();
                println!("--------------- regression.diffs:\n{}", buffer);
            }
            self.dump_log_file();
        }
        regress_check
    }
    fn pg_bench(&self, clients: u32, seconds: u32) -> ExitStatus {
        let port = self.address.port().to_string();
        let clients = clients.to_string();
        let seconds = seconds.to_string();
        let _pg_bench_init = Command::new(self.env.pg_bin_dir().join("pgbench"))
            .args(&["-i", "-p", port.as_str(), "postgres"])
            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
            .status()
            .expect("pgbench -i");
        let pg_bench_run = Command::new(self.env.pg_bin_dir().join("pgbench"))
            .args(&[
                "-p",
                port.as_str(),
                "-T",
                seconds.as_str(),
                "-P",
                "1",
                "-c",
                clients.as_str(),
                "-M",
                "prepared",
                "postgres",
            ])
            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
            .status()
            .expect("pgbench run");
        pg_bench_run
    }
    fn start_proxy(&self, wal_acceptors: &str) -> WalProposerNode {
        let proxy_path = self.env.pg_bin_dir().join("safekeeper_proxy");
        match Command::new(proxy_path.as_path())
            .args(&["--ztimelineid", &self.timelineid.to_string()])
            .args(&["-s", wal_acceptors])
            .args(&["-h", &self.address.ip().to_string()])
            .args(&["-p", &self.address.port().to_string()])
            .arg("-v")
            .stderr(
                OpenOptions::new()
                    .create(true)
                    .append(true)
                    .open(self.pgdata().join("safekeeper_proxy.log"))
                    .unwrap(),
            )
            .spawn()
        {
            Ok(child) => WalProposerNode { pid: child.id() },
            Err(e) => panic!("Failed to launch {:?}: {}", proxy_path, e),
        }
    }
    fn dump_log_file(&self) {
        if let Ok(mut file) = File::open(self.env.pageserver_data_dir().join("pageserver.log")) {
            let mut buffer = String::new();
            file.read_to_string(&mut buffer).unwrap();
            println!("--------------- pageserver.log:\n{}", buffer);
        }
    }
    fn safe_psql(&self, db: &str, sql: &str) -> Vec<postgres::Row> {
        let connstring = format!(
            "host={} port={} dbname={} user={}",
            self.address.ip(),
            self.address.port(),
            db,
            self.whoami()
        );
        let mut client = postgres::Client::connect(connstring.as_str(), postgres::NoTls).unwrap();
        println!("Running {}", sql);
        let result = client.query(sql, &[]);
        if result.is_err() {
            self.dump_log_file();
        }
        result.unwrap()
    }
    fn open_psql(&self, db: &str) -> postgres::Client {
        let connstring = format!(
            "host={} port={} dbname={} user={}",
            self.address.ip(),
            self.address.port(),
            db,
            self.whoami()
        );
        postgres::Client::connect(connstring.as_str(), postgres::NoTls).unwrap()
    }
 }
 ///////////////////////////////////////////////////////////////////////////////
 //
 // WalAcceptorNode
 //
 ///////////////////////////////////////////////////////////////////////////////
 //
 // Control routines for WalAcceptor.
 //
 // Now used only in test setups.
 //
 pub struct WalAcceptorNode {
    listen: SocketAddr,
    data_dir: PathBuf,
    systemid: u64,
    env: LocalEnv,
    pass_to_pageserver: bool,
 }
 impl WalAcceptorNode {
    pub fn init(&self) {
        if self.data_dir.exists() {
            fs::remove_dir_all(self.data_dir.clone()).unwrap();
        }
        fs::create_dir_all(self.data_dir.clone()).unwrap();
    }
    pub fn start(&self) {
        println!(
            "Starting wal_acceptor in {} listening '{}'",
            self.data_dir.to_str().unwrap(),
            self.listen
        );
        let ps_arg = if self.pass_to_pageserver {
            // Tell page server it can receive WAL from this WAL safekeeper
            ["--pageserver", "127.0.0.1:64000"].to_vec()
        } else {
            [].to_vec()
        };
        let status = Command::new(
            self.env
                .zenith_distrib_dir
                .as_ref()
                .unwrap()
                .join("wal_acceptor"),
        )
        .args(&["-D", self.data_dir.to_str().unwrap()])
        .args(&["-l", self.listen.to_string().as_str()])
        .args(&["--systemid", self.systemid.to_string().as_str()])
        .args(&ps_arg)
        .arg("-d")
        .arg("-n")
        .status()
        .expect("failed to start wal_acceptor");
        if !status.success() {
            panic!("wal_acceptor start failed");
        }
    }
    pub fn stop(&self) -> Result<()> {
        println!("Stopping wal acceptor on {}", self.listen);
        let pidfile = self.data_dir.join("wal_acceptor.pid");
        let pid = read_pidfile(&pidfile)?;
        let pid = Pid::from_raw(pid);
        if kill(pid, Signal::SIGTERM).is_err() {
            bail!("Failed to kill wal_acceptor with pid {}", pid);
        }
        Ok(())
    }
 }
 impl Drop for WalAcceptorNode {
    fn drop(&mut self) {
        // Ignore errors.
        let _ = self.stop();
    }
 }
 ///////////////////////////////////////////////////////////////////////////////
 //
 // WalProposerNode
 //
 ///////////////////////////////////////////////////////////////////////////////
 pub struct WalProposerNode {
    pub pid: u32,
 }
 impl WalProposerNode {
    pub fn stop(&self) {
        // std::process::Child::id() returns u32, we need i32.
        let pid: i32 = self.pid.try_into().unwrap();
        let pid = Pid::from_raw(pid);
        kill(pid, Signal::SIGTERM).expect("failed to execute kill");
    }
 }
 impl Drop for WalProposerNode {
    fn drop(&mut self) {
        self.stop();
    }
 }
@@ -0,0 +1,332 @@
 use rand::Rng;
 use std::sync::Arc;
 use std::time::SystemTime;
 use std::{thread, time};
 use control_plane::compute::{ComputeControlPlane, PostgresNode};
 use integration_tests;
 use integration_tests::PostgresNodeExt;
 use integration_tests::TestStorageControlPlane;
 const DOWNTIME: u64 = 2;
 fn start_node_with_wal_proposer(
    timeline: &str,
    compute_cplane: &mut ComputeControlPlane,
    wal_acceptors: &String,
 ) -> Arc<PostgresNode> {
    let node = compute_cplane.new_test_master_node(timeline);
    let _node = node.append_conf(
        "postgresql.conf",
        &format!("wal_acceptors='{}'\n", wal_acceptors),
    );
    node.start().unwrap();
    node
 }
 #[test]
 fn test_embedded_wal_proposer() {
    let local_env = integration_tests::create_test_env("test_embedded_wal_proposer");
    const REDUNDANCY: usize = 3;
    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
    // start postgres
    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
    // check basic work with table
    node.safe_psql(
        "postgres",
        "CREATE TABLE t(key int primary key, value text)",
    );
    node.safe_psql(
        "postgres",
        "INSERT INTO t SELECT generate_series(1,100000), 'payload'",
    );
    let count: i64 = node
        .safe_psql("postgres", "SELECT sum(key) FROM t")
        .first()
        .unwrap()
        .get(0);
    println!("sum = {}", count);
    assert_eq!(count, 5000050000);
    // check wal files equality
 }
 #[test]
 fn test_acceptors_normal_work() {
    let local_env = integration_tests::create_test_env("test_acceptors_normal_work");
    const REDUNDANCY: usize = 3;
    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
    // start postgres
    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
    // check basic work with table
    node.safe_psql(
        "postgres",
        "CREATE TABLE t(key int primary key, value text)",
    );
    node.safe_psql(
        "postgres",
        "INSERT INTO t SELECT generate_series(1,100000), 'payload'",
    );
    let count: i64 = node
        .safe_psql("postgres", "SELECT sum(key) FROM t")
        .first()
        .unwrap()
        .get(0);
    println!("sum = {}", count);
    assert_eq!(count, 5000050000);
    // check wal files equality
 }
 // Run page server and multiple safekeepers, and multiple compute nodes running
 // against different timelines.
 #[test]
 fn test_many_timelines() {
    // Initialize a new repository, and set up WAL safekeepers and page server.
    const REDUNDANCY: usize = 3;
    const N_TIMELINES: usize = 5;
    let local_env = integration_tests::create_test_env("test_many_timelines");
    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
    // Create branches
    let mut timelines: Vec<String> = Vec::new();
    timelines.push("main".to_string());
    for i in 1..N_TIMELINES {
        let branchname = format!("experimental{}", i);
        storage_cplane
            .pageserver
            .branch_create(&branchname, "main")
            .unwrap();
        timelines.push(branchname);
    }
    // start postgres on each timeline
    let mut nodes = Vec::new();
    for tli_name in timelines {
        let node = start_node_with_wal_proposer(&tli_name, &mut compute_cplane, &wal_acceptors);
        nodes.push(node.clone());
    }
    // create schema
    for node in &nodes {
        node.safe_psql(
            "postgres",
            "CREATE TABLE t(key int primary key, value text)",
        );
    }
    // Populate data
    for node in &nodes {
        node.safe_psql(
            "postgres",
            "INSERT INTO t SELECT generate_series(1,100000), 'payload'",
        );
    }
    // Check data
    for node in &nodes {
        let count: i64 = node
            .safe_psql("postgres", "SELECT sum(key) FROM t")
            .first()
            .unwrap()
            .get(0);
        println!("sum = {}", count);
        assert_eq!(count, 5000050000);
    }
 }
 // Majority is always alive
 #[test]
 fn test_acceptors_restarts() {
    let local_env = integration_tests::create_test_env("test_acceptors_restarts");
    // Start pageserver that reads WAL directly from that postgres
    const REDUNDANCY: usize = 3;
    const FAULT_PROBABILITY: f32 = 0.01;
    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
    let mut rng = rand::thread_rng();
    // start postgres
    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
    let mut failed_node: Option<usize> = None;
    // check basic work with table
    node.safe_psql(
        "postgres",
        "CREATE TABLE t(key int primary key, value text)",
    );
    let mut psql = node.open_psql("postgres");
    for i in 1..=1000 {
        psql.execute("INSERT INTO t values ($1, 'payload')", &[&i])
            .unwrap();
        let prob: f32 = rng.gen();
        if prob <= FAULT_PROBABILITY {
            if let Some(node) = failed_node {
                storage_cplane.wal_acceptors[node].start();
                failed_node = None;
            } else {
                let node: usize = rng.gen_range(0..REDUNDANCY);
                failed_node = Some(node);
                storage_cplane.wal_acceptors[node].stop().unwrap();
            }
        }
    }
    let count: i64 = node
        .safe_psql("postgres", "SELECT sum(key) FROM t")
        .first()
        .unwrap()
        .get(0);
    println!("sum = {}", count);
    assert_eq!(count, 500500);
 }
 fn start_acceptor(cplane: &Arc<TestStorageControlPlane>, no: usize) {
    let cp = cplane.clone();
    thread::spawn(move || {
        thread::sleep(time::Duration::from_secs(DOWNTIME));
        cp.wal_acceptors[no].start();
    });
 }
 // Stop majority of acceptors while compute is under the load. Boot
 // them again and check that nothing was losed. Repeat.
 // N_CRASHES env var
 #[test]
 fn test_acceptors_unavailability() {
    let local_env = integration_tests::create_test_env("test_acceptors_unavailability");
    // Start pageserver that reads WAL directly from that postgres
    const REDUNDANCY: usize = 2;
    let storage_cplane = TestStorageControlPlane::fault_tolerant(&local_env, REDUNDANCY);
    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
    // start postgres
    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
    // check basic work with table
    node.safe_psql(
        "postgres",
        "CREATE TABLE t(key int primary key, value text)",
    );
    let mut psql = node.open_psql("postgres");
    psql.execute("INSERT INTO t values (1, 'payload')", &[])
        .unwrap();
    // Shut down all wal acceptors
    storage_cplane.wal_acceptors[0].stop().unwrap();
    let cp = Arc::new(storage_cplane);
    start_acceptor(&cp, 0);
    let now = SystemTime::now();
    psql.execute("INSERT INTO t values (2, 'payload')", &[])
        .unwrap();
    // Here we check that the query above was hanging
    // while wal_acceptor was unavailiable
    assert!(now.elapsed().unwrap().as_secs() >= DOWNTIME);
    psql.execute("INSERT INTO t values (3, 'payload')", &[])
        .unwrap();
    cp.wal_acceptors[1].stop().unwrap();
    start_acceptor(&cp, 1);
    psql.execute("INSERT INTO t values (4, 'payload')", &[])
        .unwrap();
    // Here we check that the query above was hanging
    // while wal_acceptor was unavailiable
    assert!(now.elapsed().unwrap().as_secs() >= 2 * DOWNTIME);
    psql.execute("INSERT INTO t values (5, 'payload')", &[])
        .unwrap();
    let count: i64 = node
        .safe_psql("postgres", "SELECT sum(key) FROM t")
        .first()
        .unwrap()
        .get(0);
    println!("sum = {}", count);
    // Ensure that all inserts succeeded.
    // Including ones that were waiting for wal acceptor restart.
    assert_eq!(count, 15);
 }
 fn simulate_failures(cplane: Arc<TestStorageControlPlane>) {
    let mut rng = rand::thread_rng();
    let n_acceptors = cplane.wal_acceptors.len();
    let failure_period = time::Duration::from_secs(1);
    while cplane.is_running() {
        thread::sleep(failure_period);
        let mask: u32 = rng.gen_range(0..(1 << n_acceptors));
        for i in 0..n_acceptors {
            if (mask & (1 << i)) != 0 {
                cplane.wal_acceptors[i].stop().unwrap();
            }
        }
        thread::sleep(failure_period);
        for i in 0..n_acceptors {
            if (mask & (1 << i)) != 0 {
                cplane.wal_acceptors[i].start();
            }
        }
    }
 }
 // Race condition test
 #[test]
 fn test_race_conditions() {
    let local_env = integration_tests::create_test_env("test_race_conditions");
    // Start pageserver that reads WAL directly from that postgres
    const REDUNDANCY: usize = 3;
    let storage_cplane = Arc::new(TestStorageControlPlane::fault_tolerant(
        &local_env, REDUNDANCY,
    ));
    let mut compute_cplane = ComputeControlPlane::local(&local_env, &storage_cplane.pageserver);
    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
    // start postgres
    let node = start_node_with_wal_proposer("main", &mut compute_cplane, &wal_acceptors);
    // check basic work with table
    node.safe_psql(
        "postgres",
        "CREATE TABLE t(key int primary key, value text)",
    );
    let cp = storage_cplane.clone();
    let failures_thread = thread::spawn(move || {
        simulate_failures(cp);
    });
    let mut psql = node.open_psql("postgres");
    for i in 1..=1000 {
        psql.execute("INSERT INTO t values ($1, 'payload')", &[&i])
            .unwrap();
    }
    let count: i64 = node
        .safe_psql("postgres", "SELECT sum(key) FROM t")
        .first()
        .unwrap()
        .get(0);
    println!("sum = {}", count);
    assert_eq!(count, 500500);
    storage_cplane.stop();
    failures_thread.join().unwrap();
 }
@@ -1,25 +0,0 @@
 version: "3"
 services:
  prometheus:
    container_name: prometheus
    image: prom/prometheus:latest
    volumes:
      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
    # ports:
    #   - "9090:9090"
    # TODO: find a proper portable solution
    network_mode: "host"
  grafana:
    image: grafana/grafana:latest
    volumes:
      - ./grafana.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
      - GF_AUTH_DISABLE_LOGIN_FORM=true
    # ports:
    #   - "3000:3000"
    # TODO: find a proper portable solution
    network_mode: "host"
@@ -1,12 +0,0 @@
 apiVersion: 1
 datasources:
 - name: Prometheus
  type: prometheus
  access: proxy
  orgId: 1
  url: http://localhost:9090
  basicAuth: false
  isDefault: false
  version: 1
  editable: false
@@ -1,5 +0,0 @@
 scrape_configs:
  - job_name: 'default'
    scrape_interval: 10s
    static_configs:
      - targets: ['localhost:9898']
@@ -7,11 +7,10 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [dependencies]
 bookfile = "^0.3"
 chrono = "0.4.19"
 rand = "0.8.3"
 regex = "1.4.5"
-bytes = { version = "1.0.1", features = ['serde'] }
+bytes = "1.0.1"
 byteorder = "1.4.3"
 futures = "0.3.13"
 lazy_static = "1.4.0"
@@ -22,29 +21,27 @@ slog-term = "2.8.0"
 slog = "2.7.0"
 log = "0.4.14"
 clap = "2.33.0"
 termion = "1.5.6"
 tui = "0.14.0"
 daemonize = "0.4.1"
 rust-s3 = { version = "0.27.0-rc4", features = ["no-verify-ssl"] }
-tokio = { version = "1.5.0", features = ["full"] }
+tokio = { version = "1.3.0", features = ["full"] }
-tokio-stream = { version = "0.1.5" }
+tokio-stream = { version = "0.1.4" }
 postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-# by default rust-rocksdb tries to build a lot of compression algos. Use lz4 only for now as it is simplest dependency.
+rocksdb = "0.16.0"
 rocksdb = { version = "0.16.0", features = ["lz4"], default-features = false }
 anyhow = "1.0"
 crc32c = "0.6.0"
 walkdir = "2"
 thiserror = "1.0"
 hex = "0.4.3"
 tar = "0.4.33"
-humantime = "2.1.0"
+parse_duration = "2.1.1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 fs_extra = "1.2.0"
 toml = "0.5"
 scopeguard = "1.1.0"
 postgres_ffi = { path = "../postgres_ffi" }
 zenith_metrics = { path = "../zenith_metrics" }
 zenith_utils = { path = "../zenith_utils" }
 workspace_hack = { path = "../workspace_hack" }
@@ -1,4 +1,82 @@
-## Page server architecture
+Page Server
 ===========
 How to test
 -----------
 1. Compile and install Postgres from this repository (there are
   modifications, so vanilla Postgres won't do)
    ./configure --prefix=/home/heikki/zenith-install
 2. Compile the page server
    cd pageserver
    cargo build
 3. Create another "dummy" cluster that will be used by the page server when it applies
   the WAL records. (shouldn't really need this, getting rid of it is a TODO):
    /home/heikki/zenith-install/bin/initdb -D /data/zenith-dummy
 4. Initialize and start a new postgres cluster
    /home/heikki/zenith-install/bin/initdb -D /data/zenith-test-db --username=postgres
    /home/heikki/zenith-install/bin/postgres -D /data/zenith-test-db
 5. In another terminal, start the page server.
    PGDATA=/data/zenith-dummy PATH=/home/heikki/zenith-install/bin:$PATH ./target/debug/pageserver
   It should connect to the postgres instance using streaming replication, and print something
   like this:
    $ PGDATA=/data/zenith-dummy PATH=/home/heikki/zenith-install/bin:$PATH ./target/debug/pageserver
    Starting WAL receiver
    connecting...
    Starting page server on 127.0.0.1:5430
    connected!
    page cache is empty
 6. You can now open another terminal and issue DDL commands. Generated WAL records will
   be streamed to the page servers, and attached to blocks that they apply to in its
   page cache
    $ psql postgres -U postgres
    psql (14devel)
    Type "help" for help.
    postgres=# create table mydata (i int4);
    CREATE TABLE
    postgres=# insert into mydata select g from generate_series(1,100) g;
    INSERT 0 100
    postgres=# 
 7. The GetPage@LSN interface to the compute nodes isn't working yet, but to simulate
   that, the page server generates a test GetPage@LSN call every 5 seconds on a random
   block that's in the page cache. In a few seconds, you should see output from that:
    testing GetPage@LSN for block 0
    WAL record at LSN 23584576 initializes the page
    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167DF40
    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167DF80
    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167DFC0
    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E018
    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E058
    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E098
    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E0D8
    2021-03-19 11:03:13.792 EET [11439] LOG:  applied WAL record at 0/167E118
    2021-03-19 11:03:13.792 EET [11439] LOG:  applied WAL record at 0/167E158
    2021-03-19 11:03:13.792 EET [11439] LOG:  applied WAL record at 0/167E198
    applied 10 WAL records to produce page image at LSN 18446744073709547246
 Architecture
 ============
 The Page Server is responsible for all operations on a number of
 "chunks" of relation data. A chunk corresponds to a PostgreSQL
@@ -6,10 +84,8 @@ relation segment (i.e. one max. 1 GB file in the data directory), but
 it holds all the different versions of every page in the segment that
 are still needed by the system.
-Currently we do not specifically organize data in chunks.
+Determining which chunk each Page Server holds is handled elsewhere. (TODO:
-All page images and corresponding WAL records are stored as entries in a key-value storage,
+currently, there is only one Page Server which holds all chunks)
 where StorageKey is a zenith_timeline_id + BufferTag + LSN.
 The Page Server has a few different duties:
@@ -78,33 +154,11 @@ and stores them to the page cache.
 Page Cache
 ----------
-The Page Cache is a switchboard to access different Repositories.
+The Page Cache is a data structure, to hold all the different page versions.
 It is accessed by all the other threads, to perform their duties.
-#### Repository
+Currently, the page cache is implemented fully in-memory. TODO: Store it
-Repository corresponds to one .zenith directory.
+on disk. Define a file format.
 Repository is needed to manage Timelines.
 #### Timeline
 Timeline is a page cache workhorse that accepts page changes
 and serves get_page_at_lsn() and get_rel_size() requests.
 Note: this has nothing to do with PostgreSQL WAL timeline.
 #### Branch
 We can create branch at certain LSN.
 Each Branch lives in a corresponding timeline and has an ancestor.
 To get full snapshot of data at certain moment we need to traverse timeline and its ancestors.
 #### ObjectRepository
 ObjectRepository implements Repository and has associated ObjectStore and WAL redo service.
 #### ObjectStore
 ObjectStore is an interface for key-value store for page images and wal records.
 Currently it has one implementation - RocksDB.
 #### WAL redo service
 WAL redo service - service that runs PostgreSQL in a special wal_redo mode
 to apply given WAL records over an old page image and return new page image.
 TODO: Garbage Collection / Compaction
@@ -123,7 +177,3 @@ The backup service is responsible for periodically pushing the chunks to S3.
 TODO: How/when do restore from S3? Whenever we get a GetPage@LSN request for
 a chunk we don't currently have? Or when an external Control Plane tells us?
 TODO: Sharding
 --------------------
 We should be able to run multiple Page Servers that handle sharded data.
@@ -0,0 +1,62 @@
 #!/bin/sh
 #
 # Set up a simple Compute Node + Page Server combination locally.
 #
 # NOTE: This doesn't clean up between invocations. You'll need to manually:
 #
 # - Kill any previous 'postgres' and 'pageserver' processes
 # - Clear the S3 bucket
 # - Remove the 'zenith-pgdata' directory
 set -e
 # Set up some config.
 #
 # CHANGE THESE ACCORDING TO YOUR S3 INSTALLATION
 export S3_REGION=auto
 export S3_ENDPOINT=https://localhost:9000
 export S3_ACCESSKEY=minioadmin
 export S3_SECRET=pikkunen
 export S3_BUCKET=zenith-testbucket
 COMPUTE_NODE_PGDATA=zenith-pgdata
 # 1. Initialize a cluster.
 initdb -D $COMPUTE_NODE_PGDATA -U zenith
 echo "port=65432" >> $COMPUTE_NODE_PGDATA/postgresql.conf
 echo "log_connections=on" >> $COMPUTE_NODE_PGDATA/postgresql.conf
 # Use a small shared_buffers, so that we hit the Page Server more
 # easily.
 echo "shared_buffers = 1MB" >> $COMPUTE_NODE_PGDATA/postgresql.conf
 # TODO: page server should use a replication slot, or some other mechanism
 # to make sure that the primary doesn't lose data that the page server still
 # needs. (The WAL safekeepers should ensure that)
 echo "wal_keep_size=10GB" >> $COMPUTE_NODE_PGDATA/postgresql.conf
 # Tell the Postgres server how to connect to the Page Server
 echo "page_server_connstring='host=localhost port=5430'" >> $COMPUTE_NODE_PGDATA/postgresql.conf
 # 2. Run zenith_push to push a base backup fo the database to an S3 bucket. The
 # Page Server will read it from there
 zenith_push -D $COMPUTE_NODE_PGDATA
 # 3. Launch page server
 rm -rf /tmp/pgdata-dummy
 initdb -N -D /tmp/pgdata-dummy
 PGDATA=/tmp/pgdata-dummy ./target/debug/pageserver  &
 # 4. Start up the Postgres server
 postgres -D $COMPUTE_NODE_PGDATA &
 echo "ALL SET! You can now connect to Postgres with something like:"
 echo ""
 echo 'psql "dbname=postgres host=localhost user=zenith port=65432"'
@@ -1,268 +1,23 @@
-//!
+use crate::ZTimelineId;
 //! Generate a tarball with files needed to bootstrap ComputeNode.
 //!
 //! TODO: this module has nothing to do with PostgreSQL pg_basebackup.
 //! It could use a better name.
 //!
 //! Stateless Postgres compute node is launched by sending a tarball
 //! which contains non-relational data (multixacts, clog, filenodemaps, twophase files),
 //! generated pg_control and dummy segment of WAL.
 //! This module is responsible for creation of such tarball
 //! from data stored in object storage.
 //!
 use bytes::{BufMut, BytesMut};
 use log::*;
 use std::io;
 use std::io::Write;
 use std::sync::Arc;
 use std::time::SystemTime;
-use tar::{Builder, EntryType, Header};
+use tar::{Builder, Header};
 use walkdir::WalkDir;
 use bytes::{BufMut, BytesMut};
-use crate::relish::*;
+use crate::repository::{BufferTag, RelTag, Timeline};
-use crate::repository::Timeline;
+use postgres_ffi::relfile_utils::*;
 use postgres_ffi::xlog_utils::*;
 use postgres_ffi::*;
 use zenith_utils::lsn::Lsn;
 /// This is short-living object only for the time of tarball creation,
 /// created mostly to avoid passing a lot of parameters between various functions
 /// used for constructing tarball.
 pub struct Basebackup<'a> {
    ar: Builder<&'a mut dyn Write>,
    timeline: &'a Arc<dyn Timeline>,
    lsn: Lsn,
    prev_record_lsn: Lsn,
 }
 impl<'a> Basebackup<'a> {
    pub fn new(
        write: &'a mut dyn Write,
        timeline: &'a Arc<dyn Timeline>,
        lsn: Lsn,
        prev_record_lsn: Lsn,
    ) -> Basebackup<'a> {
        Basebackup {
            ar: Builder::new(write),
            timeline,
            lsn,
            prev_record_lsn,
        }
    }
    pub fn send_tarball(&mut self) -> anyhow::Result<()> {
        // Create pgdata subdirs structure
        for dir in pg_constants::PGDATA_SUBDIRS.iter() {
            info!("send subdir {:?}", *dir);
            let header = new_tar_header_dir(*dir)?;
            self.ar.append(&header, &mut io::empty())?;
        }
        // Send empty config files.
        for filepath in pg_constants::PGDATA_SPECIAL_FILES.iter() {
            if *filepath == "pg_hba.conf" {
                let data = pg_constants::PG_HBA.as_bytes();
                let header = new_tar_header(&filepath, data.len() as u64)?;
                self.ar.append(&header, &data[..])?;
            } else {
                let header = new_tar_header(&filepath, 0)?;
                self.ar.append(&header, &mut io::empty())?;
            }
        }
        // Gather non-relational files from object storage pages.
        for obj in self.timeline.list_nonrels(self.lsn)? {
            match obj {
                RelishTag::Slru { slru, segno } => {
                    self.add_slru_segment(slru, segno)?;
                }
                RelishTag::FileNodeMap { spcnode, dbnode } => {
                    self.add_relmap_file(spcnode, dbnode)?;
                }
                RelishTag::TwoPhase { xid } => {
                    self.add_twophase_file(xid)?;
                }
                _ => {}
            }
        }
        // Generate pg_control and bootstrap WAL segment.
        self.add_pgcontrol_file()?;
        self.ar.finish()?;
        debug!("all tarred up!");
        Ok(())
    }
    //
    // Generate SLRU segment files from repository.
    //
    fn add_slru_segment(&mut self, slru: SlruKind, segno: u32) -> anyhow::Result<()> {
        let seg_size = self
            .timeline
            .get_relish_size(RelishTag::Slru { slru, segno }, self.lsn)?;
        if seg_size == None {
            trace!(
                "SLRU segment {}/{:>04X} was truncated",
                slru.to_str(),
                segno
            );
            return Ok(());
        }
        let nblocks = seg_size.unwrap();
        let mut slru_buf: Vec<u8> =
            Vec::with_capacity(nblocks as usize * pg_constants::BLCKSZ as usize);
        for blknum in 0..nblocks {
            let img = self.timeline.get_page_at_lsn_nowait(
                RelishTag::Slru { slru, segno },
                blknum,
                self.lsn,
            )?;
            assert!(img.len() == pg_constants::BLCKSZ as usize);
            slru_buf.extend_from_slice(&img);
        }
        let segname = format!("{}/{:>04X}", slru.to_str(), segno);
        let header = new_tar_header(&segname, slru_buf.len() as u64)?;
        self.ar.append(&header, slru_buf.as_slice())?;
        trace!("Added to basebackup slru {} relsize {}", segname, nblocks);
        Ok(())
    }
    //
    // Extract pg_filenode.map files from repository
    // Along with them also send PG_VERSION for each database.
    //
    fn add_relmap_file(&mut self, spcnode: u32, dbnode: u32) -> anyhow::Result<()> {
        let img = self.timeline.get_page_at_lsn_nowait(
            RelishTag::FileNodeMap { spcnode, dbnode },
            0,
            self.lsn,
        )?;
        let path = if spcnode == pg_constants::GLOBALTABLESPACE_OID {
            let dst_path = "PG_VERSION";
            let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
            self.ar.append(&header, &version_bytes[..])?;
            let dst_path = format!("global/PG_VERSION");
            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
            self.ar.append(&header, &version_bytes[..])?;
            String::from("global/pg_filenode.map") // filenode map for global tablespace
        } else {
            // User defined tablespaces are not supported
            assert!(spcnode == pg_constants::DEFAULTTABLESPACE_OID);
            // Append dir path for each database
            let path = format!("base/{}", dbnode);
            let header = new_tar_header_dir(&path)?;
            self.ar.append(&header, &mut io::empty())?;
            let dst_path = format!("base/{}/PG_VERSION", dbnode);
            let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
            self.ar.append(&header, &version_bytes[..])?;
            format!("base/{}/pg_filenode.map", dbnode)
        };
        assert!(img.len() == 512);
        let header = new_tar_header(&path, img.len() as u64)?;
        self.ar.append(&header, &img[..])?;
        Ok(())
    }
    //
    // Extract twophase state files
    //
    fn add_twophase_file(&mut self, xid: TransactionId) -> anyhow::Result<()> {
        if let Ok(img) =
            self.timeline
                .get_page_at_lsn_nowait(RelishTag::TwoPhase { xid }, 0, self.lsn)
        {
            let mut buf = BytesMut::new();
            buf.extend_from_slice(&img[..]);
            let crc = crc32c::crc32c(&img[..]);
            buf.put_u32_le(crc);
            let path = format!("pg_twophase/{:>08X}", xid);
            let header = new_tar_header(&path, buf.len() as u64)?;
            self.ar.append(&header, &buf[..])?;
        }
        Ok(())
    }
    //
    // Add generated pg_control file and bootstrap WAL segment.
    // Also send zenith.signal file with extra bootstrap data.
    //
    fn add_pgcontrol_file(&mut self) -> anyhow::Result<()> {
        let checkpoint_bytes =
            self.timeline
                .get_page_at_lsn_nowait(RelishTag::Checkpoint, 0, self.lsn)?;
        let pg_control_bytes =
            self.timeline
                .get_page_at_lsn_nowait(RelishTag::ControlFile, 0, self.lsn)?;
        let mut pg_control = ControlFileData::decode(&pg_control_bytes)?;
        let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
        // Generate new pg_control and WAL needed for bootstrap
        let checkpoint_segno = self.lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE);
        let checkpoint_lsn = XLogSegNoOffsetToRecPtr(
            checkpoint_segno,
            XLOG_SIZE_OF_XLOG_LONG_PHD as u32,
            pg_constants::WAL_SEGMENT_SIZE,
        );
        checkpoint.redo = self.lsn.0 + self.lsn.calc_padding(8u32);
        //reset some fields we don't want to preserve
        //TODO Check this.
        //We may need to determine the value from twophase data.
        checkpoint.oldestActiveXid = 0;
        //save new values in pg_control
        pg_control.checkPoint = checkpoint_lsn;
        pg_control.checkPointCopy = checkpoint;
        pg_control.state = pg_constants::DB_SHUTDOWNED;
        // add zenith.signal file
        self.ar.append(
            &new_tar_header("zenith.signal", 8)?,
            &self.prev_record_lsn.0.to_le_bytes()[..],
        )?;
        //send pg_control
        let pg_control_bytes = pg_control.encode();
        let header = new_tar_header("global/pg_control", pg_control_bytes.len() as u64)?;
        self.ar.append(&header, &pg_control_bytes[..])?;
        //send wal segment
        let wal_file_name = XLogFileName(
            1, // FIXME: always use Postgres timeline 1
            checkpoint_segno,
            pg_constants::WAL_SEGMENT_SIZE,
        );
        let wal_file_path = format!("pg_wal/{}", wal_file_name);
        let header = new_tar_header(&wal_file_path, pg_constants::WAL_SEGMENT_SIZE as u64)?;
        let wal_seg = generate_wal_segment(&pg_control);
        assert!(wal_seg.len() == pg_constants::WAL_SEGMENT_SIZE);
        self.ar.append(&header, &wal_seg[..])?;
        Ok(())
    }
 }
 //
 // Create new tarball entry header
 //
 fn new_tar_header(path: &str, size: u64) -> anyhow::Result<Header> {
    let mut header = Header::new_gnu();
    header.set_size(size);
    header.set_path(path)?;
-    header.set_mode(0b110000000); // -rw-------
+    header.set_mode(0b110000000);
    header.set_mtime(
        // use currenttime as last modified time
        SystemTime::now()
            .duration_since(SystemTime::UNIX_EPOCH)
            .unwrap()
@@ -272,19 +27,373 @@ fn new_tar_header(path: &str, size: u64) -> anyhow::Result<Header> {
    Ok(header)
 }
-fn new_tar_header_dir(path: &str) -> anyhow::Result<Header> {
+//
-    let mut header = Header::new_gnu();
+// Generate SRLU segment files from repository
-    header.set_size(0);
+//
-    header.set_path(path)?;
+fn add_slru_segments(
-    header.set_mode(0o755); // -rw-------
+    ar: &mut Builder<&mut dyn Write>,
-    header.set_entry_type(EntryType::dir());
+    timeline: &Arc<dyn Timeline>,
-    header.set_mtime(
+    path: &str,
-        // use currenttime as last modified time
+    forknum: u8,
-        SystemTime::now()
+    lsn: Lsn,
-            .duration_since(SystemTime::UNIX_EPOCH)
+) -> anyhow::Result<()> {
-            .unwrap()
+    let rel = RelTag {
-            .as_secs(),
+        spcnode: 0,
-    );
+        dbnode: 0,
-    header.set_cksum();
+        relnode: 0,
-    Ok(header)
+        forknum,
    };
    let (first, last) = timeline.get_range(rel, lsn)?;
    const SEG_SIZE: usize =
        pg_constants::BLCKSZ as usize * pg_constants::SLRU_PAGES_PER_SEGMENT as usize;
    let mut seg_buf = [0u8; SEG_SIZE];
    let mut curr_segno: Option<u32> = None;
    for page in first..last {
        let tag = BufferTag { rel, blknum: page };
        let img = timeline.get_page_at_lsn(tag, lsn)?;
        // Zero length image indicates truncated segment: just skip it
        if img.len() != 0 {
            assert!(img.len() == pg_constants::BLCKSZ as usize);
            let segno = page / pg_constants::SLRU_PAGES_PER_SEGMENT;
            if curr_segno.is_some() && curr_segno.unwrap() != segno {
                let segname = format!("{}/{:>04X}", path, curr_segno.unwrap());
                let header = new_tar_header(&segname, SEG_SIZE as u64)?;
                ar.append(&header, &seg_buf[..])?;
                seg_buf = [0u8; SEG_SIZE];
            }
            curr_segno = Some(segno);
            let offs_start = (page % pg_constants::SLRU_PAGES_PER_SEGMENT) as usize
                * pg_constants::BLCKSZ as usize;
            let offs_end = offs_start + pg_constants::BLCKSZ as usize;
            seg_buf[offs_start..offs_end].copy_from_slice(&img);
        }
    }
    if curr_segno.is_some() {
        let segname = format!("{}/{:>04X}", path, curr_segno.unwrap());
        let header = new_tar_header(&segname, SEG_SIZE as u64)?;
        ar.append(&header, &seg_buf[..])?;
    }
    Ok(())
 }
 //
 // Extract pg_filenode.map files from repository
 //
 fn add_relmap_files(
    ar: &mut Builder<&mut dyn Write>,
    timeline: &Arc<dyn Timeline>,
    lsn: Lsn,
    snappath: &str,
 ) -> anyhow::Result<()> {
    for db in timeline.get_databases(lsn)?.iter() {
        let tag = BufferTag {
            rel: *db,
            blknum: 0,
        };
        let img = timeline.get_page_at_lsn(tag, lsn)?;
        let path = if db.spcnode == pg_constants::GLOBALTABLESPACE_OID {
            String::from("global/pg_filenode.map")
        } else {
            // User defined tablespaces are not supported
            assert!(db.spcnode == pg_constants::DEFAULTTABLESPACE_OID);
            let src_path = format!("{}/base/1/PG_VERSION", snappath);
            let dst_path = format!("base/{}/PG_VERSION", db.dbnode);
            ar.append_path_with_name(&src_path, &dst_path)?;
            format!("base/{}/pg_filenode.map", db.dbnode)
        };
        assert!(img.len() == 512);
        let header = new_tar_header(&path, img.len() as u64)?;
        ar.append(&header, &img[..])?;
    }
    Ok(())
 }
 //
 // Extract twophase state files
 //
 fn add_twophase_files(
    ar: &mut Builder<&mut dyn Write>,
    timeline: &Arc<dyn Timeline>,
    lsn: Lsn,
 ) -> anyhow::Result<()> {
    for xid in timeline.get_twophase(lsn)?.iter() {
        let tag = BufferTag {
            rel: RelTag {
                spcnode: 0,
                dbnode: 0,
                relnode: 0,
                forknum: pg_constants::PG_TWOPHASE_FORKNUM,
            },
            blknum: *xid,
        };
        let img = timeline.get_page_at_lsn(tag, lsn)?;
 		let mut buf = BytesMut::new();
 		buf.extend_from_slice(&img[..]);
 		let crc = crc32c::crc32c(&img[..]);
 		buf.put_u32_le(crc);
        let path = format!("pg_twophase/{:>08X}", xid);
        let header = new_tar_header(&path, buf.len() as u64)?;
        ar.append(&header, &buf[..])?;
    }
    Ok(())
 }
 //
 // Add generated pg_control file
 //
 fn add_pgcontrol_file(
    ar: &mut Builder<&mut dyn Write>,
    timeline: &Arc<dyn Timeline>,
    lsn: Lsn,
 ) -> anyhow::Result<()> {
    if let Some(checkpoint_bytes) =
        timeline.get_page_image(BufferTag::fork(pg_constants::PG_CHECKPOINT_FORKNUM), Lsn(0))?
    {
        if let Some(pg_control_bytes) = timeline.get_page_image(
            BufferTag::fork(pg_constants::PG_CONTROLFILE_FORKNUM),
            Lsn(0),
        )? {
            let mut pg_control = postgres_ffi::decode_pg_control(pg_control_bytes)?;
            let mut checkpoint = postgres_ffi::decode_checkpoint(checkpoint_bytes)?;
            checkpoint.redo = lsn.0;
            checkpoint.nextXid.value += 1;
            // TODO: When we restart master there are no active transaction and oldestXid is
            // equal to nextXid if there are no prepared transactions.
            // Let's ignore them for a while...
            checkpoint.oldestXid = checkpoint.nextXid.value as u32;
            pg_control.checkPointCopy = checkpoint;
            let pg_control_bytes = postgres_ffi::encode_pg_control(pg_control);
            let header = new_tar_header("global/pg_control", pg_control_bytes.len() as u64)?;
            ar.append(&header, &pg_control_bytes[..])?;
        }
    }
    Ok(())
 }
 ///
 /// Generate tarball with non-relational files from repository
 ///
 pub fn send_tarball_at_lsn(
    write: &mut dyn Write,
    timelineid: ZTimelineId,
    timeline: &Arc<dyn Timeline>,
    lsn: Lsn,
    snapshot_lsn: Lsn,
 ) -> anyhow::Result<()> {
    let mut ar = Builder::new(write);
    let snappath = format!("timelines/{}/snapshots/{:016X}", timelineid, snapshot_lsn.0);
    debug!("sending tarball of snapshot in {}", snappath);
    for entry in WalkDir::new(&snappath) {
        let entry = entry?;
        let fullpath = entry.path();
        let relpath = entry.path().strip_prefix(&snappath).unwrap();
        if relpath.to_str().unwrap() == "" {
            continue;
        }
        if entry.file_type().is_dir() {
            trace!(
                "sending dir {} as {}",
                fullpath.display(),
                relpath.display()
            );
            ar.append_dir(relpath, fullpath)?;
        } else if entry.file_type().is_symlink() {
            error!("ignoring symlink in snapshot dir");
        } else if entry.file_type().is_file() {
            // Shared catalogs are exempt
            if relpath.starts_with("global/") {
                trace!("sending shared catalog {}", relpath.display());
                ar.append_path_with_name(fullpath, relpath)?;
            } else if !is_rel_file_path(relpath.to_str().unwrap()) {
                if entry.file_name() != "pg_filenode.map"
                    && entry.file_name() != "pg_control"
                    && !relpath.starts_with("pg_xact/")
                    && !relpath.starts_with("pg_multixact/")
                {
                    trace!("sending {}", relpath.display());
                    ar.append_path_with_name(fullpath, relpath)?;
                }
            } else {
                trace!("not sending {}", relpath.display());
            }
        } else {
            error!("unknown file type: {}", fullpath.display());
        }
    }
    add_slru_segments(
        &mut ar,
        timeline,
        "pg_xact",
        pg_constants::PG_XACT_FORKNUM,
        lsn,
    )?;
    add_slru_segments(
        &mut ar,
        timeline,
        "pg_multixact/members",
        pg_constants::PG_MXACT_MEMBERS_FORKNUM,
        lsn,
    )?;
    add_slru_segments(
        &mut ar,
        timeline,
        "pg_multixact/offsets",
        pg_constants::PG_MXACT_OFFSETS_FORKNUM,
        lsn,
    )?;
    add_relmap_files(&mut ar, timeline, lsn, &snappath)?;
    add_twophase_files(&mut ar, timeline, lsn)?;
    add_pgcontrol_file(&mut ar, timeline, lsn)?;
    ar.finish()?;
    debug!("all tarred up!");
    Ok(())
 }
 ///
 /// Send a tarball containing a snapshot of all non-relation files in the
 /// PostgreSQL data directory, at given LSN
 ///
 /// There must be a snapshot at the given LSN in the snapshots directory, we cannot
 /// reconstruct the state at an arbitrary LSN at the moment.
 ///
 pub fn send_snapshot_tarball(
    write: &mut dyn Write,
    timelineid: ZTimelineId,
    snapshotlsn: Lsn,
 ) -> Result<(), std::io::Error> {
    let mut ar = Builder::new(write);
    let snappath = format!("timelines/{}/snapshots/{:016X}", timelineid, snapshotlsn.0);
    let walpath = format!("timelines/{}/wal", timelineid);
    debug!("sending tarball of snapshot in {}", snappath);
    //ar.append_dir_all("", &snappath)?;
    for entry in WalkDir::new(&snappath) {
        let entry = entry?;
        let fullpath = entry.path();
        let relpath = entry.path().strip_prefix(&snappath).unwrap();
        if relpath.to_str().unwrap() == "" {
            continue;
        }
        if entry.file_type().is_dir() {
            trace!(
                "sending dir {} as {}",
                fullpath.display(),
                relpath.display()
            );
            ar.append_dir(relpath, fullpath)?;
        } else if entry.file_type().is_symlink() {
            error!("ignoring symlink in snapshot dir");
        } else if entry.file_type().is_file() {
            // Shared catalogs are exempt
            if relpath.starts_with("global/") {
                trace!("sending shared catalog {}", relpath.display());
                ar.append_path_with_name(fullpath, relpath)?;
            } else if !is_rel_file_path(relpath.to_str().unwrap()) {
                trace!("sending {}", relpath.display());
                ar.append_path_with_name(fullpath, relpath)?;
            } else {
                trace!("not sending {}", relpath.display());
                // FIXME: For now, also send all the relation files.
                // This really shouldn't be necessary, and kind of
                // defeats the point of having a page server in the
                // first place. But it is useful at least when
                // debugging with the DEBUG_COMPARE_LOCAL option (see
                // vendor/postgres/src/backend/storage/smgr/pagestore_smgr.c)
                ar.append_path_with_name(fullpath, relpath)?;
            }
        } else {
            error!("unknown file type: {}", fullpath.display());
        }
    }
    // FIXME: Also send all the WAL. The compute node would only need
    // the WAL that applies to non-relation files, because the page
    // server handles all the relation files. But we don't have a
    // mechanism for separating relation and non-relation WAL at the
    // moment.
    for entry in std::fs::read_dir(&walpath)? {
        let entry = entry?;
        let fullpath = &entry.path();
        let relpath = fullpath.strip_prefix(&walpath).unwrap();
        if !entry.path().is_file() {
            continue;
        }
        let archive_fname = relpath.to_str().unwrap();
        let archive_fname = archive_fname
            .strip_suffix(".partial")
            .unwrap_or(&archive_fname);
        let archive_path = "pg_wal/".to_owned() + archive_fname;
        ar.append_path_with_name(fullpath, archive_path)?;
    }
    ar.finish()?;
    debug!("all tarred up!");
    Ok(())
 }
 ///
 /// Parse a path, relative to the root of PostgreSQL data directory, as
 /// a PostgreSQL relation data file.
 ///
 fn parse_rel_file_path(path: &str) -> Result<(), FilePathError> {
    /*
     * Relation data files can be in one of the following directories:
     *
     * global/
     *		shared relations
     *
     * base/<db oid>/
     *		regular relations, default tablespace
     *
     * pg_tblspc/<tblspc oid>/<tblspc version>/
     *		within a non-default tablespace (the name of the directory
     *		depends on version)
     *
     * And the relation data files themselves have a filename like:
     *
     * <oid>.<segment number>
     */
    if let Some(fname) = path.strip_prefix("global/") {
        let (_relnode, _forknum, _segno) = parse_relfilename(fname)?;
        Ok(())
    } else if let Some(dbpath) = path.strip_prefix("base/") {
        let mut s = dbpath.split('/');
        let dbnode_str = s.next().ok_or(FilePathError::InvalidFileName)?;
        let _dbnode = dbnode_str.parse::<u32>()?;
        let fname = s.next().ok_or(FilePathError::InvalidFileName)?;
        if s.next().is_some() {
            return Err(FilePathError::InvalidFileName);
        };
        let (_relnode, _forknum, _segno) = parse_relfilename(fname)?;
        Ok(())
    } else if let Some(_) = path.strip_prefix("pg_tblspc/") {
        // TODO
        error!("tablespaces not implemented yet");
        Err(FilePathError::InvalidFileName)
    } else {
        Err(FilePathError::InvalidFileName)
    }
 }
 fn is_rel_file_path(path: &str) -> bool {
    parse_rel_file_path(path).is_ok()
 }
@@ -3,170 +3,29 @@
 //
 use log::*;
-use serde::{Deserialize, Serialize};
+use parse_duration::parse;
 use std::io;
 use std::process::exit;
 use std::thread;
 use std::time::Duration;
 use std::{env, path::PathBuf};
 use std::{
-    env,
+    fs::{File, OpenOptions},
    net::TcpListener,
    path::{Path, PathBuf},
    process::exit,
    str::FromStr,
    sync::Arc,
    thread,
    time::Duration,
 };
 use zenith_utils::{auth::JwtAuth, postgres_backend::AuthType};
-use anyhow::{ensure, Result};
+use anyhow::{Context, Result};
-use clap::{App, Arg, ArgMatches};
+use clap::{App, Arg};
 use daemonize::Daemonize;
-use pageserver::{branches, logger, page_cache, page_service, PageServerConf, RepositoryFormat};
+use slog::{Drain, FnValue};
 use zenith_utils::http_endpoint;
-const DEFAULT_LISTEN_ADDR: &str = "127.0.0.1:64000";
+use pageserver::{branches, page_cache, page_service, tui, PageServerConf};
 const DEFAULT_HTTP_ENDPOINT_ADDR: &str = "127.0.0.1:9898";
 const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
-const DEFAULT_GC_PERIOD: Duration = Duration::from_secs(10);
+const DEFAULT_GC_PERIOD_SEC: u64 = 10;
-
+//const DEFAULT_GC_HORIZON: u64 = 1024 * 1024 * 1024;
-const DEFAULT_SUPERUSER: &str = "zenith_admin";
+//const DEFAULT_GC_PERIOD_SEC: u64 = 600;
 /// String arguments that can be declared via CLI or config file
 #[derive(Serialize, Deserialize)]
 struct CfgFileParams {
    listen_addr: Option<String>,
    http_endpoint_addr: Option<String>,
    gc_horizon: Option<String>,
    gc_period: Option<String>,
    pg_distrib_dir: Option<String>,
    auth_validation_public_key_path: Option<String>,
    auth_type: Option<String>,
    repository_format: Option<String>,
 }
 impl CfgFileParams {
    /// Extract string arguments from CLI
    fn from_args(arg_matches: &ArgMatches) -> Self {
        let get_arg = |arg_name: &str| -> Option<String> {
            arg_matches.value_of(arg_name).map(str::to_owned)
        };
        Self {
            listen_addr: get_arg("listen"),
            http_endpoint_addr: get_arg("http_endpoint"),
            gc_horizon: get_arg("gc_horizon"),
            gc_period: get_arg("gc_period"),
            pg_distrib_dir: get_arg("postgres-distrib"),
            auth_validation_public_key_path: get_arg("auth-validation-public-key-path"),
            auth_type: get_arg("auth-type"),
            repository_format: get_arg("repository-format"),
        }
    }
    /// Fill missing values in `self` with `other`
    fn or(self, other: CfgFileParams) -> Self {
        // TODO cleaner way to do this
        Self {
            listen_addr: self.listen_addr.or(other.listen_addr),
            http_endpoint_addr: self.http_endpoint_addr.or(other.http_endpoint_addr),
            gc_horizon: self.gc_horizon.or(other.gc_horizon),
            gc_period: self.gc_period.or(other.gc_period),
            pg_distrib_dir: self.pg_distrib_dir.or(other.pg_distrib_dir),
            auth_validation_public_key_path: self
                .auth_validation_public_key_path
                .or(other.auth_validation_public_key_path),
            auth_type: self.auth_type.or(other.auth_type),
            repository_format: self.repository_format.or(other.repository_format),
        }
    }
    /// Create a PageServerConf from these string parameters
    fn try_into_config(&self) -> Result<PageServerConf> {
        let workdir = PathBuf::from(".");
        let listen_addr = match self.listen_addr.as_ref() {
            Some(addr) => addr.clone(),
            None => DEFAULT_LISTEN_ADDR.to_owned(),
        };
        let http_endpoint_addr = match self.http_endpoint_addr.as_ref() {
            Some(addr) => addr.clone(),
            None => DEFAULT_HTTP_ENDPOINT_ADDR.to_owned(),
        };
        let gc_horizon: u64 = match self.gc_horizon.as_ref() {
            Some(horizon_str) => horizon_str.parse()?,
            None => DEFAULT_GC_HORIZON,
        };
        let gc_period = match self.gc_period.as_ref() {
            Some(period_str) => humantime::parse_duration(period_str)?,
            None => DEFAULT_GC_PERIOD,
        };
        let pg_distrib_dir = match self.pg_distrib_dir.as_ref() {
            Some(pg_distrib_dir_str) => PathBuf::from(pg_distrib_dir_str),
            None => env::current_dir()?.join("tmp_install"),
        };
        let auth_validation_public_key_path = self
            .auth_validation_public_key_path
            .as_ref()
            .map(PathBuf::from);
        let auth_type = self
            .auth_type
            .as_ref()
            .map_or(Ok(AuthType::Trust), |auth_type| {
                AuthType::from_str(&auth_type)
            })?;
        if !pg_distrib_dir.join("bin/postgres").exists() {
            anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
        }
        if auth_type == AuthType::ZenithJWT {
            ensure!(
                auth_validation_public_key_path.is_some(),
                "Missing auth_validation_public_key_path when auth_type is ZenithJWT"
            );
            let path_ref = auth_validation_public_key_path.as_ref().unwrap();
            ensure!(
                path_ref.exists(),
                format!("Can't find auth_validation_public_key at {:?}", path_ref)
            );
        }
        let repository_format = match self.repository_format.as_ref() {
            Some(repo_format_str) if repo_format_str == "rocksdb" => RepositoryFormat::RocksDb,
            Some(repo_format_str) if repo_format_str == "layered" => RepositoryFormat::Layered,
            Some(repo_format_str) => anyhow::bail!(
                "invalid --repository-format '{}', must be 'rocksdb' or 'layered'",
                repo_format_str
            ),
            None => RepositoryFormat::Layered, // default
        };
        Ok(PageServerConf {
            daemonize: false,
            listen_addr,
            http_endpoint_addr,
            gc_horizon,
            gc_period,
            superuser: String::from(DEFAULT_SUPERUSER),
            workdir,
            pg_distrib_dir,
            auth_validation_public_key_path,
            auth_type,
            repository_format,
        })
    }
 }
 fn main() -> Result<()> {
    let arg_matches = App::new("Zenith page server")
@@ -178,6 +37,13 @@ fn main() -> Result<()> {
                .takes_value(true)
                .help("listen for incoming page requests on ip:port (default: 127.0.0.1:5430)"),
        )
        .arg(
            Arg::with_name("interactive")
                .short("i")
                .long("interactive")
                .takes_value(false)
                .help("Interactive mode"),
        )
        .arg(
            Arg::with_name("daemonize")
                .short("d")
@@ -210,97 +76,115 @@ fn main() -> Result<()> {
                .takes_value(true)
                .help("Working directory for the pageserver"),
        )
        .arg(
            Arg::with_name("postgres-distrib")
                .long("postgres-distrib")
                .takes_value(true)
                .help("Postgres distribution directory"),
        )
        .arg(
            Arg::with_name("create-tenant")
                .long("create-tenant")
                .takes_value(true)
                .help("Create tenant during init")
                .requires("init"),
        )
        .arg(
            Arg::with_name("auth-validation-public-key-path")
                .long("auth-validation-public-key-path")
                .takes_value(true)
                .help("Path to public key used to validate jwt signature"),
        )
        .arg(
            Arg::with_name("auth-type")
                .long("auth-type")
                .takes_value(true)
                .help("Authentication scheme type. One of: Trust, MD5, ZenithJWT"),
        )
        .arg(
            Arg::with_name("repository-format")
                .long("repository-format")
                .takes_value(true)
                .help("Which repository implementation to use, 'rocksdb' or 'layered'"),
        )
        .get_matches();
-    let workdir = Path::new(arg_matches.value_of("workdir").unwrap_or(".zenith"));
+    let workdir = if let Some(workdir_arg) = arg_matches.value_of("workdir") {
-    let cfg_file_path = workdir.canonicalize()?.join("pageserver.toml");
+        PathBuf::from(workdir_arg)
-
+    } else if let Some(workdir_arg) = std::env::var_os("ZENITH_REPO_DIR") {
-    let args_params = CfgFileParams::from_args(&arg_matches);
+        PathBuf::from(workdir_arg.to_str().unwrap())
    let init = arg_matches.is_present("init");
    let create_tenant = arg_matches.value_of("create-tenant");
    let params = if init {
        // We're initializing the repo, so there's no config file yet
        args_params
    } else {
-        // Supplement the CLI arguments with the config file
+        PathBuf::from(".zenith")
        let cfg_file_contents = std::fs::read_to_string(&cfg_file_path)?;
        let file_params: CfgFileParams = toml::from_str(&cfg_file_contents)?;
        args_params.or(file_params)
    };
-    // Set CWD to workdir for non-daemon modes
+    let pg_distrib_dir: PathBuf = {
-    env::set_current_dir(&workdir)?;
+        if let Some(postgres_bin) = env::var_os("POSTGRES_DISTRIB_DIR") {
            postgres_bin.into()
        } else {
            let cwd = env::current_dir()?;
            cwd.join("tmp_install")
        }
    };
-    // Ensure the config is valid, even if just init-ing
+    if !pg_distrib_dir.join("bin/postgres").exists() {
-    let mut conf = params.try_into_config()?;
+        anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
    }
-    conf.daemonize = arg_matches.is_present("daemonize");
+    let mut conf = PageServerConf {
        daemonize: false,
        interactive: false,
        gc_horizon: DEFAULT_GC_HORIZON,
        gc_period: Duration::from_secs(DEFAULT_GC_PERIOD_SEC),
        listen_addr: "127.0.0.1:64000".parse().unwrap(),
        // we will change the current working directory to the repository below,
        // so always set 'workdir' to '.'
        workdir: PathBuf::from("."),
        pg_distrib_dir,
    };
-    if init && conf.daemonize {
+    if arg_matches.is_present("daemonize") {
-        eprintln!("--daemonize cannot be used with --init");
+        conf.daemonize = true;
    }
    if arg_matches.is_present("interactive") {
        conf.interactive = true;
    }
    if conf.daemonize && conf.interactive {
        eprintln!("--daemonize is not allowed with --interactive: choose one");
        exit(1);
    }
    if let Some(addr) = arg_matches.value_of("listen") {
        conf.listen_addr = addr.parse()?;
    }
    if let Some(horizon) = arg_matches.value_of("gc_horizon") {
        conf.gc_horizon = horizon.parse()?;
    }
    if let Some(period) = arg_matches.value_of("gc_period") {
        conf.gc_period = parse(period)?;
    }
    // The configuration is all set up now. Turn it into a 'static
    // that can be freely stored in structs and passed across threads
    // as a ref.
    let conf: &'static PageServerConf = Box::leak(Box::new(conf));
    // Create repo and exit if init was requested
-    if init {
+    if arg_matches.is_present("init") {
-        branches::init_pageserver(conf, create_tenant)?;
+        branches::init_repo(conf, &workdir)?;
        // write the config file
        let cfg_file_contents = toml::to_string_pretty(&params)?;
        // TODO support enable-auth flag
        std::fs::write(&cfg_file_path, cfg_file_contents)?;
        return Ok(());
    }
    // Set CWD to workdir for non-daemon modes
    env::set_current_dir(&workdir)?;
    start_pageserver(conf)
 }
 fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
    let log_filename = "pageserver.log";
    // Don't open the same file for output multiple times;
    // the different fds could overwrite each other's output.
    let log_file = OpenOptions::new()
        .create(true)
        .append(true)
        .open(&log_filename)
        .with_context(|| format!("failed to open {:?}", &log_filename))?;
    // Initialize logger
-    let (_scope_guard, log_file) = logger::init_logging(&conf, "pageserver.log")?;
+    let logger_file = log_file.try_clone().unwrap();
    let _scope_guard = init_logging(&conf, logger_file)?;
    let _log_guard = slog_stdlog::init()?;
    // Note: this `info!(...)` macro comes from `log` crate
    info!("standard logging redirected to slog");
    let tui_thread = if conf.interactive {
        // Initialize the UI
        Some(
            thread::Builder::new()
                .name("UI thread".into())
                .spawn(|| {
                    let _ = tui::ui_main();
                })
                .unwrap(),
        )
    } else {
        None
    };
    // TODO: Check that it looks like a valid repository before going further
    if conf.daemonize {
@@ -323,40 +207,79 @@ fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
        }
    }
-    // Spawn a new thread for the http endpoint
+    // Check that we can bind to address before further initialization
    thread::Builder::new()
        .name("Metrics thread".into())
        .spawn(move || http_endpoint::thread_main(conf.http_endpoint_addr.clone()))?;
    // Check that we can bind to address before starting threads to simplify shutdown
    // sequence if port is occupied.
    info!("Starting pageserver on {}", conf.listen_addr);
-    let pageserver_listener = TcpListener::bind(conf.listen_addr.clone())?;
+    let pageserver_listener = TcpListener::bind(conf.listen_addr)?;
    // Initialize page cache, this will spawn walredo_thread
    page_cache::init(conf);
    // initialize authentication for incoming connections
    let auth = match &conf.auth_type {
        AuthType::Trust | AuthType::MD5 => Arc::new(None),
        AuthType::ZenithJWT => {
            // unwrap is ok because check is performed when creating config, so path is set and file exists
            let key_path = conf.auth_validation_public_key_path.as_ref().unwrap();
            Arc::new(Some(JwtAuth::from_key_path(key_path)?))
        }
    };
    info!("Using auth: {:#?}", conf.auth_type);
    // Spawn a thread to listen for connections. It will spawn further threads
    // for each connection.
    let page_service_thread = thread::Builder::new()
        .name("Page Service thread".into())
-        .spawn(move || {
+        .spawn(move || page_service::thread_main(conf, pageserver_listener))?;
            page_service::thread_main(conf, auth, pageserver_listener, conf.auth_type)
        })?;
-    page_service_thread
+    if let Some(tui_thread) = tui_thread {
-        .join()
+        // The TUI thread exits when the user asks to Quit.
-        .expect("Page service thread has panicked")?;
+        tui_thread.join().unwrap();
    } else {
        page_service_thread
            .join()
            .expect("Page service thread has panicked")?
    }
    Ok(())
 }
 fn init_logging(
    conf: &PageServerConf,
    log_file: File,
 ) -> Result<slog_scope::GlobalLoggerGuard, io::Error> {
    if conf.interactive {
        Ok(tui::init_logging())
    } else if conf.daemonize {
        let decorator = slog_term::PlainSyncDecorator::new(log_file);
        let drain = slog_term::FullFormat::new(decorator).build();
        let drain = slog::Filter::new(drain, |record: &slog::Record| {
            if record.level().is_at_least(slog::Level::Info) {
                return true;
            }
            false
        });
        let drain = std::sync::Mutex::new(drain).fuse();
        let logger = slog::Logger::root(
            drain,
            slog::o!(
                "location" =>
                FnValue(move |record| {
                    format!("{}, {}:{}",
                            record.module(),
                            record.file(),
                            record.line()
                            )
                    }
                )
            ),
        );
        Ok(slog_scope::set_global_logger(logger))
    } else {
        let decorator = slog_term::TermDecorator::new().build();
        let drain = slog_term::FullFormat::new(decorator).build().fuse();
        let drain = slog_async::Async::new(drain).chan_size(1000).build().fuse();
        let drain = slog::Filter::new(drain, |record: &slog::Record| {
            if record.level().is_at_least(slog::Level::Info) {
                return true;
            }
            if record.level().is_at_least(slog::Level::Debug)
                && record.module().starts_with("pageserver")
            {
                return true;
            }
            false
        })
        .fuse();
        let logger = slog::Logger::root(drain, slog::o!());
        Ok(slog_scope::set_global_logger(logger))
    }
 }
@@ -1,30 +1,28 @@
-//!
+//
-//! Branch management code
+// Branch management code
-//!
+//
 // TODO: move all paths construction to conf impl
 //
-use anyhow::{bail, ensure, Context, Result};
+use anyhow::{anyhow, bail, Context, Result};
-use postgres_ffi::ControlFileData;
+use bytes::Bytes;
 use fs::File;
 use fs_extra;
 use postgres_ffi::{pg_constants, xlog_utils};
 use rand::Rng;
 use serde::{Deserialize, Serialize};
 use std::env;
 use std::io::{Read, Write};
 use std::{
-    fs,
+    collections::HashMap,
-    path::Path,
+    fs, io,
    path::{Path, PathBuf},
    process::{Command, Stdio},
    str::FromStr,
    sync::Arc,
 };
 use zenith_utils::zid::{ZTenantId, ZTimelineId};
 use log::*;
 use zenith_utils::lsn::Lsn;
-use crate::logger;
+use crate::{repository::Repository, PageServerConf, ZTimelineId};
 use crate::object_repository::ObjectRepository;
 use crate::page_cache;
 use crate::restore_local_repo;
 use crate::walredo::WalRedoManager;
 use crate::{repository::Repository, PageServerConf, RepositoryFormat};
 #[derive(Serialize, Deserialize, Clone)]
 pub struct BranchInfo {
@@ -41,106 +39,38 @@ pub struct PointInTime {
    pub lsn: Lsn,
 }
-pub fn init_pageserver(conf: &'static PageServerConf, create_tenant: Option<&str>) -> Result<()> {
+pub fn init_repo(conf: &PageServerConf, repo_dir: &Path) -> Result<()> {
    // Initialize logger
    let (_scope_guard, _log_file) = logger::init_logging(&conf, "pageserver.log")?;
    let _log_guard = slog_stdlog::init()?;
    if let Some(tenantid) = create_tenant {
        let tenantid = ZTenantId::from_str(tenantid)?;
        println!("initializing tenantid {}", tenantid);
        create_repo(
            conf,
            tenantid,
            Arc::new(crate::walredo::DummyRedoManager {}),
        )
        .with_context(|| "failed to create repo")?;
    }
    fs::create_dir_all(conf.tenants_path())?;
    println!("pageserver init succeeded");
    Ok(())
 }
 pub fn create_repo(
    conf: &'static PageServerConf,
    tenantid: ZTenantId,
    wal_redo_manager: Arc<dyn WalRedoManager + Send + Sync>,
 ) -> Result<Arc<dyn Repository>> {
    let repo_dir = conf.tenant_path(&tenantid);
    if repo_dir.exists() {
        bail!("repo for {} already exists", tenantid)
    }
    // top-level dir may exist if we are creating it through CLI
-    fs::create_dir_all(&repo_dir)
+    fs::create_dir_all(repo_dir)
        .with_context(|| format!("could not create directory {}", repo_dir.display()))?;
-    // Note: this `info!(...)` macro comes from `log` crate
+    env::set_current_dir(repo_dir)?;
    info!("standard logging redirected to slog");
-    fs::create_dir(conf.timelines_path(&tenantid))?;
+    fs::create_dir(std::path::Path::new("timelines"))?;
-    fs::create_dir_all(conf.branches_path(&tenantid))?;
+    fs::create_dir(std::path::Path::new("refs"))?;
-    fs::create_dir_all(conf.tags_path(&tenantid))?;
+    fs::create_dir(std::path::Path::new("refs").join("branches"))?;
    fs::create_dir(std::path::Path::new("refs").join("tags"))?;
    fs::create_dir(std::path::Path::new("wal-redo"))?;
-    info!("created directory structure in {}", repo_dir.display());
+    println!("created directory structure in {}", repo_dir.display());
-    let tli = create_timeline(conf, None, &tenantid)?;
+    // Create initial timeline
    let tli = create_timeline(conf, None)?;
    let timelinedir = conf.timeline_path(tli);
    println!("created initial timeline {}", tli);
-    // We don't use page_cache here, because we don't want to spawn the WAL redo thread during
+    // Run initdb
    // repository initialization.
    //
-    // FIXME: That caused trouble, because the WAL redo thread launched initdb in the background,
+    // We create the cluster temporarily in a "tmp" directory inside the repository,
-    // and it kept running even after the "zenith init" had exited. In tests, we started the
+    // and move it to the right location from there.
-    // page server immediately after that, so that initdb was still running in the background,
+    let tmppath = std::path::Path::new("tmp");
    // and we failed to run initdb again in the same directory. This has been solved for the
    // rapid init+start case now, but the general race condition remains if you restart the
    // server quickly.
    let repo: Arc<dyn Repository + Sync + Send> =
        match conf.repository_format {
            RepositoryFormat::Layered => Arc::new(
                crate::layered_repository::LayeredRepository::new(conf, wal_redo_manager, tenantid),
            ),
            RepositoryFormat::RocksDb => {
                let obj_store = crate::rocksdb_storage::RocksObjectStore::create(conf, &tenantid)?;
-                Arc::new(ObjectRepository::new(
+    print!("running initdb... ");
-                    conf,
+    io::stdout().flush()?;
                    Arc::new(obj_store),
                    wal_redo_manager,
                    tenantid,
                ))
            }
        };
    // Load data into pageserver
    // TODO To implement zenith import we need to
    //      move data loading out of create_repo()
    bootstrap_timeline(conf, tenantid, tli, &*repo)?;
    Ok(repo)
 }
 // Returns checkpoint LSN from controlfile
 fn get_lsn_from_controlfile(path: &Path) -> Result<Lsn> {
    // Read control file to extract the LSN
    let controlfile_path = path.join("global").join("pg_control");
    let controlfile = ControlFileData::decode(&fs::read(controlfile_path)?)?;
    let lsn = controlfile.checkPoint;
    Ok(Lsn(lsn))
 }
 // Create the cluster temporarily in a initdbpath directory inside the repository
 // to get bootstrap data for timeline initialization.
 //
 fn run_initdb(conf: &'static PageServerConf, initdbpath: &Path) -> Result<()> {
    info!("running initdb... ");
    let initdb_path = conf.pg_bin_dir().join("initdb");
-    let initdb_output = Command::new(initdb_path)
+    let initdb_otput = Command::new(initdb_path)
-        .args(&["-D", initdbpath.to_str().unwrap()])
+        .args(&["-D", tmppath.to_str().unwrap()])
        .args(&["-U", &conf.superuser])
        .arg("--no-instructions")
        .env_clear()
        .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
@@ -148,77 +78,53 @@ fn run_initdb(conf: &'static PageServerConf, initdbpath: &Path) -> Result<()> {
        .stdout(Stdio::null())
        .output()
        .with_context(|| "failed to execute initdb")?;
-    if !initdb_output.status.success() {
+    if !initdb_otput.status.success() {
-        anyhow::bail!(
+        anyhow::bail!("initdb failed");
            "initdb failed: '{}'",
            String::from_utf8_lossy(&initdb_output.stderr)
        );
    }
-    info!("initdb succeeded");
+    println!("initdb succeeded");
-    Ok(())
+    // Read control file to extract the LSN and system id
-}
+    let controlfile_path = tmppath.join("global").join("pg_control");
    let controlfile = postgres_ffi::decode_pg_control(Bytes::from(fs::read(controlfile_path)?))?;
    // let systemid = controlfile.system_identifier;
    let lsn = controlfile.checkPoint;
    let lsnstr = format!("{:016X}", lsn);
-//
+    // Move the initial WAL file
-// - run initdb to init temporary instance and get bootstrap data
+    fs::rename(
-// - after initialization complete, remove the temp dir.
+        tmppath.join("pg_wal").join("000000010000000000000001"),
-//
+        timelinedir
-fn bootstrap_timeline(
+            .join("wal")
-    conf: &'static PageServerConf,
+            .join("000000010000000000000001.partial"),
-    tenantid: ZTenantId,
+    )?;
-    tli: ZTimelineId,
+    println!("moved initial WAL file");
    repo: &dyn Repository,
 ) -> Result<()> {
    let initdb_path = conf.tenant_path(&tenantid).join("tmp");
-    // Init temporarily repo to get bootstrap data
+    // Remove pg_wal
-    run_initdb(conf, &initdb_path)?;
+    fs::remove_dir_all(tmppath.join("pg_wal"))?;
    let pgdata_path = initdb_path;
-    let lsn = get_lsn_from_controlfile(&pgdata_path)?;
+    let target = timelinedir.join("snapshots").join(&lsnstr);
-
+    fs::rename(tmppath, &target)?;
    info!("bootstrap_timeline {:?} at lsn {}", pgdata_path, lsn);
    let timeline = repo.create_empty_timeline(tli, lsn)?;
    restore_local_repo::import_timeline_from_postgres_datadir(&pgdata_path, &*timeline, lsn)?;
    let wal_dir = pgdata_path.join("pg_wal");
    restore_local_repo::import_timeline_wal(&wal_dir, &*timeline, timeline.get_last_record_lsn())?;
    println!(
        "created initial timeline {} timeline.lsn {}",
        tli,
        timeline.get_last_record_lsn()
    );
    // Create 'main' branch to refer to the initial timeline
    let data = tli.to_string();
-    fs::write(conf.branch_path("main", &tenantid), data)?;
+    fs::write(conf.branch_path("main"), data)?;
    println!("created main branch");
-    // Remove temp dir. We don't need it anymore
+    println!(
-    fs::remove_dir_all(pgdata_path)?;
+        "new zenith repository was created in {}",
        repo_dir.display()
    );
    Ok(())
 }
-pub(crate) fn get_tenants(conf: &PageServerConf) -> Result<Vec<String>> {
+pub(crate) fn get_branches(
-    let tenants_dir = conf.tenants_path();
+    conf: &PageServerConf,
-
+    repository: &dyn Repository,
-    std::fs::read_dir(&tenants_dir)?
+) -> Result<Vec<BranchInfo>> {
        .map(|dir_entry_res| {
            let dir_entry = dir_entry_res?;
            ensure!(dir_entry.file_type()?.is_dir());
            Ok(dir_entry.file_name().to_str().unwrap().to_owned())
        })
        .collect()
 }
 pub(crate) fn get_branches(conf: &PageServerConf, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
    let repo = page_cache::get_repository_for_tenant(tenantid)?;
    // Each branch has a corresponding record (text file) in the refs/branches
    // with timeline_id.
-    let branches_dir = conf.branches_path(tenantid);
+    let branches_dir = std::path::Path::new("refs").join("branches");
    std::fs::read_dir(&branches_dir)?
        .map(|dir_entry_res| {
@@ -226,12 +132,12 @@ pub(crate) fn get_branches(conf: &PageServerConf, tenantid: &ZTenantId) -> Resul
            let name = dir_entry.file_name().to_str().unwrap().to_string();
            let timeline_id = std::fs::read_to_string(dir_entry.path())?.parse::<ZTimelineId>()?;
-            let latest_valid_lsn = repo
+            let latest_valid_lsn = repository
                .get_timeline(timeline_id)
                .map(|timeline| timeline.get_last_valid_lsn())
                .ok();
-            let ancestor_path = conf.ancestor_path(&timeline_id, tenantid);
+            let ancestor_path = conf.ancestor_path(timeline_id);
            let mut ancestor_id: Option<String> = None;
            let mut ancestor_lsn: Option<String> = None;
@@ -264,40 +170,67 @@ pub(crate) fn get_branches(conf: &PageServerConf, tenantid: &ZTenantId) -> Resul
        .collect()
 }
 pub(crate) fn get_system_id(conf: &PageServerConf) -> Result<u64> {
    // let branches = get_branches();
    let branches_dir = std::path::Path::new("refs").join("branches");
    let branches = std::fs::read_dir(&branches_dir)?
        .map(|dir_entry_res| {
            let dir_entry = dir_entry_res?;
            let name = dir_entry.file_name().to_str().unwrap().to_string();
            let timeline_id = std::fs::read_to_string(dir_entry.path())?.parse::<ZTimelineId>()?;
            Ok((name, timeline_id))
        })
        .collect::<Result<HashMap<String, ZTimelineId>>>()?;
    let main_tli = branches
        .get("main")
        .ok_or_else(|| anyhow!("Branch main not found"))?;
    let (_, main_snap_dir) = find_latest_snapshot(conf, *main_tli)?;
    let controlfile_path = main_snap_dir.join("global").join("pg_control");
    let controlfile = postgres_ffi::decode_pg_control(Bytes::from(fs::read(controlfile_path)?))?;
    Ok(controlfile.system_identifier)
 }
 pub(crate) fn create_branch(
    conf: &PageServerConf,
    branchname: &str,
    startpoint_str: &str,
    tenantid: &ZTenantId,
 ) -> Result<BranchInfo> {
-    let repo = page_cache::get_repository_for_tenant(tenantid)?;
+    if conf.branch_path(&branchname).exists() {
    if conf.branch_path(branchname, tenantid).exists() {
        anyhow::bail!("branch {} already exists", branchname);
    }
-    let mut startpoint = parse_point_in_time(conf, startpoint_str, tenantid)?;
+    let mut startpoint = parse_point_in_time(conf, startpoint_str)?;
    if startpoint.lsn == Lsn(0) {
        // Find end of WAL on the old timeline
-        let end_of_wal = repo
+        let end_of_wal = find_end_of_wal(conf, startpoint.timelineid)?;
            .get_timeline(startpoint.timelineid)?
            .get_last_record_lsn();
        println!("branching at end of WAL: {}", end_of_wal);
        startpoint.lsn = end_of_wal;
    }
-    // create a new timeline directory for it
+    // create a new timeline for it
-    let newtli = create_timeline(conf, Some(startpoint), tenantid)?;
+    let newtli = create_timeline(conf, Some(startpoint))?;
    let newtimelinedir = conf.timeline_path(newtli);
    // Let the Repository backend do its initialization
    repo.branch_timeline(startpoint.timelineid, newtli, startpoint.lsn)?;
    // Remember the human-readable branch name for the new timeline.
    // FIXME: there's a race condition, if you create a branch with the same
    // name concurrently.
    let data = newtli.to_string();
-    fs::write(conf.branch_path(&branchname, tenantid), data)?;
+    fs::write(conf.branch_path(&branchname), data)?;
    // Copy the latest snapshot (TODO: before the startpoint) and all WAL
    // TODO: be smarter and avoid the copying...
    let (_maxsnapshot, oldsnapshotdir) = find_latest_snapshot(conf, startpoint.timelineid)?;
    let copy_opts = fs_extra::dir::CopyOptions::new();
    fs_extra::dir::copy(oldsnapshotdir, newtimelinedir.join("snapshots"), &copy_opts)?;
    let oldtimelinedir = conf.timeline_path(startpoint.timelineid);
    copy_wal(
        &oldtimelinedir.join("wal"),
        &newtimelinedir.join("wal"),
        startpoint.lsn,
        pg_constants::WAL_SEGMENT_SIZE,
    )?;
    Ok(BranchInfo {
        name: branchname.to_string(),
@@ -327,11 +260,7 @@ pub(crate) fn create_branch(
 //    mytag
 //
 //
-fn parse_point_in_time(
+fn parse_point_in_time(conf: &PageServerConf, s: &str) -> Result<PointInTime> {
    conf: &PageServerConf,
    s: &str,
    tenantid: &ZTenantId,
 ) -> Result<PointInTime> {
    let mut strings = s.split('@');
    let name = strings.next().unwrap();
@@ -346,21 +275,21 @@ fn parse_point_in_time(
    // Check if it's a tag
    if lsn.is_none() {
-        let tagpath = conf.tag_path(name, &tenantid);
+        let tagpath = conf.tag_path(name);
        if tagpath.exists() {
            let pointstr = fs::read_to_string(tagpath)?;
-            return parse_point_in_time(conf, &pointstr, &tenantid);
+            return parse_point_in_time(conf, &pointstr);
        }
    }
    // Check if it's a branch
    // Check if it's branch @ LSN
-    let branchpath = conf.branch_path(name, &tenantid);
+    let branchpath = conf.branch_path(name);
    if branchpath.exists() {
        let pointstr = fs::read_to_string(branchpath)?;
-        let mut result = parse_point_in_time(conf, &pointstr, &tenantid)?;
+        let mut result = parse_point_in_time(conf, &pointstr)?;
        result.lsn = lsn.unwrap_or(Lsn(0));
        return Ok(result);
@@ -369,7 +298,7 @@ fn parse_point_in_time(
    // Check if it's a timelineid
    // Check if it's timelineid @ LSN
    if let Ok(timelineid) = ZTimelineId::from_str(name) {
-        let tlipath = conf.timeline_path(&timelineid, &tenantid);
+        let tlipath = conf.timeline_path(timelineid);
        if tlipath.exists() {
            return Ok(PointInTime {
                timelineid,
@@ -381,18 +310,16 @@ fn parse_point_in_time(
    bail!("could not parse point-in-time {}", s);
 }
-fn create_timeline(
+fn create_timeline(conf: &PageServerConf, ancestor: Option<PointInTime>) -> Result<ZTimelineId> {
    conf: &PageServerConf,
    ancestor: Option<PointInTime>,
    tenantid: &ZTenantId,
 ) -> Result<ZTimelineId> {
    // Create initial timeline
    let mut tli_buf = [0u8; 16];
    rand::thread_rng().fill(&mut tli_buf);
    let timelineid = ZTimelineId::from(tli_buf);
-    let timelineid = ZTimelineId::generate();
+    let timelinedir = conf.timeline_path(timelineid);
    let timelinedir = conf.timeline_path(&timelineid, tenantid);
    fs::create_dir(&timelinedir)?;
    fs::create_dir(&timelinedir.join("snapshots"))?;
    fs::create_dir(&timelinedir.join("wal"))?;
    if let Some(ancestor) = ancestor {
@@ -402,3 +329,81 @@ fn create_timeline(
    Ok(timelineid)
 }
 ///
 /// Copy all WAL segments from one directory to another, up to given LSN.
 ///
 /// If the given LSN is in the middle of a segment, the last segment containing it
 /// is written out as .partial, and padded with zeros.
 ///
 fn copy_wal(src_dir: &Path, dst_dir: &Path, upto: Lsn, wal_seg_size: usize) -> Result<()> {
    let last_segno = upto.segment_number(wal_seg_size);
    let last_segoff = upto.segment_offset(wal_seg_size);
    for entry in fs::read_dir(src_dir).unwrap() {
        if let Ok(entry) = entry {
            let entry_name = entry.file_name();
            let fname = entry_name.to_str().unwrap();
            // Check if the filename looks like an xlog file, or a .partial file.
            if !xlog_utils::IsXLogFileName(fname) && !xlog_utils::IsPartialXLogFileName(fname) {
                continue;
            }
            let (segno, _tli) = xlog_utils::XLogFromFileName(fname, wal_seg_size as usize);
            let copylen;
            let mut dst_fname = PathBuf::from(fname);
            if segno > last_segno {
                // future segment, skip
                continue;
            } else if segno < last_segno {
                copylen = wal_seg_size;
                dst_fname.set_extension("");
            } else {
                copylen = last_segoff;
                dst_fname.set_extension("partial");
            }
            let src_file = File::open(entry.path())?;
            let mut dst_file = File::create(dst_dir.join(&dst_fname))?;
            std::io::copy(&mut src_file.take(copylen as u64), &mut dst_file)?;
            if copylen < wal_seg_size {
                std::io::copy(
                    &mut std::io::repeat(0).take((wal_seg_size - copylen) as u64),
                    &mut dst_file,
                )?;
            }
        }
    }
    Ok(())
 }
 // Find the end of valid WAL in a wal directory
 pub fn find_end_of_wal(conf: &PageServerConf, timeline: ZTimelineId) -> Result<Lsn> {
    let waldir = conf.timeline_path(timeline).join("wal");
    let (lsn, _tli) = xlog_utils::find_end_of_wal(&waldir, pg_constants::WAL_SEGMENT_SIZE, true);
    Ok(Lsn(lsn))
 }
 // Find the latest snapshot for a timeline
 fn find_latest_snapshot(conf: &PageServerConf, timeline: ZTimelineId) -> Result<(Lsn, PathBuf)> {
    let snapshotsdir = conf.snapshots_path(timeline);
    let paths = fs::read_dir(&snapshotsdir)?;
    let mut maxsnapshot = Lsn(0);
    let mut snapshotdir: Option<PathBuf> = None;
    for path in paths {
        let path = path?;
        let filename = path.file_name().to_str().unwrap().to_owned();
        if let Ok(lsn) = Lsn::from_hex(&filename) {
            maxsnapshot = std::cmp::max(lsn, maxsnapshot);
            snapshotdir = Some(path.path());
        }
    }
    if maxsnapshot == Lsn(0) {
        // TODO: check ancestor timeline
        anyhow::bail!("no snapshot found in {}", snapshotsdir.display());
    }
    Ok((maxsnapshot, snapshotdir.unwrap()))
 }
@@ -1,298 +0,0 @@
 # Overview
 The on-disk format is based on immutable files. The page server
 receives a stream of incoming WAL, parses the WAL records to determine
 which pages they apply to, and accumulates the incoming changes in
 memory. Every now and then, the accumulated changes are written out to
 new files.
 The files are called "snapshot files". Each snapshot file corresponds
 to one 10 MB slice of a PostgreSQL relation fork. The snapshot files
 for each timeline are stored in the timeline's subdirectory under
 .zenith/tenants/<tenantid>/timelines.
 The files are named like this:
    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>_<end LSN>
 For example:
    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000
 Some non-relation files are also stored in repository. For example,
 a CLOG segment would be named like this:
    pg_xact_0000_0_00000000198B06B0_00000000198C2550
 There is no difference in how the relation and non-relation files are
 managed, except that the first part of file names is different.
 Internally, the relations and non-relation files that are managed in
 the versioned store are together called "relishes".
 Each snapshot file contains a full snapshot, that is, full copy of all
 pages in the relation, as of the "start LSN". It also contains all WAL
 records applicable to the relation between the start and end
 LSNs. With this information, the page server can reconstruct any page
 version of the relation in the LSN range.
 If a file has been dropped, the last snapshot file for it is created
 with the _DROPPED suffix, e.g.
    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000_DROPPED
 In addition to the relations, with "rel_*" prefix, we use the same
 format for storing various smaller files from the PostgreSQL data
 directory. They will use different suffixes and the naming scheme
 up to the LSN range varies. The Zenith source code uses the term
 "relish" to mean "a relation, or other file that's treated like a
 relation in the storage"
 ## Notation used in this document
 The full path of a snapshot file looks like this:
    .zenith/tenants/941ddc8604413b88b3d208bddf90396c/timelines/4af489b06af8eed9e27a841775616962/rel_1663_13990_2609_0_10_000000000169C348_0000000001702000
 For simplicity, the examples below use a simplified notation for the
 paths.  The tenant ID is left out, the timeline ID is replaced with
 the human-readable branch name, and spcnode+dbnode+relnode+forkum+segno
 with a human-readable table name. The LSNs are also shorter. For
 example, a snapshot file for 'orders' table on 'main' branch, with LSN
 range 100-200 would be:
    main/orders_100_200
 # Creating snapshot files
 Let's start with a simple example with a system that contains one
 branch called 'main' and two tables, 'orders' and 'customers'. The end
 of WAL is currently at LSN 250. In this starting situation, you would
 have two files on disk:
 	main/orders_100_200
 	main/customers_100_200
 In addition to those files, the recent changes between LSN 200 and the
 end of WAL at 250 are kept in memory. If the page server crashes, the
 latest records between 200-250 need to be re-read from the WAL.
 Whenever enough WAL has been accumulated in memory, the page server
 writes out the changes in memory into new snapshot files. This process
 is called "checkpointing" (not to be confused with the PostgreSQL
 checkpoints, that's a different thing). The page server only creates
 snapshot files for relations that have been modified since the last
 checkpoint. For example, if the current end of WAL is at LSN 450, and
 the last checkpoint happened at LSN 400 but there hasn't been any
 recent changes to 'customers' table, you would have these files on
 disk:
 	main/orders_100_200
 	main/orders_200_300
 	main/orders_300_400
 	main/customers_100_200
 If the customers table is modified later, a new file is created for it
 at the next checkpoint. The new file will cover the "gap" from the
 last snapshot file, so the LSN ranges are always contiguous:
 	main/orders_100_200
 	main/orders_200_300
 	main/orders_300_400
 	main/customers_100_200
 	main/customers_200_500
 ## Reading page versions
 Whenever a GetPage@LSN request comes in from the compute node, the
 page server needs to reconstruct the requested page, as it was at the
 requested LSN. To do that, the page server first checks the recent
 in-memory layer; if the requested page version is found there, it can
 be returned immediatedly without looking at the files on
 disk. Otherwise the page server needs to locate the snapshot file that
 contains the requested page version.
 For example, if a request comes in for table 'orders' at LSN 250, the
 page server would load the 'main/orders_200_300' file into memory, and
 reconstruct and return the requested page from it, as it was at
 LSN 250. Because the snapshot file consists of a full image of the
 relation at the start LSN and the WAL, reconstructing the page
 involves replaying any WAL records applicable to the page between LSNs
 200-250, starting from the base image at LSN 200.
 A request at a file boundary can be satisfied using either file. For
 example, if there are two files on disk:
 	main/orders_100_200
 	main/orders_200_300
 And a request comes with LSN 200, either file can be used for it. It
 is better to use the later file, however, because it contains an
 already materialized version of all the pages at LSN 200. Using the
 first file, you would need to apply any WAL records between 100 and
 200 to reconstruct the requested page.
 # Multiple branches
 Imagine that a child branch is created at LSN 250:
            @250
    ----main--+-------------------------->
               \
                +---child-------------->
 Then, the 'orders' table is updated differently on the 'main' and
 'child' branches. You now have this situation on disk:
    main/orders_100_200
    main/orders_200_300
    main/orders_300_400
    main/customers_100_200
    child/orders_250_300
    child/orders_300_400
 Because the 'customers' table hasn't been modified on the child
 branch, there is no file for it there. If you request a page for it on
 the 'child' branch, the page server will not find any snapshot file
 for it in the 'child' directory, so it will recurse to look into the
 parent 'main' branch instead.
 From the 'child' branch's point of view, the history for each relation
 is linear, and the request's LSN identifies unambiguously which file
 you need to look at. For example, the history for the 'orders' table
 on the 'main' branch consists of these files:
    main/orders_100_200
    main/orders_200_300
    main/orders_300_400
 And from the 'child' branch's point of view, it consists of these
 files:
    main/orders_100_200
    main/orders_200_300
    child/orders_250_300
    child/orders_300_400
 The branch metadata includes the point where the child branch was
 created, LSN 250. If a page request comes with LSN 275, we read the
 page version from the 'child/orders_250_300' file. If the request LSN
 is 225, we read it from the 'main/orders_200_300' file instead.  The
 page versions between 250-300 in the 'main/orders_200_300' file are
 ignored when operating on the child branch.
 Note: It doesn't make any difference if the child branch is created
 when the end of the main branch was at LSN 250, or later when the tip of
 the main branch had already moved on. The latter case, creating a
 branch at a historic LSN, is how we support PITR in Zenith.
 # Garbage collection
 In this scheme, we keep creating new snapshot files over time. We also
 need a mechanism to remove old files that are no longer needed,
 because disk space isn't infinite.
 What files are still needed? Currently, the page server supports PITR
 and branching from any branch at any LSN that is "recent enough" from
 the tip of the branch.  "Recent enough" is defined as an LSN horizon,
 which by default is 64 MB.  (See DEFAULT_GC_HORIZON). For this
 example, let's assume that the LSN horizon is 150 units.
 Let's look at the single branch scenario again. Imagine that the end
 of the branch is LSN 525, so that the GC horizon is currently at
 525-150 = 375
 	main/orders_100_200
 	main/orders_200_300
 	main/orders_300_400
 	main/orders_400_500
 	main/customers_100_200
 We can remove files 'main/orders_100_200' and 'main/orders_200_300',
 because the end LSNs of those files are older than GC horizon 375, and
 there are more recent snapshot files for the table. 'main/orders_300_400'
 and 'main/orders_400_500' are still within the horizon, so they must be
 retained. 'main/customers_100_200' is old enough, but it cannot be
 removed because there is no newer snapshot file for the table.
 Things get slightly more complicated with multiple branches. All of
 the above still holds, but in addition to recent files we must also
 retain older shapshot files that are still needed by child branches.
 For example, if child branch is created at LSN 150, and the 'customers'
 table is updated on the branch, you would have these files:
 	main/orders_100_200
 	main/orders_200_300
 	main/orders_300_400
 	main/orders_400_500
 	main/customers_100_200
 	child/customers_150_300
 In this situation, the 'main/orders_100_200' file cannot be removed,
 even though it is older than the GC horizon, because it is still
 needed by the child branch.  'main/orders_200_300' can still be
 removed. So after garbage collection, these files would remain:
 	main/orders_100_200
 	main/orders_300_400
 	main/orders_400_500
 	main/customers_100_200
 	child/customers_150_300
 If 'orders' is modified later on the 'child' branch, we will create a
 snapshot file for it on the child:
 	main/orders_100_200
 	main/orders_300_400
 	main/orders_400_500
 	main/customers_100_200
 	child/customers_150_300
 	child/orders_150_400
 After this, the 'main/orders_100_200' file can be removed. It is no
 longer needed by the child branch, because there is a newer snapshot
 file there. TODO: This optimization hasn't been implemented! The GC
 algorithm will currently keep the file on the 'main' branch anyway, for
 as long as the child branch exists.
 # TODO: On LSN ranges
 In principle, each relation can be checkpointed separately, i.e. the
 LSN ranges of the files don't need to line up. So this would be legal:
 	main/orders_100_200
 	main/orders_200_300
 	main/orders_300_400
 	main/customers_150_250
 	main/customers_250_500
 However, the code currently always checkpoints all relations together.
 So that situation doesn't arise in practice.
 It would also be OK to have overlapping LSN ranges for the same relation:
 	main/orders_100_200
 	main/orders_200_300
 	main/orders_250_350
 	main/orders_300_400
 The code that reads the snapshot files should cope with this, but this
 situation doesn't arise either, because the checkpointing code never
 does that.  It could be useful, however, as a transient state when
 garbage collecting around branch points, or explicit recovery
 points. For example, if we start with this:
 	main/orders_100_200
 	main/orders_200_300
 	main/orders_300_400
 And there is a branch or explicit recovery point at LSN 150, we could
 replace 'main/orders_100_200' with 'main/orders_150_150' to keep a
 snapshot only at that exact point that's still needed, removing the
 other page versions around it. But such compaction has not been
 implemented yet.
@@ -1,491 +0,0 @@
 //!
 //! An in-memory layer stores recently received page versions in memory. The page versions
 //! are held in a BTreeMap, and there's another BTreeMap to track the size of the relation.
 //!
 use crate::layered_repository::storage_layer::{
    Layer, PageReconstructData, PageVersion, SegmentTag, RELISH_SEG_SIZE,
 };
 use crate::layered_repository::LayeredTimeline;
 use crate::layered_repository::SnapshotLayer;
 use crate::repository::WALRecord;
 use crate::PageServerConf;
 use crate::{ZTenantId, ZTimelineId};
 use anyhow::{bail, Result};
 use bytes::Bytes;
 use log::*;
 use std::collections::BTreeMap;
 use std::ops::Bound::Included;
 use std::sync::{Arc, Mutex};
 use zenith_utils::lsn::Lsn;
 pub struct InMemoryLayer {
    conf: &'static PageServerConf,
    tenantid: ZTenantId,
    timelineid: ZTimelineId,
    seg: SegmentTag,
    ///
    /// This layer contains all the changes from 'start_lsn'. The
    /// start is inclusive. There is no end LSN; we only use in-memory
    /// layer at the end of a timeline.
    ///
    start_lsn: Lsn,
    /// The above fields never change. The parts that do change are in 'inner',
    /// and protected by mutex.
    inner: Mutex<InMemoryLayerInner>,
 }
 pub struct InMemoryLayerInner {
    /// If this relation was dropped, remember when that happened.
    drop_lsn: Option<Lsn>,
    ///
    /// All versions of all pages in the layer are are kept here.
    /// Indexed by block number and LSN.
    ///
    page_versions: BTreeMap<(u32, Lsn), PageVersion>,
    ///
    /// `segsizes` tracks the size of the segment at different points in time.
    ///
    segsizes: BTreeMap<Lsn, u32>,
 }
 impl Layer for InMemoryLayer {
    fn get_timeline_id(&self) -> ZTimelineId {
        return self.timelineid;
    }
    fn get_seg_tag(&self) -> SegmentTag {
        return self.seg;
    }
    fn get_start_lsn(&self) -> Lsn {
        return self.start_lsn;
    }
    fn get_end_lsn(&self) -> Lsn {
        let inner = self.inner.lock().unwrap();
        if let Some(drop_lsn) = inner.drop_lsn {
            drop_lsn
        } else {
            Lsn(u64::MAX)
        }
    }
    fn is_dropped(&self) -> bool {
        let inner = self.inner.lock().unwrap();
        inner.drop_lsn.is_some()
    }
    /// Look up given page in the cache.
    fn get_page_reconstruct_data(
        &self,
        blknum: u32,
        lsn: Lsn,
        reconstruct_data: &mut PageReconstructData,
    ) -> Result<Option<Lsn>> {
        // Scan the BTreeMap backwards, starting from reconstruct_data.lsn.
        let mut need_base_image_lsn: Option<Lsn> = Some(lsn);
        assert!(self.seg.blknum_in_seg(blknum));
        {
            let inner = self.inner.lock().unwrap();
            let minkey = (blknum, Lsn(0));
            let maxkey = (blknum, lsn);
            let mut iter = inner
                .page_versions
                .range((Included(&minkey), Included(&maxkey)));
            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
                if let Some(img) = &entry.page_image {
                    reconstruct_data.page_img = Some(img.clone());
                    need_base_image_lsn = None;
                    break;
                } else if let Some(rec) = &entry.record {
                    reconstruct_data.records.push(rec.clone());
                    if rec.will_init {
                        // This WAL record initializes the page, so no need to go further back
                        need_base_image_lsn = None;
                        break;
                    } else {
                        need_base_image_lsn = Some(*entry_lsn);
                    }
                } else {
                    // No base image, and no WAL record. Huh?
                    bail!("no page image or WAL record for requested page");
                }
            }
            // release lock on 'page_versions'
        }
        Ok(need_base_image_lsn)
    }
    /// Get size of the relation at given LSN
    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
        // Scan the BTreeMap backwards, starting from the given entry.
        let inner = self.inner.lock().unwrap();
        let mut iter = inner.segsizes.range((Included(&Lsn(0)), Included(&lsn)));
        if let Some((_entry_lsn, entry)) = iter.next_back() {
            let result = *entry;
            drop(inner);
            trace!("get_seg_size: {} at {} -> {}", self.seg, lsn, result);
            Ok(result)
        } else {
            bail!("No size found for {} at {} in memory", self.seg, lsn);
        }
    }
    /// Does this segment exist at given LSN?
    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
        let inner = self.inner.lock().unwrap();
        // Is the requested LSN after the segment was dropped?
        if let Some(drop_lsn) = inner.drop_lsn {
            if lsn >= drop_lsn {
                return Ok(false);
            }
        }
        // Otherwise, it exists
        Ok(true)
    }
 }
 impl InMemoryLayer {
    ///
    /// Create a new, empty, in-memory layer
    ///
    pub fn create(
        conf: &'static PageServerConf,
        timelineid: ZTimelineId,
        tenantid: ZTenantId,
        seg: SegmentTag,
        start_lsn: Lsn,
    ) -> Result<InMemoryLayer> {
        trace!(
            "initializing new empty InMemoryLayer for writing {} on timeline {} at {}",
            seg,
            timelineid,
            start_lsn
        );
        Ok(InMemoryLayer {
            conf,
            timelineid,
            tenantid,
            seg,
            start_lsn,
            inner: Mutex::new(InMemoryLayerInner {
                drop_lsn: None,
                page_versions: BTreeMap::new(),
                segsizes: BTreeMap::new(),
            }),
        })
    }
    // Write operations
    /// Remember new page version, as a WAL record over previous version
    pub fn put_wal_record(&self, blknum: u32, rec: WALRecord) -> Result<()> {
        self.put_page_version(
            blknum,
            rec.lsn,
            PageVersion {
                page_image: None,
                record: Some(rec),
            },
        )
    }
    /// Remember new page version, as a full page image
    pub fn put_page_image(&self, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()> {
        self.put_page_version(
            blknum,
            lsn,
            PageVersion {
                page_image: Some(img),
                record: None,
            },
        )
    }
    /// Common subroutine of the public put_wal_record() and put_page_image() functions.
    /// Adds the page version to the in-memory tree
    pub fn put_page_version(&self, blknum: u32, lsn: Lsn, pv: PageVersion) -> Result<()> {
        assert!(self.seg.blknum_in_seg(blknum));
        trace!(
            "put_page_version blk {} of {} at {}/{}",
            blknum,
            self.seg.rel,
            self.timelineid,
            lsn
        );
        let mut inner = self.inner.lock().unwrap();
        let old = inner.page_versions.insert((blknum, lsn), pv);
        if old.is_some() {
            // We already had an entry for this LSN. That's odd..
            warn!(
                "Page version of rel {} blk {} at {} already exists",
                self.seg.rel, blknum, lsn
            );
        }
        // Also update the relation size, if this extended the relation.
        if self.seg.rel.is_blocky() {
            let newsize = blknum - self.seg.segno * RELISH_SEG_SIZE + 1;
            let mut iter = inner.segsizes.range((Included(&Lsn(0)), Included(&lsn)));
            let oldsize;
            if let Some((_entry_lsn, entry)) = iter.next_back() {
                oldsize = *entry;
            } else {
                oldsize = 0;
                //bail!("No old size found for {} at {}", self.tag, lsn);
            }
            if newsize > oldsize {
                trace!(
                    "enlarging segment {} from {} to {} blocks at {}",
                    self.seg,
                    oldsize,
                    newsize,
                    lsn
                );
                inner.segsizes.insert(lsn, newsize);
            }
        }
        Ok(())
    }
    /// Remember that the relation was truncated at given LSN
    pub fn put_truncation(&self, lsn: Lsn, segsize: u32) -> anyhow::Result<()> {
        let mut inner = self.inner.lock().unwrap();
        let old = inner.segsizes.insert(lsn, segsize);
        if old.is_some() {
            // We already had an entry for this LSN. That's odd..
            warn!("Inserting truncation, but had an entry for the LSN already");
        }
        Ok(())
    }
    /// Remember that the segment was dropped at given LSN
    pub fn put_unlink(&self, lsn: Lsn) -> anyhow::Result<()> {
        let mut inner = self.inner.lock().unwrap();
        assert!(inner.drop_lsn.is_none());
        inner.drop_lsn = Some(lsn);
        info!("dropped segment {} at {}", self.seg, lsn);
        Ok(())
    }
    ///
    /// Initialize a new InMemoryLayer for, by copying the state at the given
    /// point in time from given existing layer.
    ///
    pub fn copy_snapshot(
        conf: &'static PageServerConf,
        timeline: &LayeredTimeline,
        src: &dyn Layer,
        timelineid: ZTimelineId,
        tenantid: ZTenantId,
        lsn: Lsn,
    ) -> Result<InMemoryLayer> {
        trace!(
            "initializing new InMemoryLayer for writing {} on timeline {} at {}",
            src.get_seg_tag(),
            timelineid,
            lsn
        );
        let mut page_versions = BTreeMap::new();
        let mut segsizes = BTreeMap::new();
        let seg = src.get_seg_tag();
        let startblk;
        let size;
        if seg.rel.is_blocky() {
            size = src.get_seg_size(lsn)?;
            segsizes.insert(lsn, size);
            startblk = seg.segno * RELISH_SEG_SIZE;
        } else {
            size = 1;
            startblk = 0;
        }
        for blknum in startblk..(startblk + size) {
            let img = timeline.materialize_page(seg, blknum, lsn, src)?;
            let pv = PageVersion {
                page_image: Some(img),
                record: None,
            };
            page_versions.insert((blknum, lsn), pv);
        }
        Ok(InMemoryLayer {
            conf,
            timelineid,
            tenantid,
            seg: src.get_seg_tag(),
            start_lsn: lsn,
            inner: Mutex::new(InMemoryLayerInner {
                drop_lsn: None,
                page_versions: page_versions,
                segsizes: segsizes,
            }),
        })
    }
    ///
    /// Write the this in-memory layer to disk, as a snapshot layer.
    ///
    /// The cutoff point for the layer that's written to disk is 'end_lsn'.
    ///
    /// Returns new layers that replace this one. Always returns a
    /// SnapshotLayer containing the page versions that were written to disk,
    /// but if there were page versions newer than 'end_lsn', also return a new
    /// in-memory layer containing those page versions. The caller replaces
    /// this layer with the returned layers in the layer map.
    ///
    pub fn freeze(
        &self,
        cutoff_lsn: Lsn,
        // This is needed just to call materialize_page()
        timeline: &LayeredTimeline,
    ) -> Result<(Option<Arc<SnapshotLayer>>, Option<Arc<InMemoryLayer>>)> {
        info!(
            "freezing in memory layer for {} on timeline {} at {}",
            self.seg, self.timelineid, cutoff_lsn
        );
        let inner = self.inner.lock().unwrap();
        // Normally, use the cutoff LSN as the end of the frozen layer.
        // But if the relation was dropped, we know that there are no
        // more changes coming in for it, and in particular we know that
        // there are no changes "in flight" for the LSN anymore, so we use
        // the drop LSN instead. The drop-LSN could be ahead of the
        // caller-specified LSN!
        let dropped = inner.drop_lsn.is_some();
        let end_lsn = if dropped {
            inner.drop_lsn.unwrap()
        } else {
            cutoff_lsn
        };
        // Divide all the page versions into old and new at the 'end_lsn' cutoff point.
        let mut before_page_versions;
        let mut before_segsizes;
        let mut after_page_versions;
        let mut after_segsizes;
        if !dropped {
            before_segsizes = BTreeMap::new();
            after_segsizes = BTreeMap::new();
            for (lsn, size) in inner.segsizes.iter() {
                if *lsn > end_lsn {
                    after_segsizes.insert(*lsn, *size);
                } else {
                    before_segsizes.insert(*lsn, *size);
                }
            }
            before_page_versions = BTreeMap::new();
            after_page_versions = BTreeMap::new();
            for ((blknum, lsn), pv) in inner.page_versions.iter() {
                if *lsn > end_lsn {
                    after_page_versions.insert((*blknum, *lsn), pv.clone());
                } else {
                    before_page_versions.insert((*blknum, *lsn), pv.clone());
                }
            }
        } else {
            before_page_versions = inner.page_versions.clone();
            before_segsizes = inner.segsizes.clone();
            after_segsizes = BTreeMap::new();
            after_page_versions = BTreeMap::new();
        }
        // we can release the lock now.
        drop(inner);
        // Write the page versions before the cutoff to disk.
        let snapfile = SnapshotLayer::create(
            self.conf,
            self.timelineid,
            self.tenantid,
            self.seg,
            self.start_lsn,
            end_lsn,
            dropped,
            before_page_versions,
            before_segsizes,
        )?;
        // If there were any "new" page versions, initialize a new in-memory layer to hold
        // them
        let new_open = if !after_segsizes.is_empty() || !after_page_versions.is_empty() {
            info!("created new in-mem layer for {} {}-", self.seg, end_lsn);
            let new_open = Self::copy_snapshot(
                self.conf,
                timeline,
                &snapfile,
                self.timelineid,
                self.tenantid,
                end_lsn,
            )?;
            let mut new_inner = new_open.inner.lock().unwrap();
            new_inner.page_versions.append(&mut after_page_versions);
            new_inner.segsizes.append(&mut after_segsizes);
            drop(new_inner);
            Some(Arc::new(new_open))
        } else {
            None
        };
        let new_historic = Some(Arc::new(snapfile));
        Ok((new_historic, new_open))
    }
    /// debugging function to print out the contents of the layer
    #[allow(unused)]
    pub fn dump(&self) -> String {
        let mut result = format!(
            "----- inmemory layer for {} {}-> ----\n",
            self.seg, self.start_lsn
        );
        let inner = self.inner.lock().unwrap();
        for (k, v) in inner.segsizes.iter() {
            result += &format!("{}: {}\n", k, v);
        }
        for (k, v) in inner.page_versions.iter() {
            result += &format!(
                "blk {} at {}: {}/{}\n",
                k.0,
                k.1,
                v.page_image.is_some(),
                v.record.is_some()
            );
        }
        result
    }
 }
@@ -1,281 +0,0 @@
 //!
 //! The layer map tracks what layers exist for all the relations in a timeline.
 //!
 //! When the timeline is first accessed, the server lists of all snapshot files
 //! in the timelines/<timelineid> directory, and populates this map with
 //! SnapshotLayers corresponding to each file. When new WAL is received,
 //! we create InMemoryLayers to hold the incoming records. Now and then,
 //! in the checkpoint() function, the in-memory layers are frozen, forming
 //! new snapshot layers and corresponding files are written to disk.
 //!
 use crate::layered_repository::storage_layer::{Layer, SegmentTag};
 use crate::layered_repository::{InMemoryLayer, SnapshotLayer};
 use crate::relish::*;
 use anyhow::Result;
 use log::*;
 use std::collections::HashSet;
 use std::collections::{BTreeMap, HashMap};
 use std::ops::Bound::Included;
 use std::sync::Arc;
 use zenith_utils::lsn::Lsn;
 ///
 /// LayerMap tracks what layers exist or a timeline. The last layer that is
 /// open for writes is always an InMemoryLayer, and is tracked separately
 /// because there can be only one for each segment. The older layers,
 /// stored on disk, are kept in a BTreeMap keyed by the layer's start LSN.
 ///
 pub struct LayerMap {
    segs: HashMap<SegmentTag, SegEntry>,
 }
 struct SegEntry {
    pub open: Option<Arc<InMemoryLayer>>,
    pub historic: BTreeMap<Lsn, Arc<SnapshotLayer>>,
 }
 impl LayerMap {
    ///
    /// Look up using the given segment tag and LSN. This differs from a plain
    /// key-value lookup in that if there is any layer that covers the
    /// given LSN, or precedes the given LSN, it is returned. In other words,
    /// you don't need to know the exact start LSN of the layer.
    ///
    pub fn get(&self, tag: &SegmentTag, lsn: Lsn) -> Option<Arc<dyn Layer>> {
        let segentry = self.segs.get(tag)?;
        if let Some(open) = &segentry.open {
            if open.get_start_lsn() <= lsn {
                let x: Arc<dyn Layer> = Arc::clone(&open) as _;
                return Some(x);
            }
        }
        if let Some((_k, v)) = segentry
            .historic
            .range((Included(Lsn(0)), Included(lsn)))
            .next_back()
        {
            let x: Arc<dyn Layer> = Arc::clone(&v) as _;
            Some(x)
        } else {
            None
        }
    }
    ///
    /// Get the open layer for given segment for writing. Or None if no open
    /// layer exists.
    ///
    pub fn get_open(&self, tag: &SegmentTag) -> Option<Arc<InMemoryLayer>> {
        let segentry = self.segs.get(tag)?;
        if let Some(open) = &segentry.open {
            Some(Arc::clone(open))
        } else {
            None
        }
    }
    ///
    /// Insert an open in-memory layer
    ///
    pub fn insert_open(&mut self, layer: Arc<InMemoryLayer>) {
        let tag = layer.get_seg_tag();
        if let Some(segentry) = self.segs.get_mut(&tag) {
            if let Some(_old) = &segentry.open {
                // FIXME: shouldn't exist, but check
            }
            segentry.open = Some(layer);
        } else {
            let segentry = SegEntry {
                open: Some(layer),
                historic: BTreeMap::new(),
            };
            self.segs.insert(tag, segentry);
        }
    }
    ///
    /// Insert an on-disk layer
    ///
    pub fn insert_historic(&mut self, layer: Arc<SnapshotLayer>) {
        let tag = layer.get_seg_tag();
        let start_lsn = layer.get_start_lsn();
        if let Some(segentry) = self.segs.get_mut(&tag) {
            segentry.historic.insert(start_lsn, layer);
        } else {
            let mut historic = BTreeMap::new();
            historic.insert(start_lsn, layer);
            let segentry = SegEntry {
                open: None,
                historic,
            };
            self.segs.insert(tag, segentry);
        }
    }
    ///
    /// Remove an on-disk layer from the map.
    ///
    /// This should be called when the corresponding file on disk has been deleted.
    ///
    pub fn remove_historic(&mut self, layer: &SnapshotLayer) {
        let tag = layer.get_seg_tag();
        let start_lsn = layer.get_start_lsn();
        if let Some(segentry) = self.segs.get_mut(&tag) {
            segentry.historic.remove(&start_lsn);
        }
    }
    pub fn list_rels(&self, spcnode: u32, dbnode: u32) -> Result<HashSet<RelTag>> {
        let mut rels: HashSet<RelTag> = HashSet::new();
        for (seg, _entry) in self.segs.iter() {
            if let RelishTag::Relation(reltag) = seg.rel {
                // FIXME: skip if it was dropped before the requested LSN. But there is no
                // LSN argument
                if (spcnode == 0 || reltag.spcnode == spcnode)
                    && (dbnode == 0 || reltag.dbnode == dbnode)
                {
                    rels.insert(reltag);
                }
            }
        }
        Ok(rels)
    }
    pub fn list_nonrels(&self, _lsn: Lsn) -> Result<HashSet<RelishTag>> {
        let mut rels: HashSet<RelishTag> = HashSet::new();
        // Scan the timeline directory to get all rels in this timeline.
        for (seg, _entry) in self.segs.iter() {
            // FIXME: skip if it was dropped before the requested LSN.
            if let RelishTag::Relation(_) = seg.rel {
            } else {
                rels.insert(seg.rel);
            }
        }
        Ok(rels)
    }
    /// Is there a newer layer for given segment?
    pub fn newer_layer_exists(&self, seg: SegmentTag, lsn: Lsn) -> bool {
        if let Some(segentry) = self.segs.get(&seg) {
            if let Some(_open) = &segentry.open {
                return true;
            }
            for (newer_lsn, layer) in segentry
                .historic
                .range((Included(lsn), Included(Lsn(u64::MAX))))
            {
                if layer.get_end_lsn() > lsn {
                    trace!(
                        "found later layer for {}, {} {}-{}",
                        seg,
                        lsn,
                        newer_lsn,
                        layer.get_end_lsn()
                    );
                    return true;
                } else {
                    trace!("found singleton layer for {}, {} {}", seg, lsn, newer_lsn);
                    continue;
                }
            }
        }
        trace!("no later layer found for {}, {}", seg, lsn);
        false
    }
    pub fn iter_open_layers(&mut self) -> OpenLayerIter {
        OpenLayerIter {
            last: None,
            segiter: self.segs.iter_mut(),
        }
    }
    pub fn iter_historic_layers(&self) -> HistoricLayerIter {
        HistoricLayerIter {
            segiter: self.segs.iter(),
            iter: None,
        }
    }
 }
 impl Default for LayerMap {
    fn default() -> Self {
        LayerMap {
            segs: HashMap::new(),
        }
    }
 }
 pub struct OpenLayerIter<'a> {
    last: Option<&'a mut SegEntry>,
    segiter: std::collections::hash_map::IterMut<'a, SegmentTag, SegEntry>,
 }
 impl<'a> OpenLayerIter<'a> {
    pub fn replace(&mut self, replacement: Option<Arc<InMemoryLayer>>) {
        let segentry = self.last.as_mut().unwrap();
        segentry.open = replacement;
    }
    pub fn insert_historic(&mut self, new_layer: Arc<SnapshotLayer>) {
        let start_lsn = new_layer.get_start_lsn();
        let segentry = self.last.as_mut().unwrap();
        segentry.historic.insert(start_lsn, new_layer);
    }
 }
 impl<'a> Iterator for OpenLayerIter<'a> {
    type Item = Arc<InMemoryLayer>;
    fn next(&mut self) -> std::option::Option<<Self as std::iter::Iterator>::Item> {
        while let Some((_seg, entry)) = self.segiter.next() {
            if let Some(open) = &entry.open {
                let op = Arc::clone(&open);
                self.last = Some(entry);
                return Some(op);
            }
        }
        self.last = None;
        None
    }
 }
 pub struct HistoricLayerIter<'a> {
    segiter: std::collections::hash_map::Iter<'a, SegmentTag, SegEntry>,
    iter: Option<std::collections::btree_map::Iter<'a, Lsn, Arc<SnapshotLayer>>>,
 }
 impl<'a> Iterator for HistoricLayerIter<'a> {
    type Item = Arc<SnapshotLayer>;
    fn next(&mut self) -> std::option::Option<<Self as std::iter::Iterator>::Item> {
        loop {
            if let Some(x) = &mut self.iter {
                if let Some(x) = x.next() {
                    return Some(Arc::clone(&*x.1));
                }
            }
            if let Some(seg) = self.segiter.next() {
                self.iter = Some(seg.1.historic.iter());
                continue;
            } else {
                return None;
            }
        }
    }
 }
@@ -1,547 +0,0 @@
 //!
 //! A SnapshotLayer represents one snapshot file on disk. One file holds all page
 //! version and size information of one relation, in a range of LSN.
 //! The name "snapshot file" is a bit of a misnomer because a snapshot file doesn't
 //! contain a snapshot at a specific LSN, but rather all the page versions in a range
 //! of LSNs.
 //!
 //! Currently, a snapshot file contains full information needed to reconstruct any
 //! page version in the LSN range, without consulting any other snapshot files. When
 //! a new snapshot file is created for writing, the full contents of relation are
 //! materialized as it is at the beginning of the LSN range. That can be very expensive,
 //! we should find a way to store differential files. But this keeps the read-side
 //! of things simple. You can find the correct snapshot file based on RelishTag and
 //! timeline+LSN, and once you've located it, you have all the data you need to in that
 //! file.
 //!
 //! When a snapshot file needs to be accessed, we slurp the whole file into memory, into
 //! the SnapshotLayer struct. See load() and unload() functions.
 //!
 //! On disk, the snapshot files are stored in timelines/<timelineid> directory.
 //! Currently, there are no subdirectories, and each snapshot file is named like this:
 //!
 //!    <spcnode>_<dbnode>_<relnode>_<forknum>_<start LSN>_<end LSN>
 //!
 //! For example:
 //!
 //!    1663_13990_2609_0_000000000169C348_000000000169C349
 //!
 //! If a relation is dropped, we add a '_DROPPED' to the end of the filename to indicate that.
 //! So the above example would become:
 //!
 //!    1663_13990_2609_0_000000000169C348_000000000169C349_DROPPED
 //!
 //! The end LSN indicates when it was dropped in that case, we don't store it in the
 //! file contents in any way.
 //!
 //! A snapshot file is constructed using the 'bookfile' crate. Each file consists of two
 //! parts: the page versions and the relation sizes. They are stored as separate chapters.
 //!
 use crate::layered_repository::storage_layer::{
    Layer, PageReconstructData, PageVersion, SegmentTag,
 };
 use crate::relish::*;
 use crate::PageServerConf;
 use crate::{ZTenantId, ZTimelineId};
 use anyhow::{bail, Result};
 use log::*;
 use std::collections::BTreeMap;
 use std::fmt;
 use std::fs;
 use std::fs::File;
 use std::io::Write;
 use std::ops::Bound::Included;
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex, MutexGuard};
 use bookfile::{Book, BookWriter};
 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;
 // Magic constant to identify a Zenith snapshot file
 static SNAPSHOT_FILE_MAGIC: u32 = 0x5A616E01;
 static PAGE_VERSIONS_CHAPTER: u64 = 1;
 static REL_SIZES_CHAPTER: u64 = 2;
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
 struct SnapshotFileName {
    seg: SegmentTag,
    start_lsn: Lsn,
    end_lsn: Lsn,
    dropped: bool,
 }
 impl SnapshotFileName {
    fn from_str(fname: &str) -> Option<Self> {
        // Split the filename into parts
        //
        //    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>
        //
        // or if it was dropped:
        //
        //    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>_DROPPED
        //
        let rel;
        let mut parts;
        if let Some(rest) = fname.strip_prefix("rel_") {
            parts = rest.split('_');
            rel = RelishTag::Relation(RelTag {
                spcnode: parts.next()?.parse::<u32>().ok()?,
                dbnode: parts.next()?.parse::<u32>().ok()?,
                relnode: parts.next()?.parse::<u32>().ok()?,
                forknum: parts.next()?.parse::<u8>().ok()?,
            });
        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
            parts = rest.split('_');
            rel = RelishTag::Slru {
                slru: SlruKind::Clog,
                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
            };
        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
            parts = rest.split('_');
            rel = RelishTag::Slru {
                slru: SlruKind::MultiXactMembers,
                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
            };
        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
            parts = rest.split('_');
            rel = RelishTag::Slru {
                slru: SlruKind::MultiXactOffsets,
                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
            };
        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
            parts = rest.split('_');
            rel = RelishTag::FileNodeMap {
                spcnode: parts.next()?.parse::<u32>().ok()?,
                dbnode: parts.next()?.parse::<u32>().ok()?,
            };
        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
            parts = rest.split('_');
            rel = RelishTag::TwoPhase {
                xid: parts.next()?.parse::<u32>().ok()?,
            };
        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
            parts = rest.split('_');
            rel = RelishTag::Checkpoint;
        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
            parts = rest.split('_');
            rel = RelishTag::ControlFile;
        } else {
            return None;
        }
        let segno = parts.next()?.parse::<u32>().ok()?;
        let seg = SegmentTag { rel, segno };
        let start_lsn = Lsn::from_hex(parts.next()?).ok()?;
        let end_lsn = Lsn::from_hex(parts.next()?).ok()?;
        let mut dropped = false;
        if let Some(suffix) = parts.next() {
            if suffix == "DROPPED" {
                dropped = true;
            } else {
                warn!("unrecognized filename in timeline dir: {}", fname);
                return None;
            }
        }
        if parts.next().is_some() {
            warn!("unrecognized filename in timeline dir: {}", fname);
            return None;
        }
        Some(SnapshotFileName {
            seg,
            start_lsn,
            end_lsn,
            dropped,
        })
    }
    fn to_string(&self) -> String {
        let basename = match self.seg.rel {
            RelishTag::Relation(reltag) => format!(
                "rel_{}_{}_{}_{}",
                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
            ),
            RelishTag::Slru {
                slru: SlruKind::Clog,
                segno,
            } => format!("pg_xact_{:04X}", segno),
            RelishTag::Slru {
                slru: SlruKind::MultiXactMembers,
                segno,
            } => format!("pg_multixact_members_{:04X}", segno),
            RelishTag::Slru {
                slru: SlruKind::MultiXactOffsets,
                segno,
            } => format!("pg_multixact_offsets_{:04X}", segno),
            RelishTag::FileNodeMap { spcnode, dbnode } => {
                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
            }
            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
            RelishTag::Checkpoint => format!("pg_control_checkpoint"),
            RelishTag::ControlFile => format!("pg_control"),
        };
        format!(
            "{}_{}_{:016X}_{:016X}{}",
            basename,
            self.seg.segno,
            u64::from(self.start_lsn),
            u64::from(self.end_lsn),
            if self.dropped { "_DROPPED" } else { "" }
        )
    }
 }
 impl fmt::Display for SnapshotFileName {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}", self.to_string())
    }
 }
 ///
 /// SnapshotLayer is the in-memory data structure associated with an
 /// on-disk snapshot file.  We keep a SnapshotLayer in memory for each
 /// file, in the LayerMap. If a layer is in "loaded" state, we have a
 /// copy of the file in memory, in 'inner'. Otherwise the struct is
 /// just a placeholder for a file that exists on disk, and it needs to
 /// be loaded before using it in queries.
 ///
 pub struct SnapshotLayer {
    conf: &'static PageServerConf,
    pub tenantid: ZTenantId,
    pub timelineid: ZTimelineId,
    pub seg: SegmentTag,
    //
    // This entry contains all the changes from 'start_lsn' to 'end_lsn'. The
    // start is inclusive, and end is exclusive.
    pub start_lsn: Lsn,
    pub end_lsn: Lsn,
    dropped: bool,
    inner: Mutex<SnapshotLayerInner>,
 }
 pub struct SnapshotLayerInner {
    /// If false, the 'page_versions' and 'relsizes' have not been
    /// loaded into memory yet.
    loaded: bool,
    /// All versions of all pages in the file are are kept here.
    /// Indexed by block number and LSN.
    page_versions: BTreeMap<(u32, Lsn), PageVersion>,
    /// `relsizes` tracks the size of the relation at different points in time.
    relsizes: BTreeMap<Lsn, u32>,
 }
 impl Layer for SnapshotLayer {
    fn get_timeline_id(&self) -> ZTimelineId {
        return self.timelineid;
    }
    fn get_seg_tag(&self) -> SegmentTag {
        return self.seg;
    }
    fn is_dropped(&self) -> bool {
        return self.dropped;
    }
    fn get_start_lsn(&self) -> Lsn {
        return self.start_lsn;
    }
    fn get_end_lsn(&self) -> Lsn {
        return self.end_lsn;
    }
    /// Look up given page in the cache.
    fn get_page_reconstruct_data(
        &self,
        blknum: u32,
        lsn: Lsn,
        reconstruct_data: &mut PageReconstructData,
    ) -> Result<Option<Lsn>> {
        // Scan the BTreeMap backwards, starting from the given entry.
        let mut need_base_image_lsn: Option<Lsn> = Some(lsn);
        {
            let inner = self.load()?;
            let minkey = (blknum, Lsn(0));
            let maxkey = (blknum, lsn);
            let mut iter = inner
                .page_versions
                .range((Included(&minkey), Included(&maxkey)));
            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
                if let Some(img) = &entry.page_image {
                    reconstruct_data.page_img = Some(img.clone());
                    need_base_image_lsn = None;
                    break;
                } else if let Some(rec) = &entry.record {
                    reconstruct_data.records.push(rec.clone());
                    if rec.will_init {
                        // This WAL record initializes the page, so no need to go further back
                        need_base_image_lsn = None;
                        break;
                    } else {
                        need_base_image_lsn = Some(*entry_lsn);
                    }
                } else {
                    // No base image, and no WAL record. Huh?
                    bail!("no page image or WAL record for requested page");
                }
            }
            // release lock on 'inner'
        }
        Ok(need_base_image_lsn)
    }
    /// Get size of the relation at given LSN
    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
        // Scan the BTreeMap backwards, starting from the given entry.
        let inner = self.load()?;
        let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn)));
        if let Some((_entry_lsn, entry)) = iter.next_back() {
            let result = *entry;
            drop(inner);
            trace!("get_seg_size: {} at {} -> {}", self.seg, lsn, result);
            Ok(result)
        } else {
            error!(
                "No size found for {} at {} in snapshot layer {} {}-{}",
                self.seg, lsn, self.seg, self.start_lsn, self.end_lsn
            );
            bail!(
                "No size found for {} at {} in snapshot layer",
                self.seg,
                lsn
            );
        }
    }
    /// Does this segment exist at given LSN?
    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
        // Is the requested LSN after the rel was dropped?
        if self.dropped && lsn >= self.end_lsn {
            return Ok(false);
        }
        // Otherwise, it exists.
        Ok(true)
    }
 }
 impl SnapshotLayer {
    fn path(&self) -> PathBuf {
        Self::path_for(
            self.conf,
            self.timelineid,
            self.tenantid,
            &SnapshotFileName {
                seg: self.seg,
                start_lsn: self.start_lsn,
                end_lsn: self.end_lsn,
                dropped: self.dropped,
            },
        )
    }
    fn path_for(
        conf: &'static PageServerConf,
        timelineid: ZTimelineId,
        tenantid: ZTenantId,
        fname: &SnapshotFileName,
    ) -> PathBuf {
        conf.timeline_path(&timelineid, &tenantid)
            .join(fname.to_string())
    }
    /// Create a new snapshot file, using the given btreemaps containing the page versions and
    /// relsizes.
    ///
    /// This is used to write the in-memory layer to disk. The in-memory layer uses the same
    /// data structure with two btreemaps as we do, so passing the btreemaps is currently
    /// expedient.
    pub fn create(
        conf: &'static PageServerConf,
        timelineid: ZTimelineId,
        tenantid: ZTenantId,
        seg: SegmentTag,
        start_lsn: Lsn,
        end_lsn: Lsn,
        dropped: bool,
        page_versions: BTreeMap<(u32, Lsn), PageVersion>,
        relsizes: BTreeMap<Lsn, u32>,
    ) -> Result<SnapshotLayer> {
        let snapfile = SnapshotLayer {
            conf: conf,
            timelineid: timelineid,
            tenantid: tenantid,
            seg: seg,
            start_lsn: start_lsn,
            end_lsn,
            dropped,
            inner: Mutex::new(SnapshotLayerInner {
                loaded: true,
                page_versions: page_versions,
                relsizes: relsizes,
            }),
        };
        let inner = snapfile.inner.lock().unwrap();
        // Write the in-memory btreemaps into a file
        let path = snapfile.path();
        // Note: This overwrites any existing file. There shouldn't be any.
        // FIXME: throw an error instead?
        let file = File::create(&path)?;
        let book = BookWriter::new(file, SNAPSHOT_FILE_MAGIC)?;
        // Write out page versions
        let mut chapter = book.new_chapter(PAGE_VERSIONS_CHAPTER);
        let buf = BTreeMap::ser(&inner.page_versions)?;
        chapter.write_all(&buf)?;
        let book = chapter.close()?;
        // and relsizes to separate chapter
        let mut chapter = book.new_chapter(REL_SIZES_CHAPTER);
        let buf = BTreeMap::ser(&inner.relsizes)?;
        chapter.write_all(&buf)?;
        let book = chapter.close()?;
        book.close()?;
        trace!("saved {}", &path.display());
        drop(inner);
        Ok(snapfile)
    }
    ///
    /// Load the contents of the file into memory
    ///
    fn load(&self) -> Result<MutexGuard<SnapshotLayerInner>> {
        // quick exit if already loaded
        let mut inner = self.inner.lock().unwrap();
        if inner.loaded {
            return Ok(inner);
        }
        let path = Self::path_for(
            self.conf,
            self.timelineid,
            self.tenantid,
            &SnapshotFileName {
                seg: self.seg,
                start_lsn: self.start_lsn,
                end_lsn: self.end_lsn,
                dropped: self.dropped,
            },
        );
        let file = File::open(&path)?;
        let book = Book::new(file)?;
        let chapter = book.read_chapter(PAGE_VERSIONS_CHAPTER)?;
        let page_versions = BTreeMap::des(&chapter)?;
        let chapter = book.read_chapter(REL_SIZES_CHAPTER)?;
        let relsizes = BTreeMap::des(&chapter)?;
        debug!("loaded from {}", &path.display());
        *inner = SnapshotLayerInner {
            loaded: true,
            page_versions,
            relsizes,
        };
        Ok(inner)
    }
    /// Create SnapshotLayers representing all files on disk
    ///
    // TODO: returning an Iterator would be more idiomatic
    pub fn list_snapshot_files(
        conf: &'static PageServerConf,
        timelineid: ZTimelineId,
        tenantid: ZTenantId,
    ) -> Result<Vec<Arc<SnapshotLayer>>> {
        let path = conf.timeline_path(&timelineid, &tenantid);
        let mut snapfiles: Vec<Arc<SnapshotLayer>> = Vec::new();
        for direntry in fs::read_dir(path)? {
            let fname = direntry?.file_name();
            let fname = fname.to_str().unwrap();
            if let Some(snapfilename) = SnapshotFileName::from_str(fname) {
                let snapfile = SnapshotLayer {
                    conf,
                    timelineid,
                    tenantid,
                    seg: snapfilename.seg,
                    start_lsn: snapfilename.start_lsn,
                    end_lsn: snapfilename.end_lsn,
                    dropped: snapfilename.dropped,
                    inner: Mutex::new(SnapshotLayerInner {
                        loaded: false,
                        page_versions: BTreeMap::new(),
                        relsizes: BTreeMap::new(),
                    }),
                };
                snapfiles.push(Arc::new(snapfile));
            }
        }
        return Ok(snapfiles);
    }
    pub fn delete(&self) -> Result<()> {
        // delete underlying file
        fs::remove_file(self.path())?;
        Ok(())
    }
    ///
    /// Release most of the memory used by this layer. If it's accessed again later,
    /// it will need to be loaded back.
    ///
    pub fn unload(&self) -> Result<()> {
        let mut inner = self.inner.lock().unwrap();
        inner.page_versions = BTreeMap::new();
        inner.relsizes = BTreeMap::new();
        inner.loaded = false;
        Ok(())
    }
    /// debugging function to print out the contents of the layer
    #[allow(unused)]
    pub fn dump(&self) -> String {
        let mut result = format!(
            "----- snapshot layer for {} {}-{} ----\n",
            self.seg, self.start_lsn, self.end_lsn
        );
        let inner = self.inner.lock().unwrap();
        for (k, v) in inner.relsizes.iter() {
            result += &format!("{}: {}\n", k, v);
        }
        //for (k, v) in inner.page_versions.iter() {
        //    result += &format!("blk {} at {}: {}/{}\n", k.0, k.1, v.page_image.is_some(), v.record.is_some());
        //}
        result
    }
 }
@@ -1,128 +0,0 @@
 //!
 //! Common traits and structs for layers
 //!
 use crate::relish::RelishTag;
 use crate::repository::WALRecord;
 use crate::ZTimelineId;
 use anyhow::Result;
 use bytes::Bytes;
 use serde::{Deserialize, Serialize};
 use std::fmt;
 use zenith_utils::lsn::Lsn;
 // Size of one segment in pages (10 MB)
 pub const RELISH_SEG_SIZE: u32 = 10 * 1024 * 1024 / 8192;
 ///
 /// Each relish stored in the repository is divided into fixed-sized "segments",
 /// with 10 MB of key-space, or 1280 8k pages each.
 ///
 #[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy)]
 pub struct SegmentTag {
    pub rel: RelishTag,
    pub segno: u32,
 }
 impl fmt::Display for SegmentTag {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}.{}", self.rel, self.segno)
    }
 }
 impl SegmentTag {
    pub const fn from_blknum(rel: RelishTag, blknum: u32) -> SegmentTag {
        SegmentTag {
            rel,
            segno: blknum / RELISH_SEG_SIZE,
        }
    }
    pub fn blknum_in_seg(&self, blknum: u32) -> bool {
        blknum / RELISH_SEG_SIZE == self.segno
    }
 }
 ///
 /// Represents a version of a page at a specific LSN. The LSN is the key of the
 /// entry in the 'page_versions' hash, it is not duplicated here.
 ///
 /// A page version can be stored as a full page image, or as WAL record that needs
 /// to be applied over the previous page version to reconstruct this version.
 ///
 /// It's also possible to have both a WAL record and a page image in the same
 /// PageVersion. That happens if page version is originally stored as a WAL record
 /// but it is later reconstructed by a GetPage@LSN request by performing WAL
 /// redo. The get_page_at_lsn() code will store the reconstructed pag image next to
 /// the WAL record in that case. TODO: That's pretty accidental, not the result
 /// of any grand design. If we want to keep reconstructed page versions around, we
 /// probably should have a separate buffer cache so that we could control the
 /// replacement policy globally. Or if we keep a reconstructed page image, we
 /// could throw away the WAL record.
 ///
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct PageVersion {
    /// an 8kb page image
    pub page_image: Option<Bytes>,
    /// WAL record to get from previous page version to this one.
    pub record: Option<WALRecord>,
 }
 ///
 /// Data needed to reconstruct a page version
 ///
 /// 'page_img' is the old base image of the page to start the WAL replay with.
 /// It can be None, if the first WAL record initializes the page (will_init)
 /// 'records' contains the records to apply over the base image.
 ///
 pub struct PageReconstructData {
    pub records: Vec<WALRecord>,
    pub page_img: Option<Bytes>,
 }
 ///
 /// A Layer holds all page versions for one segment of a relish, in a range of LSNs.
 /// There are two kinds of layers, in-memory and snapshot layers. In-memory
 /// layers are used to ingest incoming WAL, and provide fast access
 /// to the recent page versions. Snaphot layers are stored on disk, and
 /// are immutable. This trait presents the common functionality of
 /// in-memory and snapshot layers.
 ///
 /// Each layer contains a full snapshot of the segment at the start
 /// LSN. In addition to that, it contains WAL (or more page images)
 /// needed to recontruct any page version up to the end LSN.
 ///
 pub trait Layer: Send + Sync {
    // These functions identify the relish segment and the LSN range
    // that this Layer holds.
    fn get_timeline_id(&self) -> ZTimelineId;
    fn get_seg_tag(&self) -> SegmentTag;
    fn get_start_lsn(&self) -> Lsn;
    fn get_end_lsn(&self) -> Lsn;
    fn is_dropped(&self) -> bool;
    ///
    /// Return data needed to reconstruct given page at LSN.
    ///
    /// It is up to the caller to collect more data from previous layer and
    /// perform WAL redo, if necessary.
    ///
    /// If returns Some, the returned data is not complete. The caller needs
    /// to continue with the returned 'lsn'.
    ///
    /// Note that the 'blknum' is the offset of the page from the beginning
    /// of the *relish*, not the beginning of the segment. The requested
    /// 'blknum' must be covered by this segment.
    fn get_page_reconstruct_data(
        &self,
        blknum: u32,
        lsn: Lsn,
        reconstruct_data: &mut PageReconstructData,
    ) -> Result<Option<Lsn>>;
    // Functions that correspond to the Timeline trait functions.
    fn get_seg_size(&self, lsn: Lsn) -> Result<u32>;
    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool>;
 }
@@ -1,46 +1,31 @@
-use zenith_utils::postgres_backend::AuthType;
+use serde::{Deserialize, Serialize};
 use zenith_utils::zid::{ZTenantId, ZTimelineId};
 use std::fmt;
 use std::net::SocketAddr;
 use std::path::PathBuf;
 use std::str::FromStr;
 use std::time::Duration;
 use lazy_static::lazy_static;
 use zenith_metrics::{register_int_gauge_vec, IntGaugeVec};
 pub mod basebackup;
 pub mod branches;
 pub mod layered_repository;
 pub mod logger;
 pub mod object_key;
 pub mod object_repository;
 pub mod object_store;
 pub mod page_cache;
 pub mod page_service;
 pub mod relish;
 pub mod repository;
 pub mod restore_local_repo;
-pub mod rocksdb_storage;
+pub mod tui;
 pub mod tui_event;
 mod tui_logger;
 pub mod waldecoder;
 pub mod walreceiver;
 pub mod walredo;
 lazy_static! {
    static ref LIVE_CONNECTIONS_COUNT: IntGaugeVec = register_int_gauge_vec!(
        "pageserver_live_connections_count",
        "Number of live network connections",
        &["pageserver_connection_kind"]
    )
    .expect("failed to define a metric");
 }
 #[derive(Debug, Clone)]
 pub struct PageServerConf {
    pub daemonize: bool,
-    pub listen_addr: String,
+    pub interactive: bool,
-    pub http_endpoint_addr: String,
+    pub listen_addr: SocketAddr,
    pub gc_horizon: u64,
    pub gc_period: Duration,
    pub superuser: String,
    // Repository directory, relative to current working directory.
    // Normally, the page server changes the current working directory
@@ -51,18 +36,6 @@ pub struct PageServerConf {
    pub workdir: PathBuf,
    pub pg_distrib_dir: PathBuf,
    pub auth_type: AuthType,
    pub auth_validation_public_key_path: Option<PathBuf>,
    pub repository_format: RepositoryFormat,
 }
 #[derive(Debug, Clone, PartialEq)]
 pub enum RepositoryFormat {
    Layered,
    RocksDb,
 }
 impl PageServerConf {
@@ -70,44 +43,24 @@ impl PageServerConf {
    // Repository paths, relative to workdir.
    //
-    fn tenants_path(&self) -> PathBuf {
+    fn tag_path(&self, name: &str) -> PathBuf {
-        self.workdir.join("tenants")
+        self.workdir.join("refs").join("tags").join(name)
    }
-    fn tenant_path(&self, tenantid: &ZTenantId) -> PathBuf {
+    fn branch_path(&self, name: &str) -> PathBuf {
-        self.tenants_path().join(tenantid.to_string())
+        self.workdir.join("refs").join("branches").join(name)
    }
-    fn tags_path(&self, tenantid: &ZTenantId) -> PathBuf {
+    fn timeline_path(&self, timelineid: ZTimelineId) -> PathBuf {
-        self.tenant_path(tenantid).join("refs").join("tags")
+        self.workdir.join("timelines").join(timelineid.to_string())
    }
-    fn tag_path(&self, tag_name: &str, tenantid: &ZTenantId) -> PathBuf {
+    fn snapshots_path(&self, timelineid: ZTimelineId) -> PathBuf {
-        self.tags_path(tenantid).join(tag_name)
+        self.timeline_path(timelineid).join("snapshots")
    }
-    fn branches_path(&self, tenantid: &ZTenantId) -> PathBuf {
+    fn ancestor_path(&self, timelineid: ZTimelineId) -> PathBuf {
-        self.tenant_path(tenantid).join("refs").join("branches")
+        self.timeline_path(timelineid).join("ancestor")
    }
    fn branch_path(&self, branch_name: &str, tenantid: &ZTenantId) -> PathBuf {
        self.branches_path(tenantid).join(branch_name)
    }
    fn timelines_path(&self, tenantid: &ZTenantId) -> PathBuf {
        self.tenant_path(tenantid).join("timelines")
    }
    fn timeline_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
        self.timelines_path(tenantid).join(timelineid.to_string())
    }
    fn ancestor_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
        self.timeline_path(timelineid, tenantid).join("ancestor")
    }
    fn wal_dir_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
        self.timeline_path(timelineid, tenantid).join("wal")
    }
    //
@@ -122,3 +75,64 @@ impl PageServerConf {
        self.pg_distrib_dir.join("lib")
    }
 }
 /// Zenith Timeline ID is a 128-bit random ID.
 ///
 /// Zenith timeline IDs are different from PostgreSQL timeline
 /// IDs. They serve a similar purpose though: they differentiate
 /// between different "histories" of the same cluster.  However,
 /// PostgreSQL timeline IDs are a bit cumbersome, because they are only
 /// 32-bits wide, and they must be in ascending order in any given
 /// timeline history.  Those limitations mean that we cannot generate a
 /// new PostgreSQL timeline ID by just generating a random number. And
 /// that in turn is problematic for the "pull/push" workflow, where you
 /// have a local copy of a zenith repository, and you periodically sync
 /// the local changes with a remote server. When you work "detached"
 /// from the remote server, you cannot create a PostgreSQL timeline ID
 /// that's guaranteed to be different from all existing timelines in
 /// the remote server. For example, if two people are having a clone of
 /// the repository on their laptops, and they both create a new branch
 /// with different name. What timeline ID would they assign to their
 /// branches? If they pick the same one, and later try to push the
 /// branches to the same remote server, they will get mixed up.
 ///
 /// To avoid those issues, Zenith has its own concept of timelines that
 /// is separate from PostgreSQL timelines, and doesn't have those
 /// limitations. A zenith timeline is identified by a 128-bit ID, which
 /// is usually printed out as a hex string.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct ZTimelineId([u8; 16]);
 impl FromStr for ZTimelineId {
    type Err = hex::FromHexError;
    fn from_str(s: &str) -> Result<ZTimelineId, Self::Err> {
        let timelineid = hex::decode(s)?;
        let mut buf: [u8; 16] = [0u8; 16];
        buf.copy_from_slice(timelineid.as_slice());
        Ok(ZTimelineId(buf))
    }
 }
 impl ZTimelineId {
    pub fn from(b: [u8; 16]) -> ZTimelineId {
        ZTimelineId(b)
    }
    pub fn get_from_buf(buf: &mut dyn bytes::Buf) -> ZTimelineId {
        let mut arr = [0u8; 16];
        buf.copy_to_slice(&mut arr);
        ZTimelineId::from(arr)
    }
    pub fn as_arr(&self) -> [u8; 16] {
        self.0
    }
 }
 impl fmt::Display for ZTimelineId {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(&hex::encode(self.0))
    }
 }
@@ -1,45 +0,0 @@
 use crate::PageServerConf;
 use anyhow::{Context, Result};
 use slog::{Drain, FnValue};
 use std::fs::{File, OpenOptions};
 pub fn init_logging(
    _conf: &PageServerConf,
    log_filename: &str,
 ) -> Result<(slog_scope::GlobalLoggerGuard, File)> {
    // Don't open the same file for output multiple times;
    // the different fds could overwrite each other's output.
    let log_file = OpenOptions::new()
        .create(true)
        .append(true)
        .open(&log_filename)
        .with_context(|| format!("failed to open {:?}", &log_filename))?;
    let logger_file = log_file.try_clone().unwrap();
    let decorator = slog_term::PlainSyncDecorator::new(logger_file);
    let drain = slog_term::FullFormat::new(decorator).build();
    let drain = slog::Filter::new(drain, |record: &slog::Record| {
        if record.level().is_at_least(slog::Level::Info) {
            return true;
        }
        false
    });
    let drain = std::sync::Mutex::new(drain).fuse();
    let logger = slog::Logger::root(
        drain,
        slog::o!(
            "location" =>
                FnValue(move |record| {
                    format!("{}, {}:{}",
                            record.module(),
                            record.file(),
                            record.line()
                    )
                }
                )
        ),
    );
    Ok((slog_scope::set_global_logger(logger), log_file))
 }
@@ -1,49 +0,0 @@
 //!
 //! Common structs shared by object_repository.rs and object_store.rs.
 //!
 use crate::relish::RelishTag;
 use serde::{Deserialize, Serialize};
 use zenith_utils::zid::ZTimelineId;
 ///
 /// ObjectKey is the key type used to identify objects stored in an object
 /// repository. It is shared between object_repository.rs and object_store.rs.
 /// It is mostly opaque to ObjectStore, it just stores and retrieves objects
 /// using the key given by the caller.
 ///
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ObjectKey {
    pub timeline: ZTimelineId,
    pub tag: ObjectTag,
 }
 ///
 /// ObjectTag is a part of ObjectKey that is specific to the type of
 /// the stored object.
 ///
 /// NB: the order of the enum values is significant!  In particular,
 /// rocksdb_storage.rs assumes that TimelineMetadataTag is first
 ///
 /// Buffer is the kind of object that is accessible by the public
 /// get_page_at_lsn() / put_page_image() / put_wal_record() functions in
 /// the repository.rs interface. The rest are internal objects stored in
 /// the key-value store, to store various metadata. They're not directly
 /// accessible outside object_repository.rs
 ///
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
 pub enum ObjectTag {
    // dummy tag preceeding all other keys
    FirstTag,
    // Metadata about a timeline. Not versioned.
    TimelineMetadataTag,
    // These objects store metadata about one relish. Currently it's used
    // just to track the relish's size. It's not used for non-blocky relishes
    // at all.
    RelationMetadata(RelishTag),
    // These are the pages exposed in the public Repository/Timeline interface.
    Buffer(RelishTag, u32),
 }
@@ -1,92 +0,0 @@
 //! Low-level key-value storage abstraction.
 //!
 use crate::object_key::*;
 use crate::relish::*;
 use anyhow::Result;
 use std::collections::HashSet;
 use std::iter::Iterator;
 use zenith_utils::lsn::Lsn;
 use zenith_utils::zid::ZTimelineId;
 ///
 /// Low-level storage abstraction.
 ///
 /// All the data in the repository is stored in a key-value store. This trait
 /// abstracts the details of the key-value store.
 ///
 /// A simple key-value store would support just GET and PUT operations with
 /// a key, but the upper layer needs slightly complicated read operations
 ///
 /// The most frequently used function is 'object_versions'. It is used
 /// to look up a page version. It is LSN aware, in that the caller
 /// specifies an LSN, and the function returns all values for that
 /// block with the same or older LSN.
 ///
 pub trait ObjectStore: Send + Sync {
    ///
    /// Store a value with given key.
    ///
    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()>;
    /// Read entry with the exact given key.
    ///
    /// This is used for retrieving metadata with special key that doesn't
    /// correspond to any real relation.
    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>>;
    /// Read key greater or equal than specified
    fn get_next_key(&self, key: &ObjectKey) -> Result<Option<ObjectKey>>;
    /// Iterate through all page versions of one object.
    ///
    /// Returns all page versions in descending LSN order, along with the LSN
    /// of each page version.
    fn object_versions<'a>(
        &'a self,
        key: &ObjectKey,
        lsn: Lsn,
    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>>;
    /// Iterate through versions of all objects in a timeline.
    ///
    /// Returns objects in increasing key-version order.
    /// Returns all versions up to and including the specified LSN.
    fn objects<'a>(
        &'a self,
        timeline: ZTimelineId,
        lsn: Lsn,
    ) -> Result<Box<dyn Iterator<Item = Result<(ObjectTag, Lsn, Vec<u8>)>> + 'a>>;
    /// Iterate through all keys with given tablespace and database ID, and LSN <= 'lsn'.
    /// Both dbnode and spcnode can be InvalidId (0) which means get all relations in tablespace/cluster
    ///
    /// This is used to implement 'create database'
    fn list_rels(
        &self,
        timelineid: ZTimelineId,
        spcnode: u32,
        dbnode: u32,
        lsn: Lsn,
    ) -> Result<HashSet<RelTag>>;
    /// Iterate through non-rel relishes
    ///
    /// This is used to prepare tarball for new node startup.
    fn list_nonrels<'a>(&'a self, timelineid: ZTimelineId, lsn: Lsn) -> Result<HashSet<RelishTag>>;
    /// Iterate through objects tags. If nonrel_only, then only non-relationa data is iterated.
    ///
    /// This is used to implement GC and preparing tarball for new node startup
    /// Returns objects in increasing key-version order.
    fn list_objects<'a>(
        &'a self,
        timelineid: ZTimelineId,
        lsn: Lsn,
    ) -> Result<Box<dyn Iterator<Item = ObjectTag> + 'a>>;
    /// Unlink object (used by GC). This mehod may actually delete object or just mark it for deletion.
    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()>;
    // Compact storage and remove versions marged for deletion
    fn compact(&self);
 }
@@ -1,91 +1,32 @@
 //! This module acts as a switchboard to access different repositories managed by this
-//! page server.
+//! page server. Currently, a Page Server can only manage one repository, so there
 //! isn't much here. If we implement multi-tenancy, this will probably be changed into
 //! a hash map, keyed by the tenant ID.
-use crate::branches;
+use crate::repository::rocksdb::RocksRepository;
 use crate::layered_repository::LayeredRepository;
 use crate::object_repository::ObjectRepository;
 use crate::repository::Repository;
 use crate::rocksdb_storage::RocksObjectStore;
 use crate::walredo::PostgresRedoManager;
-use crate::{PageServerConf, RepositoryFormat};
+use crate::PageServerConf;
 use anyhow::{anyhow, bail, Result};
 use lazy_static::lazy_static;
 use log::info;
 use std::collections::HashMap;
 use std::fs;
 use std::str::FromStr;
 use std::sync::{Arc, Mutex};
 use zenith_utils::zid::ZTenantId;
 lazy_static! {
-    pub static ref REPOSITORY: Mutex<HashMap<ZTenantId, Arc<dyn Repository>>> =
+    pub static ref REPOSITORY: Mutex<Option<Arc<dyn Repository + Send + Sync>>> = Mutex::new(None);
        Mutex::new(HashMap::new());
 }
 pub fn init(conf: &'static PageServerConf) {
    let mut m = REPOSITORY.lock().unwrap();
-    for dir_entry in fs::read_dir(conf.tenants_path()).unwrap() {
+    // Set up a WAL redo manager, for applying WAL records.
-        let tenantid =
+    let walredo_mgr = PostgresRedoManager::new(conf);
            ZTenantId::from_str(dir_entry.unwrap().file_name().to_str().unwrap()).unwrap();
-        // Set up a WAL redo manager, for applying WAL records.
+    // we have already changed current dir to the repository.
-        let walredo_mgr = PostgresRedoManager::new(conf, tenantid);
+    let repo = RocksRepository::new(conf, Arc::new(walredo_mgr));
-        // Set up an object repository, for actual data storage.
+    *m = Some(Arc::new(repo));
        let repo: Arc<dyn Repository + Sync + Send> = match conf.repository_format {
            RepositoryFormat::Layered => {
                let repo = Arc::new(LayeredRepository::new(
                    conf,
                    Arc::new(walredo_mgr),
                    tenantid,
                ));
                LayeredRepository::launch_checkpointer_thread(conf, repo.clone());
                repo
            }
            RepositoryFormat::RocksDb => {
                let obj_store = RocksObjectStore::open(conf, &tenantid).unwrap();
                Arc::new(ObjectRepository::new(
                    conf,
                    Arc::new(obj_store),
                    Arc::new(walredo_mgr),
                    tenantid,
                ))
            }
        };
        info!("initialized storage for tenant: {}", &tenantid);
        m.insert(tenantid, repo);
    }
 }
-pub fn create_repository_for_tenant(
+pub fn get_repository() -> Arc<dyn Repository + Send + Sync> {
    conf: &'static PageServerConf,
    tenantid: ZTenantId,
 ) -> Result<()> {
    let mut m = REPOSITORY.lock().unwrap();
    // First check that the tenant doesn't exist already
    if m.get(&tenantid).is_some() {
        bail!("tenant {} already exists", tenantid);
    }
    let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenantid));
    let repo = branches::create_repo(conf, tenantid, wal_redo_manager)?;
    m.insert(tenantid, repo);
    Ok(())
 }
 pub fn insert_repository_for_tenant(tenantid: ZTenantId, repo: Arc<dyn Repository>) {
    let o = &mut REPOSITORY.lock().unwrap();
    o.insert(tenantid, repo);
 }
 pub fn get_repository_for_tenant(tenantid: &ZTenantId) -> Result<Arc<dyn Repository>> {
    let o = &REPOSITORY.lock().unwrap();
-    o.get(tenantid)
+    Arc::clone(o.as_ref().unwrap())
        .map(|repo| Arc::clone(repo))
        .ok_or_else(|| anyhow!("repository not found for tenant name {}", tenantid))
 }
@@ -1,235 +0,0 @@
 //!
 //! Zenith stores PostgreSQL relations, and some other files, in the
 //! repository.  The relations (i.e. tables and indexes) take up most
 //! of the space in a typical installation, while the other files are
 //! small. We call each relation and other file that is stored in the
 //! repository a "relish". It comes from "rel"-ish, as in "kind of a
 //! rel", because it covers relations as well as other things that are
 //! not relations, but are treated similarly for the purposes of the
 //! storage layer.
 //!
 //! This source file contains the definition of the RelishTag struct,
 //! which uniquely identifies a relish.
 //!
 //! Relishes come in two flavors: blocky and non-blocky. Relations and
 //! SLRUs are blocky, that is, they are divided into 8k blocks, and
 //! the repository tracks their size. Other relishes are non-blocky:
 //! the content of the whole relish is stored as one blob. Block
 //! number must be passed as 0 for all operations on a non-blocky
 //! relish. The one "block" that you store in a non-blocky relish can
 //! have arbitrary size, but they are expected to be small, or you
 //! will have performance issues.
 //!
 //! All relishes are versioned by LSN in the repository.
 //!
 use serde::{Deserialize, Serialize};
 use std::fmt;
 use postgres_ffi::relfile_utils::forknumber_to_name;
 use postgres_ffi::{Oid, TransactionId};
 ///
 /// RelishTag identifies one relish.
 ///
 #[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
 pub enum RelishTag {
    // Relations correspond to PostgreSQL relation forks. Each
    // PostgreSQL relation fork is considered a separate relish.
    Relation(RelTag),
    // SLRUs include pg_clog, pg_multixact/members, and
    // pg_multixact/offsets. There are other SLRUs in PostgreSQL, but
    // they don't need to be stored permanently (e.g. pg_subtrans),
    // or we do not support them in zenith yet (pg_commit_ts).
    //
    // These are currently never requested directly by the compute
    // nodes, although in principle that would be possible. However,
    // when a new compute node is created, these are included in the
    // tarball that we send to the compute node to initialize the
    // PostgreSQL data directory.
    //
    // Each SLRU segment in PostgreSQL is considered a separate
    // relish. For example, pg_clog/0000, pg_clog/0001, and so forth.
    //
    // SLRU segments are divided into blocks, like relations.
    Slru { slru: SlruKind, segno: u32 },
    // Miscellaneous other files that need to be included in the
    // tarball at compute node creation. These are non-blocky, and are
    // expected to be small.
    //
    // FileNodeMap represents PostgreSQL's 'pg_filenode.map'
    // files. They are needed to map catalog table OIDs to filenode
    // numbers. Usually the mapping is done by looking up a relation's
    // 'relfilenode' field in the 'pg_class' system table, but that
    // doesn't work for 'pg_class' itself and a few other such system
    // relations. See PostgreSQL relmapper.c for details.
    //
    // Each database has a map file for its local mapped catalogs,
    // and there is a separate map file for shared catalogs.
    //
    // These files are always 512 bytes long (although we don't check
    // or care about that in the page server).
    //
    FileNodeMap { spcnode: Oid, dbnode: Oid },
    //
    // State files for prepared transactions (e.g pg_twophase/1234)
    //
    TwoPhase { xid: TransactionId },
    // The control file, stored in global/pg_control
    ControlFile,
    // Special entry that represents PostgreSQL checkpoint. It doesn't
    // correspond to to any physical file in PostgreSQL, but we use it
    // to track fields needed to restore the checkpoint data in the
    // control file, when a compute node is created.
    Checkpoint,
 }
 impl RelishTag {
    pub const fn is_blocky(&self) -> bool {
        match self {
            // These relishes work with blocks
            RelishTag::Relation(_) | RelishTag::Slru { slru: _, segno: _ } => true,
            // and these don't
            RelishTag::FileNodeMap {
                spcnode: _,
                dbnode: _,
            }
            | RelishTag::TwoPhase { xid: _ }
            | RelishTag::ControlFile
            | RelishTag::Checkpoint => false,
        }
    }
    // Physical relishes represent files and use
    // RelationSizeEntry to track existing and dropped files.
    // They can be both blocky and non-blocky.
    pub const fn is_physical(&self) -> bool {
        match self {
            // These relishes represent physical files
            RelishTag::Relation(_)
            | RelishTag::Slru { .. }
            | RelishTag::FileNodeMap { .. }
            | RelishTag::TwoPhase { .. } => true,
            // and these don't
            RelishTag::ControlFile | RelishTag::Checkpoint => false,
        }
    }
    // convenience function to check if this relish is a normal relation.
    pub const fn is_relation(&self) -> bool {
        if let RelishTag::Relation(_) = self {
            true
        } else {
            false
        }
    }
 }
 ///
 /// Relation data file segment id throughout the Postgres cluster.
 ///
 /// Every data file in Postgres is uniquely identified by 4 numbers:
 /// - relation id / node (`relnode`)
 /// - database id (`dbnode`)
 /// - tablespace id (`spcnode`), in short this is a unique id of a separate
 ///   directory to store data files.
 /// - forknumber (`forknum`) is used to split different kinds of data of the same relation
 ///   between some set of files (`relnode`, `relnode_fsm`, `relnode_vm`).
 ///
 /// In native Postgres code `RelFileNode` structure and individual `ForkNumber` value
 /// are used for the same purpose.
 /// [See more related comments here](https:///github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/relfilenode.h#L57).
 ///
 #[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy, Serialize, Deserialize)]
 pub struct RelTag {
    pub forknum: u8,
    pub spcnode: Oid,
    pub dbnode: Oid,
    pub relnode: Oid,
 }
 /// Display RelTag in the same format that's used in most PostgreSQL debug messages:
 ///
 /// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
 ///
 impl fmt::Display for RelTag {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        if let Some(forkname) = forknumber_to_name(self.forknum) {
            write!(
                f,
                "{}/{}/{}_{}",
                self.spcnode, self.dbnode, self.relnode, forkname
            )
        } else {
            write!(f, "{}/{}/{}", self.spcnode, self.dbnode, self.relnode)
        }
    }
 }
 /// Display RelTag in the same format that's used in most PostgreSQL debug messages:
 ///
 /// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
 ///
 impl fmt::Display for RelishTag {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            RelishTag::Relation(rel) => rel.fmt(f),
            RelishTag::Slru { slru, segno } => {
                // e.g. pg_clog/0001
                write!(f, "{}/{:04X}", slru.to_str(), segno)
            }
            RelishTag::FileNodeMap { spcnode, dbnode } => {
                write!(f, "relmapper file for spc {} db {}", spcnode, dbnode)
            }
            RelishTag::TwoPhase { xid } => {
                write!(f, "pg_twophase/{:08X}", xid)
            }
            RelishTag::ControlFile => {
                write!(f, "control file")
            }
            RelishTag::Checkpoint => {
                write!(f, "checkpoint")
            }
        }
    }
 }
 ///
 /// Non-relation transaction status files (clog (a.k.a. pg_xact) and
 /// pg_multixact) in Postgres are handled by SLRU (Simple LRU) buffer,
 /// hence the name.
 ///
 /// These files are global for a postgres instance.
 ///
 /// These files are divided into segments, which are divided into
 /// pages of the same BLCKSZ as used for relation files.
 ///
 #[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
 pub enum SlruKind {
    Clog,
    MultiXactMembers,
    MultiXactOffsets,
 }
 impl SlruKind {
    pub fn to_str(&self) -> &'static str {
        match self {
            Self::Clog => "pg_xact",
            Self::MultiXactMembers => "pg_multixact/members",
            Self::MultiXactOffsets => "pg_multixact/offsets",
        }
    }
 }
 pub const FIRST_NONREL_RELISH_TAG: RelishTag = RelishTag::Slru {
    slru: SlruKind::Clog,
    segno: 0,
 };
@@ -1,141 +1,55 @@
-use crate::object_key::*;
+pub mod rocksdb;
-use crate::relish::*;
+
 use crate::waldecoder::{DecodedWALRecord, Oid, TransactionId, XlCreateDatabase, XlSmgrTruncate};
 use crate::ZTimelineId;
 use anyhow::Result;
 use bytes::{Buf, BufMut, Bytes, BytesMut};
-use serde::{Deserialize, Serialize};
+use postgres_ffi::pg_constants;
-use std::collections::HashSet;
+use postgres_ffi::relfile_utils::forknumber_to_name;
-use std::iter::Iterator;
+use std::fmt;
 use std::ops::AddAssign;
 use std::sync::Arc;
 use std::time::Duration;
 use zenith_utils::lsn::Lsn;
 use zenith_utils::zid::ZTimelineId;
 ///
 /// A repository corresponds to one .zenith directory. One repository holds multiple
 /// timelines, forked off from the same initial call to 'initdb'.
-pub trait Repository: Send + Sync {
+pub trait Repository {
    /// Get Timeline handle for given zenith timeline ID.
    ///
    /// The Timeline is expected to be already "open", i.e. `get_or_restore_timeline`
    /// should've been called on it earlier already.
    fn get_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>>;
-    /// Create a new, empty timeline. The caller is responsible for loading data into it
+    /// Get Timeline handle for given zenith timeline ID.
    fn create_empty_timeline(
        &self,
        timelineid: ZTimelineId,
        start_lsn: Lsn,
    ) -> Result<Arc<dyn Timeline>>;
    /// Branch a timeline
    fn branch_timeline(&self, src: ZTimelineId, dst: ZTimelineId, start_lsn: Lsn) -> Result<()>;
    /// perform one garbage collection iteration.
    /// garbage collection is periodically performed by gc thread,
    /// but it can be explicitly requested through page server api.
    ///
-    /// 'timelineid' specifies the timeline to GC, or None for all.
+    /// Creates a new Timeline object if it's not "open" already.
-    /// `horizon` specifies delta from last lsn to preserve all object versions (pitr interval).
+    fn get_or_restore_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>>;
-    /// `compact` parameter is used to force compaction of storage.
+
-    /// some storage implementation are based on lsm tree and require periodic merge (compaction).
+    /// Create an empty timeline, without loading any data into it from possible on-disk snapshot.
-    /// usually storage implementation determines itself when compaction should be performed.
+    ///
-    /// but for gc tests it way be useful to force compaction just after completion of gc iteration
+    /// For unit tests.
-    /// to make sure that all detected garbage is removed.
+    #[cfg(test)]
-    /// so right now `compact` is set to true when gc explicitly requested through page srver api,
+    fn create_empty_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>>;
    /// and is st to false in gc threads which infinitely repeats gc iterations in loop.
    fn gc_iteration(
        &self,
        timelineid: Option<ZTimelineId>,
        horizon: u64,
        compact: bool,
    ) -> Result<GcResult>;
    // TODO get timelines?
    //fn get_stats(&self) -> RepositoryStats;
 }
-///
+pub trait Timeline {
 /// Result of performing GC
 ///
 #[derive(Default)]
 pub struct GcResult {
    // FIXME: These counters make sense for the ObjectRepository. They are not used
    // by the LayeredRepository.
    pub n_relations: u64,
    pub inspected: u64,
    pub truncated: u64,
    pub deleted: u64,
    pub prep_deleted: u64,        // RelishTag::Twophase
    pub slru_deleted: u64,        // RelishTag::Slru
    pub chkp_deleted: u64,        // RelishTag::Checkpoint
    pub control_deleted: u64,     // RelishTag::ControlFile
    pub filenodemap_deleted: u64, // RelishTag::FileNodeMap
    pub dropped: u64,
    // These are used for the LayeredRepository instead
    pub snapshot_relfiles_total: u64,
    pub snapshot_relfiles_needed_by_cutoff: u64,
    pub snapshot_relfiles_needed_by_branches: u64,
    pub snapshot_relfiles_not_updated: u64,
    pub snapshot_relfiles_removed: u64, // # of snapshot files removed because they have been made obsolete by newer snapshot files.
    pub snapshot_relfiles_dropped: u64, // # of snapshot files removed because the relation was dropped
    pub snapshot_nonrelfiles_total: u64,
    pub snapshot_nonrelfiles_needed_by_cutoff: u64,
    pub snapshot_nonrelfiles_needed_by_branches: u64,
    pub snapshot_nonrelfiles_not_updated: u64,
    pub snapshot_nonrelfiles_removed: u64, // # of snapshot files removed because they have been made obsolete by newer snapshot files.
    pub snapshot_nonrelfiles_dropped: u64, // # of snapshot files removed because the relation was dropped
    pub elapsed: Duration,
 }
 impl AddAssign for GcResult {
    fn add_assign(&mut self, other: Self) {
        self.n_relations += other.n_relations;
        self.truncated += other.truncated;
        self.deleted += other.deleted;
        self.dropped += other.dropped;
        self.snapshot_relfiles_total += other.snapshot_relfiles_total;
        self.snapshot_relfiles_needed_by_cutoff += other.snapshot_relfiles_needed_by_cutoff;
        self.snapshot_relfiles_needed_by_branches += other.snapshot_relfiles_needed_by_branches;
        self.snapshot_relfiles_not_updated += other.snapshot_relfiles_not_updated;
        self.snapshot_relfiles_removed += other.snapshot_relfiles_removed;
        self.snapshot_relfiles_dropped += other.snapshot_relfiles_dropped;
        self.snapshot_nonrelfiles_total += other.snapshot_nonrelfiles_total;
        self.snapshot_nonrelfiles_needed_by_cutoff += other.snapshot_nonrelfiles_needed_by_cutoff;
        self.snapshot_nonrelfiles_needed_by_branches +=
            other.snapshot_nonrelfiles_needed_by_branches;
        self.snapshot_nonrelfiles_not_updated += other.snapshot_nonrelfiles_not_updated;
        self.snapshot_nonrelfiles_removed += other.snapshot_nonrelfiles_removed;
        self.snapshot_nonrelfiles_dropped += other.snapshot_nonrelfiles_dropped;
        self.elapsed += other.elapsed;
    }
 }
 pub trait Timeline: Send + Sync {
    //------------------------------------------------------------------------------
    // Public GET functions
    //------------------------------------------------------------------------------
    /// Look up given page in the cache.
-    fn get_page_at_lsn(&self, tag: RelishTag, blknum: u32, lsn: Lsn) -> Result<Bytes>;
+    fn get_page_at_lsn(&self, tag: BufferTag, lsn: Lsn) -> Result<Bytes>;
-    /// Look up given page in the cache.
+    /// Get size of relation
-    fn get_page_at_lsn_nowait(&self, tag: RelishTag, blknum: u32, lsn: Lsn) -> Result<Bytes>;
+    fn get_relsize(&self, tag: RelTag, lsn: Lsn) -> Result<u32>;
    /// Get size of a relish
    fn get_relish_size(&self, tag: RelishTag, lsn: Lsn) -> Result<Option<u32>>;
    /// Does relation exist?
-    fn get_rel_exists(&self, tag: RelishTag, lsn: Lsn) -> Result<bool>;
+    fn get_relsize_exists(&self, tag: RelTag, lsn: Lsn) -> Result<bool>;
-    /// Get a list of all distinct relations in given tablespace and database.
+    /// Get page image at the particular LSN
-    fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashSet<RelTag>>;
+    fn get_page_image(&self, tag: BufferTag, lsn: Lsn) -> Result<Option<Bytes>>;
    /// Get a list of non-relational objects
    fn list_nonrels<'a>(&'a self, lsn: Lsn) -> Result<HashSet<RelishTag>>;
    //------------------------------------------------------------------------------
    // Public PUT functions, to update the repository with new page versions.
@@ -147,27 +61,94 @@ pub trait Timeline: Send + Sync {
    ///
    /// This will implicitly extend the relation, if the page is beyond the
    /// current end-of-file.
-    fn put_wal_record(&self, tag: RelishTag, blknum: u32, rec: WALRecord) -> Result<()>;
+    fn put_wal_record(&self, tag: BufferTag, rec: WALRecord);
    /// Like put_wal_record, but with ready-made image of the page.
-    fn put_page_image(
+    fn put_page_image(&self, tag: BufferTag, lsn: Lsn, img: Bytes);
        &self,
        tag: RelishTag,
        blknum: u32,
        lsn: Lsn,
        img: Bytes,
        update_meta: bool,
    ) -> Result<()>;
    /// Truncate relation
-    fn put_truncation(&self, rel: RelishTag, lsn: Lsn, nblocks: u32) -> Result<()>;
+    fn put_truncation(&self, rel: RelTag, lsn: Lsn, nblocks: u32) -> Result<()>;
-    /// Unlink relish.
+    /// Create a new database from a template database
-    /// This method is used for marking dropped relations and truncated SLRU segments
+    ///
-    fn put_unlink(&self, tag: RelishTag, lsn: Lsn) -> Result<()>;
+    /// In PostgreSQL, CREATE DATABASE works by scanning the data directory and
    /// copying all relation files from the template database. This is the equivalent
    /// of that.
    fn put_create_database(
        &self,
        lsn: Lsn,
        db_id: Oid,
        tablespace_id: Oid,
        src_db_id: Oid,
        src_tablespace_id: Oid,
    ) -> Result<()>;
-    /// Put raw data
+    ///
-    fn put_raw_data(&self, tag: ObjectTag, lsn: Lsn, data: &[u8]) -> Result<()>;
+    /// Helper function to parse a WAL record and call the above functions for all the
    /// relations/pages that the record affects.
    ///
    fn save_decoded_record(
        &self,
        decoded: DecodedWALRecord,
        recdata: Bytes,
        lsn: Lsn,
    ) -> Result<()> {
        // Figure out which blocks the record applies to, and "put" a separate copy
        // of the record for each block.
        for blk in decoded.blocks.iter() {
            let tag = BufferTag {
                rel: RelTag {
                    spcnode: blk.rnode_spcnode,
                    dbnode: blk.rnode_dbnode,
                    relnode: blk.rnode_relnode,
                    forknum: blk.forknum as u8,
                },
                blknum: blk.blkno,
            };
            let rec = WALRecord {
                lsn,
                will_init: blk.will_init || blk.apply_image,
                rec: recdata.clone(),
                main_data_offset: decoded.main_data_offset as u32,
            };
            self.put_wal_record(tag, rec);
        }
        // Handle a few special record types
        if decoded.xl_rmid == pg_constants::RM_SMGR_ID
            && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK)
                == pg_constants::XLOG_SMGR_TRUNCATE
        {
            let truncate = XlSmgrTruncate::decode(&decoded);
            if (truncate.flags & pg_constants::SMGR_TRUNCATE_HEAP) != 0 {
                let rel = RelTag {
                    spcnode: truncate.rnode.spcnode,
                    dbnode: truncate.rnode.dbnode,
                    relnode: truncate.rnode.relnode,
                    forknum: pg_constants::MAIN_FORKNUM,
                };
                self.put_truncation(rel, lsn, truncate.blkno)?;
            }
        } else if decoded.xl_rmid == pg_constants::RM_DBASE_ID
            && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK)
                == pg_constants::XLOG_DBASE_CREATE
        {
            let createdb = XlCreateDatabase::decode(&decoded);
            self.put_create_database(
                lsn,
                createdb.db_id,
                createdb.tablespace_id,
                createdb.src_db_id,
                createdb.src_tablespace_id,
            )?;
        }
        // Now that this record has been handled, let the repository know that
        // it is up-to-date to this LSN
        self.advance_last_record_lsn(lsn);
        Ok(())
    }
    /// Remember the all WAL before the given LSN has been processed.
    ///
@@ -186,54 +167,15 @@ pub trait Timeline: Send + Sync {
    fn advance_last_record_lsn(&self, lsn: Lsn);
    fn get_last_record_lsn(&self) -> Lsn;
-    // Like `advance_last_record_lsn`, but points to the start position of last record
+    /// Get range [begin,end) of stored blocks. Used mostly for SMGR pseudorelations
-    fn get_prev_record_lsn(&self) -> Lsn;
+    /// but can be also applied to normal relations.
    fn get_range(&self, rel: RelTag, lsn: Lsn) -> Result<(u32, u32)>;
-    ///
+    /// Get vector of databases (represented using RelTag only dbnode and spcnode fields are used)
-    /// Flush to disk all data that was written with the put_* functions
+    fn get_databases(&self, lsn: Lsn) -> Result<Vec<RelTag>>;
    ///
    /// NOTE: This has nothing to do with checkpoint in PostgreSQL. We don't
    /// know anything about them here in the repository.
    fn checkpoint(&self) -> Result<()>;
-    /// Events for all relations in the timeline.
+    /// Get vector of prepared twophase transactions
-    /// Contains updates from start up to the last valid LSN
+    fn get_twophase(&self, lsn: Lsn) -> Result<Vec<TransactionId>>;
    /// at time of history() call. This lsn can be read via the lsn() function.
    ///
    /// Relation size is increased implicitly and decreased with Truncate updates.
    // TODO ordering guarantee?
    fn history<'a>(&'a self) -> Result<Box<dyn History + 'a>>;
    //
    // Wait until WAL has been received up to the given LSN.
    //
    fn wait_lsn(&self, lsn: Lsn) -> Result<Lsn>;
 }
 pub trait History: Iterator<Item = Result<Modification>> {
    /// The last_valid_lsn at the time of history() call.
    fn lsn(&self) -> Lsn;
 }
 //
 // Structure representing any update operation of object storage.
 // It is used to copy object storage content in PUSH method.
 //
 #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct Modification {
    pub tag: ObjectTag,
    pub lsn: Lsn,
    pub data: Vec<u8>,
 }
 impl Modification {
    pub fn new(entry: (ObjectTag, Lsn, Vec<u8>)) -> Modification {
        Modification {
            tag: entry.0,
            lsn: entry.1,
            data: entry.2,
        }
    }
 }
 #[derive(Clone)]
@@ -244,7 +186,81 @@ pub struct RepositoryStats {
    pub num_getpage_requests: Lsn,
 }
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy)]
 pub struct RelTag {
    pub forknum: u8,
    pub spcnode: u32,
    pub dbnode: u32,
    pub relnode: u32,
 }
 impl RelTag {
    pub fn pack(&self, buf: &mut BytesMut) {
        buf.put_u8(self.forknum);
        buf.put_u32(self.spcnode);
        buf.put_u32(self.dbnode);
        buf.put_u32(self.relnode);
    }
    pub fn unpack(buf: &mut Bytes) -> RelTag {
        RelTag {
            forknum: buf.get_u8(),
            spcnode: buf.get_u32(),
            dbnode: buf.get_u32(),
            relnode: buf.get_u32(),
        }
    }
 }
 /// Display RelTag in the same format that's used in most PostgreSQL debug messages:
 ///
 /// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
 ///
 impl fmt::Display for RelTag {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        if let Some(forkname) = forknumber_to_name(self.forknum) {
            write!(
                f,
                "{}/{}/{}_{}",
                self.spcnode, self.dbnode, self.relnode, forkname
            )
        } else {
            write!(f, "{}/{}/{}", self.spcnode, self.dbnode, self.relnode)
        }
    }
 }
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
 pub struct BufferTag {
    pub rel: RelTag,
    pub blknum: u32,
 }
 impl BufferTag {
    pub fn fork(forknum: u8) -> BufferTag {
        BufferTag {
            rel: RelTag {
                forknum,
                spcnode: 0,
                dbnode: 0,
                relnode: 0,
            },
            blknum: 0,
        }
    }
    pub fn pack(&self, buf: &mut BytesMut) {
        self.rel.pack(buf);
        buf.put_u32(self.blknum);
    }
    pub fn unpack(buf: &mut Bytes) -> BufferTag {
        BufferTag {
            rel: RelTag::unpack(buf),
            blknum: buf.get_u32(),
        }
    }
 }
 #[derive(Debug, Clone)]
 pub struct WALRecord {
    pub lsn: Lsn, // LSN at the *end* of the record
    pub will_init: bool,
@@ -284,34 +300,31 @@ impl WALRecord {
 #[cfg(test)]
 mod tests {
    use super::*;
    use crate::layered_repository::LayeredRepository;
    use crate::object_repository::ObjectRepository;
    use crate::object_repository::{ObjectValue, PageEntry, RelationSizeEntry};
    use crate::rocksdb_storage::RocksObjectStore;
    use crate::walredo::{WalRedoError, WalRedoManager};
-    use crate::{PageServerConf, RepositoryFormat};
+    use crate::PageServerConf;
    use postgres_ffi::pg_constants;
    use std::fs;
    use std::path::PathBuf;
    use std::str::FromStr;
    use std::time::Duration;
    use zenith_utils::bin_ser::BeSer;
    use zenith_utils::postgres_backend::AuthType;
    use zenith_utils::zid::ZTenantId;
    /// Arbitrary relation tag, for testing.
-    const TESTREL_A: RelishTag = RelishTag::Relation(RelTag {
+    const TESTREL_A: RelTag = RelTag {
        spcnode: 0,
        dbnode: 111,
        relnode: 1000,
        forknum: 0,
-    });
+    };
-    const TESTREL_B: RelishTag = RelishTag::Relation(RelTag {
+
-        spcnode: 0,
+    /// Convenience function to create a BufferTag for testing.
-        dbnode: 111,
+    /// Helps to keeps the tests shorter.
-        relnode: 1001,
+    #[allow(non_snake_case)]
-        forknum: 0,
+    fn TEST_BUF(blknum: u32) -> BufferTag {
-    });
+        BufferTag {
            rel: TESTREL_A,
            blknum,
        }
    }
    /// Convenience function to create a page image with given string as the only content
    #[allow(non_snake_case)]
@@ -323,127 +336,97 @@ mod tests {
        buf.freeze()
    }
-    static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
+    fn get_test_repo(test_name: &str) -> Result<Box<dyn Repository>> {
    fn get_test_repo(
        test_name: &str,
        repository_format: RepositoryFormat,
    ) -> Result<Box<dyn Repository>> {
        let repo_dir = PathBuf::from(format!("../tmp_check/test_{}", test_name));
        let _ = fs::remove_dir_all(&repo_dir);
        fs::create_dir_all(&repo_dir)?;
        fs::create_dir_all(&repo_dir.join("timelines"))?;
        let conf = PageServerConf {
            daemonize: false,
            interactive: false,
            gc_horizon: 64 * 1024 * 1024,
            gc_period: Duration::from_secs(10),
-            listen_addr: "127.0.0.1:5430".to_string(),
+            listen_addr: "127.0.0.1:5430".parse().unwrap(),
-            http_endpoint_addr: "127.0.0.1:9898".to_string(),
+            workdir: repo_dir.into(),
            superuser: "zenith_admin".to_string(),
            workdir: repo_dir,
            pg_distrib_dir: "".into(),
            auth_type: AuthType::Trust,
            auth_validation_public_key_path: None,
            repository_format,
        };
        // Make a static copy of the config. This can never be free'd, but that's
        // OK in a test.
        let conf: &'static PageServerConf = Box::leak(Box::new(conf));
        let tenantid = ZTenantId::generate();
        fs::create_dir_all(conf.tenant_path(&tenantid)).unwrap();
        let walredo_mgr = TestRedoManager {};
-        let repo: Box<dyn Repository + Sync + Send> = match conf.repository_format {
+        let repo = rocksdb::RocksRepository::new(conf, Arc::new(walredo_mgr));
            RepositoryFormat::Layered => Box::new(LayeredRepository::new(
                conf,
                Arc::new(walredo_mgr),
                tenantid,
            )),
            RepositoryFormat::RocksDb => {
                let obj_store = RocksObjectStore::create(conf, &tenantid)?;
-                Box::new(ObjectRepository::new(
+        Ok(Box::new(repo))
                    conf,
                    Arc::new(obj_store),
                    Arc::new(walredo_mgr),
                    tenantid,
                ))
            }
        };
        Ok(repo)
    }
    /// Test get_relsize() and truncation.
    ///
    /// FIXME: The RocksRepository implementation returns wrong relation size, if
    /// you make a request with an old LSN. It seems to ignore the requested LSN
    /// and always return result as of latest LSN. For such cases, the expected
    /// results below match the current RocksRepository behavior, so that the test
    /// passes, and the actually correct answers are in comments like
    /// "// CORRECT: <correct answer>"
    #[test]
-    fn test_relsize_rocksdb() -> Result<()> {
+    fn test_relsize() -> Result<()> {
        let repo = get_test_repo("test_relsize_rocksdb", RepositoryFormat::RocksDb)?;
        test_relsize(&*repo)
    }
    #[test]
    fn test_relsize_layered() -> Result<()> {
        let repo = get_test_repo("test_relsize_layered", RepositoryFormat::Layered)?;
        test_relsize(&*repo)
    }
    fn test_relsize(repo: &dyn Repository) -> Result<()> {
        // get_timeline() with non-existent timeline id should fail
        //repo.get_timeline("11223344556677881122334455667788");
        // Create timeline to work on
        let repo = get_test_repo("test_relsize")?;
        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
+        let tline = repo.create_empty_timeline(timelineid)?;
        tline.init_valid_lsn(Lsn(1));
-        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
+        tline.put_page_image(TEST_BUF(0), Lsn(2), TEST_IMG("foo blk 0 at 2"));
-        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
+        tline.put_page_image(TEST_BUF(0), Lsn(2), TEST_IMG("foo blk 0 at 2"));
-        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"), true)?;
+        tline.put_page_image(TEST_BUF(0), Lsn(3), TEST_IMG("foo blk 0 at 3"));
-        tline.put_page_image(TESTREL_A, 1, Lsn(4), TEST_IMG("foo blk 1 at 4"), true)?;
+        tline.put_page_image(TEST_BUF(1), Lsn(4), TEST_IMG("foo blk 1 at 4"));
-        tline.put_page_image(TESTREL_A, 2, Lsn(5), TEST_IMG("foo blk 2 at 5"), true)?;
+        tline.put_page_image(TEST_BUF(2), Lsn(5), TEST_IMG("foo blk 2 at 5"));
        tline.advance_last_valid_lsn(Lsn(5));
-        // The relation was created at LSN 2, not visible at LSN 1 yet.
+        // rocksdb implementation erroneosly returns 'true' here
-        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(1))?, false);
+        assert_eq!(tline.get_relsize_exists(TESTREL_A, Lsn(1))?, true); // CORRECT: false
-        assert!(tline.get_relish_size(TESTREL_A, Lsn(1))?.is_none());
+                                                                        // likewise, it returns wrong size here
        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(1))?, 3); // CORRECT: 0 (or error?)
-        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(2))?, true);
+        assert_eq!(tline.get_relsize_exists(TESTREL_A, Lsn(2))?, true);
-        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(2))?.unwrap(), 1);
+        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(2))?, 3); // CORRECT: 1
-        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(5))?.unwrap(), 3);
+        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(5))?, 3);
        // Check page contents at each LSN
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(2))?,
+            tline.get_page_at_lsn(TEST_BUF(0), Lsn(2))?,
            TEST_IMG("foo blk 0 at 2")
        );
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(3))?,
+            tline.get_page_at_lsn(TEST_BUF(0), Lsn(3))?,
            TEST_IMG("foo blk 0 at 3")
        );
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(4))?,
+            tline.get_page_at_lsn(TEST_BUF(0), Lsn(4))?,
            TEST_IMG("foo blk 0 at 3")
        );
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(4))?,
+            tline.get_page_at_lsn(TEST_BUF(1), Lsn(4))?,
            TEST_IMG("foo blk 1 at 4")
        );
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(5))?,
+            tline.get_page_at_lsn(TEST_BUF(0), Lsn(5))?,
            TEST_IMG("foo blk 0 at 3")
        );
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(5))?,
+            tline.get_page_at_lsn(TEST_BUF(1), Lsn(5))?,
            TEST_IMG("foo blk 1 at 4")
        );
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 2, Lsn(5))?,
+            tline.get_page_at_lsn(TEST_BUF(2), Lsn(5))?,
            TEST_IMG("foo blk 2 at 5")
        );
@@ -452,20 +435,20 @@ mod tests {
        tline.advance_last_valid_lsn(Lsn(6));
        // Check reported size and contents after truncation
-        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(6))?.unwrap(), 2);
+        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(6))?, 2);
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(6))?,
+            tline.get_page_at_lsn(TEST_BUF(0), Lsn(6))?,
            TEST_IMG("foo blk 0 at 3")
        );
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(6))?,
+            tline.get_page_at_lsn(TEST_BUF(1), Lsn(6))?,
            TEST_IMG("foo blk 1 at 4")
        );
        // should still see the truncated block with older LSN
-        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(5))?.unwrap(), 3);
+        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(5))?, 2); // CORRECT: 3
        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 2, Lsn(5))?,
+            tline.get_page_at_lsn(TEST_BUF(2), Lsn(5))?,
            TEST_IMG("foo blk 2 at 5")
        );
@@ -478,33 +461,23 @@ mod tests {
    /// This isn't very interesting with the RocksDb implementation, as we don't pay
    /// any attention to Postgres segment boundaries there.
    #[test]
-    fn test_large_rel_rocksdb() -> Result<()> {
+    fn test_large_rel() -> Result<()> {
-        let repo = get_test_repo("test_large_rel_rocksdb", RepositoryFormat::RocksDb)?;
+        let repo = get_test_repo("test_large_rel")?;
        test_large_rel(&*repo)
    }
    #[test]
    fn test_large_rel_layered() -> Result<()> {
        let repo = get_test_repo("test_large_rel_layered", RepositoryFormat::Layered)?;
        test_large_rel(&*repo)
    }
    fn test_large_rel(repo: &dyn Repository) -> Result<()> {
        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
+        let tline = repo.create_empty_timeline(timelineid)?;
        tline.init_valid_lsn(Lsn(1));
-        let mut lsn = 1;
+        let mut lsn = 0;
-        for blknum in 0..pg_constants::RELSEG_SIZE + 1 {
+        for i in 0..pg_constants::RELSEG_SIZE + 1 {
-            let img = TEST_IMG(&format!("foo blk {} at {}", blknum, Lsn(lsn)));
+            let img = TEST_IMG(&format!("foo blk {} at {}", i, Lsn(lsn)));
            lsn += 1;
-            tline.put_page_image(TESTREL_A, blknum as u32, Lsn(lsn), img, true)?;
+            tline.put_page_image(TEST_BUF(i as u32), Lsn(lsn), img);
        }
        tline.advance_last_valid_lsn(Lsn(lsn));
        assert_eq!(
-            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
+            tline.get_relsize(TESTREL_A, Lsn(lsn))?,
            pg_constants::RELSEG_SIZE + 1
        );
@@ -513,7 +486,7 @@ mod tests {
        tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE)?;
        tline.advance_last_valid_lsn(Lsn(lsn));
        assert_eq!(
-            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
+            tline.get_relsize(TESTREL_A, Lsn(lsn))?,
            pg_constants::RELSEG_SIZE
        );
@@ -522,186 +495,10 @@ mod tests {
        tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE - 1)?;
        tline.advance_last_valid_lsn(Lsn(lsn));
        assert_eq!(
-            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
+            tline.get_relsize(TESTREL_A, Lsn(lsn))?,
            pg_constants::RELSEG_SIZE - 1
        );
        // Truncate to 1500, and then truncate all the way down to 0, one block at a time
        // This tests the behavior at segment boundaries
        let mut size: i32 = 3000;
        while size >= 0 {
            lsn += 1;
            tline.put_truncation(TESTREL_A, Lsn(lsn), size as u32)?;
            tline.advance_last_valid_lsn(Lsn(lsn));
            assert_eq!(
                tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
                size as u32
            );
            size -= 1;
        }
        Ok(())
    }
    fn skip_nonrel_objects<'a>(
        snapshot: Box<dyn History + 'a>,
    ) -> Result<impl Iterator<Item = <dyn History as Iterator>::Item> + 'a> {
        Ok(snapshot.skip_while(|r| match r {
            Ok(m) => match m.tag {
                ObjectTag::RelationMetadata(_) => false,
                _ => true,
            },
            _ => panic!("Iteration error"),
        }))
    }
    #[test]
    fn test_branch_rocksdb() -> Result<()> {
        let repo = get_test_repo("test_branch_rocksdb", RepositoryFormat::RocksDb)?;
        test_branch(&*repo)
    }
    #[test]
    fn test_branch_layered() -> Result<()> {
        let repo = get_test_repo("test_branch_layered", RepositoryFormat::Layered)?;
        test_branch(&*repo)
    }
    ///
    /// Test branch creation
    ///
    fn test_branch(repo: &dyn Repository) -> Result<()> {
        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
        // Import initial dummy checkpoint record, otherwise the get_timeline() call
        // after branching fails below
        tline.put_page_image(RelishTag::Checkpoint, 0, Lsn(1), ZERO_PAGE.clone(), false)?;
        // Create a relation on the timeline
        tline.init_valid_lsn(Lsn(1));
        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"), true)?;
        tline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("foo blk 0 at 4"), true)?;
        // Create another relation
        tline.put_page_image(TESTREL_B, 0, Lsn(2), TEST_IMG("foobar blk 0 at 2"), true)?;
        tline.advance_last_valid_lsn(Lsn(4));
        // Branch the history, modify relation differently on the new timeline
        let newtimelineid = ZTimelineId::from_str("AA223344556677881122334455667788").unwrap();
        repo.branch_timeline(timelineid, newtimelineid, Lsn(3))?;
        let newtline = repo.get_timeline(newtimelineid)?;
        newtline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("bar blk 0 at 4"), true)?;
        newtline.advance_last_valid_lsn(Lsn(4));
        // Check page contents on both branches
        assert_eq!(
            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(4))?,
            TEST_IMG("foo blk 0 at 4")
        );
        assert_eq!(
            newtline.get_page_at_lsn(TESTREL_A, 0, Lsn(4))?,
            TEST_IMG("bar blk 0 at 4")
        );
        assert_eq!(
            newtline.get_page_at_lsn(TESTREL_B, 0, Lsn(4))?,
            TEST_IMG("foobar blk 0 at 2")
        );
        assert_eq!(newtline.get_relish_size(TESTREL_B, Lsn(4))?.unwrap(), 1);
        Ok(())
    }
    #[test]
    fn test_history_rocksdb() -> Result<()> {
        let repo = get_test_repo("test_history_rocksdb", RepositoryFormat::RocksDb)?;
        test_history(&*repo)
    }
    #[test]
    // TODO: This doesn't work with the layered storage, the functions needed for push/pull
    // functionality haven't been implemented yet.
    #[ignore]
    fn test_history_layered() -> Result<()> {
        let repo = get_test_repo("test_history_layered", RepositoryFormat::Layered)?;
        test_history(&*repo)
    }
    fn test_history(repo: &dyn Repository) -> Result<()> {
        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
        let snapshot = tline.history()?;
        assert_eq!(snapshot.lsn(), Lsn(0));
        let mut snapshot = skip_nonrel_objects(snapshot)?;
        assert_eq!(None, snapshot.next().transpose()?);
        // add a page and advance the last valid LSN
        let rel = TESTREL_A;
        tline.put_page_image(rel, 1, Lsn(1), TEST_IMG("blk 1 @ lsn 1"), true)?;
        tline.advance_last_valid_lsn(Lsn(1));
        let expected_page = Modification {
            tag: ObjectTag::Buffer(rel, 1),
            lsn: Lsn(1),
            data: ObjectValue::ser(&ObjectValue::Page(PageEntry::Page(TEST_IMG(
                "blk 1 @ lsn 1",
            ))))?,
        };
        let expected_init_size = Modification {
            tag: ObjectTag::RelationMetadata(rel),
            lsn: Lsn(1),
            data: ObjectValue::ser(&ObjectValue::RelationSize(RelationSizeEntry::Size(2)))?,
        };
        let expected_trunc_size = Modification {
            tag: ObjectTag::RelationMetadata(rel),
            lsn: Lsn(2),
            data: ObjectValue::ser(&ObjectValue::RelationSize(RelationSizeEntry::Size(0)))?,
        };
        let snapshot = tline.history()?;
        assert_eq!(snapshot.lsn(), Lsn(1));
        let mut snapshot = skip_nonrel_objects(snapshot)?;
        assert_eq!(
            Some(&expected_init_size),
            snapshot.next().transpose()?.as_ref()
        );
        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
        assert_eq!(None, snapshot.next().transpose()?);
        // truncate to zero, but don't advance the last valid LSN
        tline.put_truncation(rel, Lsn(2), 0)?;
        let snapshot = tline.history()?;
        assert_eq!(snapshot.lsn(), Lsn(1));
        let mut snapshot = skip_nonrel_objects(snapshot)?;
        assert_eq!(
            Some(&expected_init_size),
            snapshot.next().transpose()?.as_ref()
        );
        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
        assert_eq!(None, snapshot.next().transpose()?);
        // advance the last valid LSN and the truncation should be observable
        tline.advance_last_valid_lsn(Lsn(2));
        let snapshot = tline.history()?;
        assert_eq!(snapshot.lsn(), Lsn(2));
        let mut snapshot = skip_nonrel_objects(snapshot)?;
        assert_eq!(
            Some(&expected_init_size),
            snapshot.next().transpose()?.as_ref()
        );
        assert_eq!(
            Some(&expected_trunc_size),
            snapshot.next().transpose()?.as_ref()
        );
        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
        assert_eq!(None, snapshot.next().transpose()?);
        Ok(())
    }
@@ -711,16 +508,15 @@ mod tests {
    impl WalRedoManager for TestRedoManager {
        fn request_redo(
            &self,
-            rel: RelishTag,
+            tag: BufferTag,
            blknum: u32,
            lsn: Lsn,
            base_img: Option<Bytes>,
            records: Vec<WALRecord>,
        ) -> Result<Bytes, WalRedoError> {
            let s = format!(
-                "redo for {} blk {} to get to {}, with {} and {} records",
+                "redo for rel {} blk {} to get to {}, with {} and {} records",
-                rel,
+                tag.rel,
-                blknum,
+                tag.blknum,
                lsn,
                if base_img.is_some() {
                    "base image"
@@ -0,0 +1,978 @@
 //
 // A Repository holds all the different page versions and WAL records
 //
 // This implementation uses RocksDB to store WAL wal records and
 // full page images, keyed by the RelFileNode, blocknumber, and the
 // LSN.
 use crate::repository::{BufferTag, RelTag, Repository, Timeline, WALRecord};
 use crate::restore_local_repo::restore_timeline;
 use crate::waldecoder::{Oid, TransactionId};
 use crate::walredo::WalRedoManager;
 use crate::PageServerConf;
 use crate::ZTimelineId;
 // use crate::PageServerConf;
 // use crate::branches;
 use anyhow::{bail, Context, Result};
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use log::*;
 use postgres_ffi::nonrelfile_utils::transaction_id_get_status;
 use postgres_ffi::*;
 use std::cmp::min;
 use std::collections::HashMap;
 use std::convert::TryInto;
 use std::sync::atomic::AtomicU64;
 use std::sync::atomic::Ordering;
 use std::sync::{Arc, Mutex};
 use std::thread;
 use std::time::{Duration, Instant};
 use zenith_utils::lsn::{AtomicLsn, Lsn};
 use zenith_utils::seqwait::SeqWait;
 // Timeout when waiting or WAL receiver to catch up to an LSN given in a GetPage@LSN call.
 static TIMEOUT: Duration = Duration::from_secs(60);
 pub struct RocksRepository {
    conf: &'static PageServerConf,
    timelines: Mutex<HashMap<ZTimelineId, Arc<RocksTimeline>>>,
    walredo_mgr: Arc<dyn WalRedoManager>,
 }
 pub struct RocksTimeline {
    // RocksDB handle
    db: rocksdb::DB,
    // WAL redo manager
    walredo_mgr: Arc<dyn WalRedoManager>,
    // What page versions do we hold in the cache? If we get a request > last_valid_lsn,
    // we need to wait until we receive all the WAL up to the request. The SeqWait
    // provides functions for that. TODO: If we get a request for an old LSN, such that
    // the versions have already been garbage collected away, we should throw an error,
    // but we don't track that currently.
    //
    // last_record_lsn points to the end of last processed WAL record.
    // It can lag behind last_valid_lsn, if the WAL receiver has received some WAL
    // after the end of last record, but not the whole next record yet. In the
    // page cache, we care about last_valid_lsn, but if the WAL receiver needs to
    // restart the streaming, it needs to restart at the end of last record, so
    // we track them separately. last_record_lsn should perhaps be in
    // walreceiver.rs instead of here, but it seems convenient to keep all three
    // values together.
    //
    last_valid_lsn: SeqWait<Lsn>,
    last_record_lsn: AtomicLsn,
    // Counters, for metrics collection.
    pub num_entries: AtomicU64,
    pub num_page_images: AtomicU64,
    pub num_wal_records: AtomicU64,
    pub num_getpage_requests: AtomicU64,
 }
 //
 // We store two kinds of entries in the repository:
 //
 // 1. Ready-made images of the block
 // 2. WAL records, to be applied on top of the "previous" entry
 //
 // Some WAL records will initialize the page from scratch. For such records,
 // the 'will_init' flag is set. They don't need the previous page image before
 // applying. The 'will_init' flag is set for records containing a full-page image,
 // and for records with the BKPBLOCK_WILL_INIT flag. These differ from PageImages
 // stored directly in the cache entry in that you still need to run the WAL redo
 // routine to generate the page image.
 //
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
 struct CacheKey {
    pub tag: BufferTag,
    pub lsn: Lsn,
 }
 impl CacheKey {
    fn pack(&self, buf: &mut BytesMut) {
        self.tag.pack(buf);
        buf.put_u64(self.lsn.0);
    }
    fn unpack(buf: &mut Bytes) -> CacheKey {
        CacheKey {
            tag: BufferTag::unpack(buf),
            lsn: Lsn::from(buf.get_u64()),
        }
    }
    fn from_slice(slice: &[u8]) -> Self {
        let mut buf = Bytes::copy_from_slice(slice);
        Self::unpack(&mut buf)
    }
    fn to_bytes(&self) -> BytesMut {
        let mut buf = BytesMut::new();
        self.pack(&mut buf);
        buf
    }
 }
 enum CacheEntryContent {
    PageImage(Bytes),
    WALRecord(WALRecord),
    Truncation,
 }
 // The serialized representation of a CacheEntryContent begins with
 // single byte that indicates what kind of entry it is. There is also
 // an UNUSED_VERSION_FLAG that is not represented in the CacheEntryContent
 // at all, you must peek into the first byte of the serialized representation
 // to read it.
 const CONTENT_PAGE_IMAGE: u8 = 1u8;
 const CONTENT_WAL_RECORD: u8 = 2u8;
 const CONTENT_TRUNCATION: u8 = 3u8;
 const CONTENT_KIND_MASK: u8 = 3u8; // bitmask that covers the above
 const UNUSED_VERSION_FLAG: u8 = 4u8;
 impl CacheEntryContent {
    pub fn pack(&self, buf: &mut BytesMut) {
        match self {
            CacheEntryContent::PageImage(image) => {
                buf.put_u8(CONTENT_PAGE_IMAGE);
                buf.put_u16(image.len() as u16);
                buf.put_slice(&image[..]);
            }
            CacheEntryContent::WALRecord(rec) => {
                buf.put_u8(CONTENT_WAL_RECORD);
                rec.pack(buf);
            }
            CacheEntryContent::Truncation => {
                buf.put_u8(CONTENT_TRUNCATION);
            }
        }
    }
    pub fn unpack(buf: &mut Bytes) -> CacheEntryContent {
        let kind = buf.get_u8() & CONTENT_KIND_MASK;
        match kind {
            CONTENT_PAGE_IMAGE => {
                let len = buf.get_u16() as usize;
                let mut dst = vec![0u8; len];
                buf.copy_to_slice(&mut dst);
                CacheEntryContent::PageImage(Bytes::from(dst))
            }
            CONTENT_WAL_RECORD => CacheEntryContent::WALRecord(WALRecord::unpack(buf)),
            CONTENT_TRUNCATION => CacheEntryContent::Truncation,
            _ => unreachable!(),
        }
    }
    fn from_slice(slice: &[u8]) -> Self {
        let mut buf = Bytes::copy_from_slice(slice);
        Self::unpack(&mut buf)
    }
    fn to_bytes(&self) -> BytesMut {
        let mut buf = BytesMut::new();
        self.pack(&mut buf);
        buf
    }
 }
 impl RocksRepository {
    pub fn new(
        conf: &'static PageServerConf,
        walredo_mgr: Arc<dyn WalRedoManager>,
    ) -> RocksRepository {
        RocksRepository {
            conf: conf,
            timelines: Mutex::new(HashMap::new()),
            walredo_mgr,
        }
    }
 }
 // Get handle to a given timeline. It is assumed to already exist.
 impl Repository for RocksRepository {
    fn get_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>> {
        let timelines = self.timelines.lock().unwrap();
        match timelines.get(&timelineid) {
            Some(timeline) => Ok(timeline.clone()),
            None => bail!("timeline not found"),
        }
    }
    fn get_or_restore_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>> {
        let mut timelines = self.timelines.lock().unwrap();
        match timelines.get(&timelineid) {
            Some(timeline) => Ok(timeline.clone()),
            None => {
                let timeline = RocksTimeline::new(self.conf, timelineid, self.walredo_mgr.clone());
                restore_timeline(self.conf, &timeline, timelineid)?;
                let timeline_rc = Arc::new(timeline);
                timelines.insert(timelineid, timeline_rc.clone());
                if self.conf.gc_horizon != 0 {
                    let timeline_rc_copy = timeline_rc.clone();
                    let conf = self.conf;
                    let _gc_thread = thread::Builder::new()
                        .name("Garbage collection thread".into())
                        .spawn(move || {
                            // FIXME
                            timeline_rc_copy.do_gc(conf).expect("GC thread died");
                        })
                        .unwrap();
                }
                Ok(timeline_rc)
            }
        }
    }
    #[cfg(test)]
    fn create_empty_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>> {
        let mut timelines = self.timelines.lock().unwrap();
        let timeline = RocksTimeline::new(&self.conf, timelineid, self.walredo_mgr.clone());
        let timeline_rc = Arc::new(timeline);
        let r = timelines.insert(timelineid, timeline_rc.clone());
        assert!(r.is_none());
        // don't start the garbage collector for unit tests, either.
        Ok(timeline_rc)
    }
 }
 impl RocksTimeline {
    fn open_rocksdb(conf: &PageServerConf, timelineid: ZTimelineId) -> rocksdb::DB {
        let path = conf.timeline_path(timelineid);
        let mut opts = rocksdb::Options::default();
        opts.create_if_missing(true);
        opts.set_use_fsync(true);
        opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
        opts.set_compaction_filter("ttl", move |_level: u32, _key: &[u8], val: &[u8]| {
            if (val[0] & UNUSED_VERSION_FLAG) != 0 {
                rocksdb::compaction_filter::Decision::Remove
            } else {
                rocksdb::compaction_filter::Decision::Keep
            }
        });
        rocksdb::DB::open(&opts, &path).unwrap()
    }
    fn new(
        conf: &'static PageServerConf,
        timelineid: ZTimelineId,
        walredo_mgr: Arc<dyn WalRedoManager>,
    ) -> RocksTimeline {
        RocksTimeline {
            db: RocksTimeline::open_rocksdb(conf, timelineid),
            walredo_mgr,
            last_valid_lsn: SeqWait::new(Lsn(0)),
            last_record_lsn: AtomicLsn::new(0),
            num_entries: AtomicU64::new(0),
            num_page_images: AtomicU64::new(0),
            num_wal_records: AtomicU64::new(0),
            num_getpage_requests: AtomicU64::new(0),
        }
    }
 }
 impl RocksTimeline {
    ///
    /// Collect all the WAL records that are needed to reconstruct a page
    /// image for the given cache entry.
    ///
    /// Returns an old page image (if any), and a vector of WAL records to apply
    /// over it.
    ///
    fn collect_records_for_apply(
        &self,
        tag: BufferTag,
        lsn: Lsn,
    ) -> (Option<Bytes>, Vec<WALRecord>) {
        let key = CacheKey { tag, lsn };
        let mut base_img: Option<Bytes> = None;
        let mut records: Vec<WALRecord> = Vec::new();
        let mut iter = self.db.raw_iterator();
        iter.seek_for_prev(key.to_bytes());
        // Scan backwards, collecting the WAL records, until we hit an
        // old page image.
        while iter.valid() {
            let key = CacheKey::from_slice(iter.key().unwrap());
            if key.tag != tag {
                break;
            }
            let content = CacheEntryContent::from_slice(iter.value().unwrap());
            if let CacheEntryContent::PageImage(img) = content {
                // We have a base image. No need to dig deeper into the list of
                // records
                base_img = Some(img);
                break;
            } else if let CacheEntryContent::WALRecord(rec) = content {
                records.push(rec.clone());
                // If this WAL record initializes the page, no need to dig deeper.
                if rec.will_init {
                    break;
                }
            } else {
                panic!("no base image and no WAL record on cache entry");
            }
            iter.prev();
        }
        records.reverse();
        (base_img, records)
    }
    // Internal functions
    //
    // Internal function to get relation size at given LSN.
    //
    // The caller must ensure that WAL has been received up to 'lsn'.
    //
    fn relsize_get_nowait(&self, rel: RelTag, lsn: Lsn) -> Result<u32> {
        assert!(lsn <= self.last_valid_lsn.load());
        let mut key = CacheKey {
            tag: BufferTag {
                rel,
                blknum: u32::MAX,
            },
            lsn,
        };
        let mut iter = self.db.raw_iterator();
        loop {
            iter.seek_for_prev(key.to_bytes());
            if iter.valid() {
                let thiskey = CacheKey::from_slice(iter.key().unwrap());
                if thiskey.tag.rel == rel {
                    let content = CacheEntryContent::from_slice(iter.value().unwrap());
                    if let CacheEntryContent::Truncation = content {
                        if thiskey.tag.blknum > 0 {
                            key.tag.blknum = thiskey.tag.blknum - 1;
                            continue;
                        }
                        break;
                    }
                    let relsize = thiskey.tag.blknum + 1;
                    debug!("Size of relation {} at {} is {}", rel, lsn, relsize);
                    return Ok(relsize);
                }
            }
            break;
        }
        debug!("Size of relation {} at {} is zero", rel, lsn);
        Ok(0)
    }
    fn do_gc(&self, conf: &'static PageServerConf) -> Result<Bytes> {
        loop {
            thread::sleep(conf.gc_period);
            let last_lsn = self.get_last_valid_lsn();
            // checked_sub() returns None on overflow.
            if let Some(horizon) = last_lsn.checked_sub(conf.gc_horizon) {
                let mut maxkey = CacheKey {
                    tag: BufferTag {
                        rel: RelTag {
                            spcnode: u32::MAX,
                            dbnode: u32::MAX,
                            relnode: u32::MAX,
                            forknum: u8::MAX,
                        },
                        blknum: u32::MAX,
                    },
                    lsn: Lsn::MAX,
                };
                let now = Instant::now();
                let mut reconstructed = 0u64;
                let mut truncated = 0u64;
                let mut inspected = 0u64;
                let mut deleted = 0u64;
                loop {
                    let mut iter = self.db.raw_iterator();
                    iter.seek_for_prev(maxkey.to_bytes());
                    if iter.valid() {
                        let key = CacheKey::from_slice(iter.key().unwrap());
                        let v = iter.value().unwrap();
                        inspected += 1;
                        // Construct boundaries for old records cleanup
                        maxkey.tag = key.tag;
                        let last_lsn = key.lsn;
                        maxkey.lsn = min(horizon, last_lsn); // do not remove last version
                        let mut minkey = maxkey.clone();
                        minkey.lsn = Lsn(0); // first version
                        // Special handling of delete of PREPARE WAL record
                        if last_lsn < horizon
                            && key.tag.rel.forknum == pg_constants::PG_TWOPHASE_FORKNUM
                        {
                            if (v[0] & UNUSED_VERSION_FLAG) == 0 {
                                let mut v = v.to_owned();
                                v[0] |= UNUSED_VERSION_FLAG;
                                self.db.put(key.to_bytes(), &v[..])?;
                                deleted += 1;
                            }
                            maxkey = minkey;
                            continue;
                        }
                        // reconstruct most recent page version
                        if (v[0] & CONTENT_KIND_MASK) == CONTENT_WAL_RECORD {
                            // force reconstruction of most recent page version
                            let (base_img, records) =
                                self.collect_records_for_apply(key.tag, key.lsn);
                            trace!(
                                "Reconstruct most recent page {} blk {} at {} from {} records",
                                key.tag.rel,
                                key.tag.blknum,
                                key.lsn,
                                records.len()
                            );
                            let new_img = self
                                .walredo_mgr
                                .request_redo(key.tag, key.lsn, base_img, records)?;
                            self.put_page_image(key.tag, key.lsn, new_img.clone());
                            reconstructed += 1;
                        }
                        iter.seek_for_prev(maxkey.to_bytes());
                        if iter.valid() {
                            // do not remove last version
                            if last_lsn > horizon {
                                // locate most recent record before horizon
                                let key = CacheKey::from_slice(iter.key().unwrap());
                                if key.tag == maxkey.tag {
                                    let v = iter.value().unwrap();
                                    if (v[0] & CONTENT_KIND_MASK) == CONTENT_WAL_RECORD {
                                        let (base_img, records) =
                                            self.collect_records_for_apply(key.tag, key.lsn);
                                        trace!("Reconstruct horizon page {} blk {} at {} from {} records",
                                              key.tag.rel, key.tag.blknum, key.lsn, records.len());
                                        let new_img = self
                                            .walredo_mgr
                                            .request_redo(key.tag, key.lsn, base_img, records)?;
                                        self.put_page_image(key.tag, key.lsn, new_img.clone());
                                        truncated += 1;
                                    } else {
                                        trace!(
                                            "Keeping horizon page {} blk {} at {}",
                                            key.tag.rel,
                                            key.tag.blknum,
                                            key.lsn
                                        );
                                    }
                                }
                            } else {
                                trace!(
                                    "Last page {} blk {} at {}, horizon {}",
                                    key.tag.rel,
                                    key.tag.blknum,
                                    key.lsn,
                                    horizon
                                );
                            }
                            // remove records prior to horizon
                            loop {
                                iter.prev();
                                if !iter.valid() {
                                    break;
                                }
                                let key = CacheKey::from_slice(iter.key().unwrap());
                                if key.tag != maxkey.tag {
                                    break;
                                }
                                let v = iter.value().unwrap();
                                if (v[0] & UNUSED_VERSION_FLAG) == 0 {
                                    let mut v = v.to_owned();
                                    v[0] |= UNUSED_VERSION_FLAG;
                                    self.db.put(key.to_bytes(), &v[..])?;
                                    deleted += 1;
                                    trace!(
                                        "deleted: {} blk {} at {}",
                                        key.tag.rel,
                                        key.tag.blknum,
                                        key.lsn
                                    );
                                } else {
                                    break;
                                }
                            }
                        }
                        maxkey = minkey;
                    } else {
                        break;
                    }
                }
                info!("Garbage collection completed in {:?}:\n{} version chains inspected, {} pages reconstructed, {} version histories truncated, {} versions deleted",
 					  now.elapsed(), inspected, reconstructed, truncated, deleted);
            }
        }
    }
    //
    // Wait until WAL has been received up to the given LSN.
    //
    fn wait_lsn(&self, mut lsn: Lsn) -> Result<Lsn> {
        // When invalid LSN is requested, it means "don't wait, return latest version of the page"
        // This is necessary for bootstrap.
        if lsn == Lsn(0) {
            let last_valid_lsn = self.last_valid_lsn.load();
            trace!(
                "walreceiver doesn't work yet last_valid_lsn {}, requested {}",
                last_valid_lsn,
                lsn
            );
            lsn = last_valid_lsn;
        }
        //trace!("Start waiting for LSN {}, valid LSN is {}", lsn,  self.last_valid_lsn.load());
        self.last_valid_lsn
            .wait_for_timeout(lsn, TIMEOUT)
            .with_context(|| {
                format!(
                    "Timed out while waiting for WAL record at LSN {} to arrive",
                    lsn
                )
            })?;
        //trace!("Stop waiting for LSN {}, valid LSN is {}", lsn,  self.last_valid_lsn.load());
        Ok(lsn)
    }
 }
 impl Timeline for RocksTimeline {
    // Public GET interface functions
    ///
    /// GetPage@LSN
    ///
    /// Returns an 8k page image
    ///
    fn get_page_at_lsn(&self, tag: BufferTag, req_lsn: Lsn) -> Result<Bytes> {
        self.num_getpage_requests.fetch_add(1, Ordering::Relaxed);
        let lsn = self.wait_lsn(req_lsn)?;
        // Look up cache entry. If it's a page image, return that. If it's a WAL record,
        // ask the WAL redo service to reconstruct the page image from the WAL records.
        let key = CacheKey { tag, lsn };
        let mut iter = self.db.raw_iterator();
        iter.seek_for_prev(key.to_bytes());
        if iter.valid() {
            let key = CacheKey::from_slice(iter.key().unwrap());
            if key.tag == tag {
                let content = CacheEntryContent::from_slice(iter.value().unwrap());
                let page_img: Bytes;
                if let CacheEntryContent::PageImage(img) = content {
                    page_img = img;
                } else if let CacheEntryContent::WALRecord(_rec) = content {
                    // Request the WAL redo manager to apply the WAL records for us.
                    let (base_img, records) = self.collect_records_for_apply(tag, lsn);
                    page_img = self.walredo_mgr.request_redo(tag, lsn, base_img, records)?;
                    self.put_page_image(tag, lsn, page_img.clone());
                } else {
                    // No base image, and no WAL record. Huh?
                    bail!("no page image or WAL record for requested page");
                }
                // FIXME: assumes little-endian. Only used for the debugging log though
                let page_lsn_hi =
                    u32::from_le_bytes(page_img.get(0..4).unwrap().try_into().unwrap());
                let page_lsn_lo =
                    u32::from_le_bytes(page_img.get(4..8).unwrap().try_into().unwrap());
                debug!(
                    "Returning page with LSN {:X}/{:X} for {} blk {}",
                    page_lsn_hi, page_lsn_lo, tag.rel, tag.blknum
                );
                return Ok(page_img);
            }
        }
        static ZERO_PAGE: [u8; 8192] = [0u8; 8192];
        debug!(
            "Page {} blk {} at {}({}) not found",
            tag.rel, tag.blknum, req_lsn, lsn
        );
        Ok(Bytes::from_static(&ZERO_PAGE))
        /* return Err("could not find page image")?; */
    }
    ///
    /// Get size of relation at given LSN.
    ///
    fn get_relsize(&self, rel: RelTag, lsn: Lsn) -> Result<u32> {
        let lsn = self.wait_lsn(lsn)?;
        self.relsize_get_nowait(rel, lsn)
    }
    /// Get vector of prepared twophase transactions
    fn get_twophase(&self, lsn: Lsn) -> Result<Vec<TransactionId>> {
        let key = CacheKey {
            // minimal key
            tag: BufferTag {
                rel: RelTag {
                    forknum: pg_constants::PG_TWOPHASE_FORKNUM,
                    spcnode: 0,
                    dbnode: 0,
                    relnode: 0,
                },
                blknum: 0,
            },
            lsn: Lsn(0),
        };
        let mut gxacts = Vec::new();
        let mut iter = self.db.raw_iterator();
        iter.seek(key.to_bytes());
        while iter.valid() {
            let key = CacheKey::from_slice(iter.key().unwrap());
            if key.tag.rel.forknum != pg_constants::PG_TWOPHASE_FORKNUM {
                break; // we are done with this fork
            }
            if key.lsn <= lsn {
                let xid = key.tag.blknum;
                let tag = BufferTag {
                    rel: RelTag {
                        forknum: pg_constants::PG_XACT_FORKNUM,
                        spcnode: 0,
                        dbnode: 0,
                        relnode: 0,
                    },
                    blknum: xid / pg_constants::CLOG_XACTS_PER_PAGE,
                };
                let clog_page = self.get_page_at_lsn(tag, lsn)?;
                let status = transaction_id_get_status(xid, &clog_page[..]);
                if status == pg_constants::TRANSACTION_STATUS_IN_PROGRESS {
                    gxacts.push(xid);
                }
            }
            iter.next();
        }
        return Ok(gxacts);
    }
    /// Get databases. This function is used to local pg_filenode.map files
    fn get_databases(&self, lsn: Lsn) -> Result<Vec<RelTag>> {
        let key = CacheKey {
            // minimal key
            tag: BufferTag {
                rel: RelTag {
                    forknum: pg_constants::PG_FILENODEMAP_FORKNUM,
                    spcnode: 0,
                    dbnode: 0,
                    relnode: 0,
                },
                blknum: 0,
            },
            lsn: Lsn(0),
        };
        let mut dbs = Vec::new();
        let mut iter = self.db.raw_iterator();
        iter.seek(key.to_bytes());
        let mut prev_tag = key.tag.rel;
        while iter.valid() {
            let key = CacheKey::from_slice(iter.key().unwrap());
            if key.tag.rel.forknum != pg_constants::PG_FILENODEMAP_FORKNUM {
                break; // we are done with this fork
            }
            if key.tag.rel != prev_tag && key.lsn <= lsn {
                prev_tag = key.tag.rel;
                dbs.push(prev_tag); // collect unique tags
            }
            iter.next();
        }
        return Ok(dbs);
    }
    /// Get range [begin,end) of stored blocks. Used mostly for SMGR pseudorelations
    /// but can be also applied to normal relations.
    fn get_range(&self, rel: RelTag, lsn: Lsn) -> Result<(u32, u32)> {
        let _lsn = self.wait_lsn(lsn)?;
        let mut key = CacheKey {
            // minimal key to start with
            tag: BufferTag { rel, blknum: 0 },
            lsn: Lsn(0),
        };
        let mut iter = self.db.raw_iterator();
        iter.seek(key.to_bytes()); // locate first entry
        if iter.valid() {
            let thiskey = CacheKey::from_slice(iter.key().unwrap());
            let tag = thiskey.tag;
            if tag.rel == rel {
                // still trversing this relation
                let first_blknum = tag.blknum;
                key.tag.blknum = u32::MAX; // maximal key
                let mut iter = self.db.raw_iterator();
                iter.seek_for_prev(key.to_bytes()); // localte last entry
                if iter.valid() {
                    let thiskey = CacheKey::from_slice(iter.key().unwrap());
                    let last_blknum = thiskey.tag.blknum;
                    return Ok((first_blknum, last_blknum + 1)); // upper boundary is exclusive
                }
            }
        }
        Ok((0, 0)) // empty range
    }
    ///
    /// Does relation exist at given LSN?
    ///
    /// FIXME: this actually returns true, if the relation exists at *any* LSN
    fn get_relsize_exists(&self, rel: RelTag, req_lsn: Lsn) -> Result<bool> {
        let lsn = self.wait_lsn(req_lsn)?;
        let key = CacheKey {
            tag: BufferTag {
                rel,
                blknum: u32::MAX,
            },
            lsn,
        };
        let mut iter = self.db.raw_iterator();
        iter.seek_for_prev(key.to_bytes());
        if iter.valid() {
            let key = CacheKey::from_slice(iter.key().unwrap());
            if key.tag.rel == rel {
                debug!("Relation {} exists at {}", rel, lsn);
                return Ok(true);
            }
        }
        debug!("Relation {} doesn't exist at {}", rel, lsn);
        Ok(false)
    }
    // Other public functions, for updating the repository.
    // These are used by the WAL receiver and WAL redo.
    ///
    /// Adds a WAL record to the repository
    ///
    fn put_wal_record(&self, tag: BufferTag, rec: WALRecord) {
        let lsn = rec.lsn;
        let key = CacheKey { tag, lsn };
        let content = CacheEntryContent::WALRecord(rec);
        let _res = self.db.put(key.to_bytes(), content.to_bytes());
        trace!(
            "put_wal_record rel {} blk {} at {}",
            tag.rel,
            tag.blknum,
            lsn
        );
        self.num_entries.fetch_add(1, Ordering::Relaxed);
        self.num_wal_records.fetch_add(1, Ordering::Relaxed);
    }
    ///
    /// Adds a relation-wide WAL record (like truncate) to the repository,
    /// associating it with all pages started with specified block number
    ///
    fn put_truncation(&self, rel: RelTag, lsn: Lsn, nblocks: u32) -> Result<()> {
        // What was the size of the relation before this record?
        let last_lsn = self.last_valid_lsn.load();
        let old_rel_size = self.relsize_get_nowait(rel, last_lsn)?;
        let content = CacheEntryContent::Truncation;
        // set new relation size
        trace!("Truncate relation {} to {} blocks at {}", rel, nblocks, lsn);
        for blknum in nblocks..old_rel_size {
            let key = CacheKey {
                tag: BufferTag { rel, blknum },
                lsn,
            };
            trace!("put_wal_record lsn: {}", key.lsn);
            let _res = self.db.put(key.to_bytes(), content.to_bytes());
        }
        let n = (old_rel_size - nblocks) as u64;
        self.num_entries.fetch_add(n, Ordering::Relaxed);
        self.num_wal_records.fetch_add(n, Ordering::Relaxed);
        Ok(())
    }
    ///
    /// Get page image at particular LSN
    ///
    fn get_page_image(&self, tag: BufferTag, lsn: Lsn) -> Result<Option<Bytes>> {
        let key = CacheKey { tag, lsn };
        if let Some(bytes) = self.db.get(key.to_bytes())? {
            let content = CacheEntryContent::from_slice(&bytes);
            if let CacheEntryContent::PageImage(img) = content {
                return Ok(Some(img));
            }
        }
        return Ok(None);
    }
    ///
    /// Memorize a full image of a page version
    ///
    fn put_page_image(&self, tag: BufferTag, lsn: Lsn, img: Bytes) {
        let img_len = img.len();
        let key = CacheKey { tag, lsn };
        let content = CacheEntryContent::PageImage(img);
        let mut val_buf = content.to_bytes();
        // Zero size of page image indicates that page can be removed
        if img_len == 0 {
            if (val_buf[0] & UNUSED_VERSION_FLAG) != 0 {
                // records already marked for deletion
                return;
            } else {
                // delete truncated multixact page
                val_buf[0] |= UNUSED_VERSION_FLAG;
            }
        }
        trace!("put_wal_record lsn: {}", key.lsn);
        let _res = self.db.put(key.to_bytes(), content.to_bytes());
        trace!(
            "put_page_image rel {} blk {} at {}",
            tag.rel,
            tag.blknum,
            lsn
        );
        self.num_page_images.fetch_add(1, Ordering::Relaxed);
    }
    fn put_create_database(
        &self,
        lsn: Lsn,
        db_id: Oid,
        tablespace_id: Oid,
        src_db_id: Oid,
        src_tablespace_id: Oid,
    ) -> Result<()> {
        let mut n = 0;
        for forknum in &[
            pg_constants::MAIN_FORKNUM,
            pg_constants::FSM_FORKNUM,
            pg_constants::VISIBILITYMAP_FORKNUM,
            pg_constants::INIT_FORKNUM,
            pg_constants::PG_FILENODEMAP_FORKNUM,
        ] {
            let key = CacheKey {
                tag: BufferTag {
                    rel: RelTag {
                        spcnode: src_tablespace_id,
                        dbnode: src_db_id,
                        relnode: 0,
                        forknum: *forknum,
                    },
                    blknum: 0,
                },
                lsn: Lsn(0),
            };
            let mut iter = self.db.raw_iterator();
            iter.seek(key.to_bytes());
            while iter.valid() {
                let mut key = CacheKey::from_slice(iter.key().unwrap());
                if key.tag.rel.spcnode != src_tablespace_id || key.tag.rel.dbnode != src_db_id {
                    break;
                }
                key.tag.rel.spcnode = tablespace_id;
                key.tag.rel.dbnode = db_id;
                key.lsn = lsn;
                let v = iter.value().unwrap();
                self.db.put(key.to_bytes(), v)?;
                n += 1;
                iter.next();
            }
        }
        info!(
            "Create database {}/{}, copy {} entries",
            tablespace_id, db_id, n
        );
        Ok(())
    }
    /// Remember that WAL has been received and added to the timeline up to the given LSN
    fn advance_last_valid_lsn(&self, lsn: Lsn) {
        let lsn = Lsn((lsn.0 + 7) & !7); // align position on 8 bytes
        let old = self.last_valid_lsn.advance(lsn);
        // Can't move backwards.
        if lsn < old {
            warn!(
                "attempted to move last valid LSN backwards (was {}, new {})",
                old, lsn
            );
        }
    }
    ///
    /// Remember the (end of) last valid WAL record remembered for the timeline.
    ///
    /// NOTE: this updates last_valid_lsn as well.
    ///
    fn advance_last_record_lsn(&self, lsn: Lsn) {
        let lsn = Lsn((lsn.0 + 7) & !7); // align position on 8 bytes
                                         // Can't move backwards.
        let old = self.last_record_lsn.fetch_max(lsn);
        assert!(old <= lsn);
        // Also advance last_valid_lsn
        let old = self.last_valid_lsn.advance(lsn);
        // Can't move backwards.
        if lsn < old {
            warn!(
                "attempted to move last record LSN backwards (was {}, new {})",
                old, lsn
            );
        }
    }
    fn get_last_record_lsn(&self) -> Lsn {
        self.last_record_lsn.load()
    }
    fn init_valid_lsn(&self, lsn: Lsn) {
        let old = self.last_valid_lsn.advance(lsn);
        assert!(old == Lsn(0));
        let old = self.last_record_lsn.fetch_max(lsn);
        assert!(old == Lsn(0));
    }
    fn get_last_valid_lsn(&self) -> Lsn {
        self.last_valid_lsn.load()
    }
    //
    // Get statistics to be displayed in the user interface.
    //
    // FIXME
    /*
    fn get_stats(&self) -> TimelineStats {
        TimelineStats {
            num_entries: self.num_entries.load(Ordering::Relaxed),
            num_page_images: self.num_page_images.load(Ordering::Relaxed),
            num_wal_records: self.num_wal_records.load(Ordering::Relaxed),
            num_getpage_requests: self.num_getpage_requests.load(Ordering::Relaxed),
        }
    }
    */
 }
@@ -0,0 +1,274 @@
 //
 // Restore chunks from S3
 //
 // This runs once at Page Server startup. It loads all the "base images" from
 // S3 into the in-memory page cache. It also initializes the "last valid LSN"
 // in the page cache to the LSN of the base image, so that when the WAL receiver
 // is started, it starts streaming from that LSN.
 //
 use bytes::{Buf, BytesMut};
 use log::*;
 use regex::Regex;
 use std::env;
 use std::fmt;
 use s3::bucket::Bucket;
 use s3::creds::Credentials;
 use s3::region::Region;
 use s3::S3Error;
 use tokio::runtime;
 use futures::future;
 use crate::{page_cache, PageServerConf};
 use postgres_ffi::pg_constants;
 use postgres_ffi::relfile_utils::*;
 struct Storage {
    region: Region,
    credentials: Credentials,
    bucket: String,
 }
 pub fn restore_main(conf: &PageServerConf) {
    // Create a new thread pool
    let runtime = runtime::Runtime::new().unwrap();
    runtime.block_on(async {
        let result = restore_chunk(conf).await;
        match result {
            Ok(_) => {}
            Err(err) => {
                error!("S3 error: {}", err);
            }
        }
    });
 }
 //
 // Restores one chunk from S3.
 //
 // 1. Fetch the last base image >= given LSN
 // 2. Fetch all WAL
 //
 // Load it all into the page cache.
 //
 async fn restore_chunk(conf: &PageServerConf) -> Result<(), S3Error> {
    let backend = Storage {
        region: Region::Custom {
            region: env::var("S3_REGION").unwrap(),
            endpoint: env::var("S3_ENDPOINT").unwrap(),
        },
        credentials: Credentials::new(
            Some(&env::var("S3_ACCESSKEY").unwrap()),
            Some(&env::var("S3_SECRET").unwrap()),
            None,
            None,
            None,
        )
        .unwrap(),
        bucket: "zenith-testbucket".to_string(),
    };
    info!("Restoring from S3...");
    // Create Bucket in REGION for BUCKET
    let bucket = Bucket::new_with_path_style(&backend.bucket, backend.region, backend.credentials)?;
    // List out contents of directory
    let results: Vec<s3::serde_types::ListBucketResult> = bucket
        .list("relationdata/".to_string(), Some("".to_string()))
        .await?;
    // TODO: get that from backup
    let sys_id: u64 = 42;
    let mut oldest_lsn = 0;
    let mut slurp_futures: Vec<_> = Vec::new();
    for result in results {
        for object in result.contents {
            // Download every relation file, slurping them into memory
            let key = object.key;
            let relpath = key.strip_prefix("relationdata/").unwrap();
            let parsed = parse_rel_file_path(&relpath);
            match parsed {
                Ok(p) => {
                    if oldest_lsn == 0 || p.lsn < oldest_lsn {
                        oldest_lsn = p.lsn;
                    }
                    let b = bucket.clone();
                    let f = slurp_base_file(conf, sys_id, b, key.to_string(), p);
                    slurp_futures.push(f);
                }
                Err(e) => {
                    warn!("unrecognized file: {} ({})", relpath, e);
                }
            };
        }
    }
    if oldest_lsn == 0 {
        panic!("no base backup found");
    }
    let pcache = page_cache::get_pagecache(conf, sys_id);
    pcache.init_valid_lsn(oldest_lsn);
    info!("{} files to restore...", slurp_futures.len());
    future::join_all(slurp_futures).await;
    info!("restored!");
    Ok(())
 }
 #[derive(Debug)]
 struct ParsedBaseImageFileName {
    pub spcnode: u32,
    pub dbnode: u32,
    pub relnode: u32,
    pub forknum: u8,
    pub segno: u32,
    pub lsn: u64,
 }
 // formats:
 // <oid>
 // <oid>_<fork name>
 // <oid>.<segment number>
 // <oid>_<fork name>.<segment number>
 fn parse_filename(fname: &str) -> Result<(u32, u8, u32, u64), FilePathError> {
    let re = Regex::new(r"^(?P<relnode>\d+)(_(?P<forkname>[a-z]+))?(\.(?P<segno>\d+))?_(?P<lsnhi>[[:xdigit:]]{8})(?P<lsnlo>[[:xdigit:]]{8})$").unwrap();
    let caps = re
        .captures(fname)
        .ok_or_else(|| FilePathError::new("invalid relation data file name"))?;
    let relnode_str = caps.name("relnode").unwrap().as_str();
    let relnode: u32 = relnode_str.parse()?;
    let forkname = caps.name("forkname").map(|f| f.as_str());
    let forknum = forkname_to_forknum(forkname)?;
    let segno_match = caps.name("segno");
    let segno = if segno_match.is_none() {
        0
    } else {
        segno_match.unwrap().as_str().parse::<u32>()?
    };
    let lsn_hi: u64 = caps.name("lsnhi").unwrap().as_str().parse()?;
    let lsn_lo: u64 = caps.name("lsnlo").unwrap().as_str().parse()?;
    let lsn = lsn_hi << 32 | lsn_lo;
    Ok((relnode, forknum, segno, lsn))
 }
 fn parse_rel_file_path(path: &str) -> Result<ParsedBaseImageFileName, FilePathError> {
    /*
     * Relation data files can be in one of the following directories:
     *
     * global/
     *		shared relations
     *
     * base/<db oid>/
     *		regular relations, default tablespace
     *
     * pg_tblspc/<tblspc oid>/<tblspc version>/
     *		within a non-default tablespace (the name of the directory
     *		depends on version)
     *
     * And the relation data files themselves have a filename like:
     *
     * <oid>.<segment number>
     */
    if let Some(fname) = path.strip_prefix("global/") {
        let (relnode, forknum, segno, lsn) = parse_filename(fname)?;
        Ok(ParsedBaseImageFileName {
            spcnode: pg_constants::GLOBALTABLESPACE_OID,
            dbnode: 0,
            relnode,
            forknum,
            segno,
            lsn,
        })
    } else if let Some(dbpath) = path.strip_prefix("base/") {
        let mut s = dbpath.split("/");
        let dbnode_str = s
            .next()
            .ok_or_else(|| FilePathError::new("invalid relation data file name"))?;
        let dbnode: u32 = dbnode_str.parse()?;
        let fname = s
            .next()
            .ok_or_else(|| FilePathError::new("invalid relation data file name"))?;
        if s.next().is_some() {
            return Err(FilePathError::new("invalid relation data file name"));
        };
        let (relnode, forknum, segno, lsn) = parse_filename(fname)?;
        Ok(ParsedBaseImageFileName {
            spcnode: pg_constants::DEFAULTTABLESPACE_OID,
            dbnode,
            relnode,
            forknum,
            segno,
            lsn,
        })
    } else if let Some(_) = path.strip_prefix("pg_tblspc/") {
        // TODO
        Err(FilePathError::new("tablespaces not supported"))
    } else {
        Err(FilePathError::new("invalid relation data file name"))
    }
 }
 //
 // Load a base file from S3, and insert it into the page cache
 //
 async fn slurp_base_file(
    conf: &PageServerConf,
    sys_id: u64,
    bucket: Bucket,
    s3path: String,
    parsed: ParsedBaseImageFileName,
 ) {
    // FIXME: rust-s3 opens a new connection for each request. Should reuse
    // the reqwest::Client object. But that requires changes to rust-s3 itself.
    let (data, code) = bucket.get_object(s3path.clone()).await.unwrap();
    trace!("got response: {} on {}", code, &s3path);
    assert_eq!(200, code);
    let mut bytes = BytesMut::from(data.as_slice()).freeze();
    let mut blknum: u32 = parsed.segno * (1024 * 1024 * 1024 / pg_constants::BLCKSZ as u32);
    let pcache = page_cache::get_pagecache(conf, sys_id);
    while bytes.remaining() >= 8192 {
        let tag = page_cache::BufferTag {
            rel: page_cache::RelTag {
                spcnode: parsed.spcnode,
                dbnode: parsed.dbnode,
                relnode: parsed.relnode,
                forknum: parsed.forknum,
            },
            blknum,
        };
        pcache.put_page_image(tag, parsed.lsn, bytes.copy_to_bytes(8192));
        blknum += 1;
    }
 }
@@ -1,475 +0,0 @@
 //!
 //! An implementation of the ObjectStore interface, backed by RocksDB
 //!
 use crate::object_key::*;
 use crate::object_store::ObjectStore;
 use crate::relish::*;
 use crate::PageServerConf;
 use anyhow::{bail, Result};
 use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
 use std::sync::{Arc, Mutex};
 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;
 use zenith_utils::zid::ZTenantId;
 use zenith_utils::zid::ZTimelineId;
 #[derive(Debug, Clone, Serialize, Deserialize)]
 struct StorageKey {
    obj_key: ObjectKey,
    lsn: Lsn,
 }
 impl StorageKey {
    /// The first key for a given timeline
    fn timeline_start(timeline: ZTimelineId) -> Self {
        Self {
            obj_key: ObjectKey {
                timeline,
                tag: ObjectTag::TimelineMetadataTag,
            },
            lsn: Lsn(0),
        }
    }
 }
 ///
 /// RocksDB very inefficiently delete random record. Instead of it we have to use merge
 /// filter, which allows to throw away records at LSM merge phase.
 /// Unfortunately, it is hard (if ever possible)  to determine whether version can be removed
 /// at merge time. Version ca be removed if:
 /// 1. It is above PITR horizon (we need to get current LSN and gc_horizon from config)
 /// 2. Page is reconstructed at horizon (all WAL records above horizon are applied and can be removed)
 ///
 /// So we have GC process which reconstructs pages at horizon and mark deteriorated WAL record
 /// for deletion. To mark object for deletion we can either set some flag in object itself.
 /// But it is complicated with new object value format, because RocksDB storage knows nothing about
 /// this format. Also updating whole record just to set one bit seems to be inefficient in any case.
 /// This is why we keep keys of marked for deletion versions in HashSet in memory.
 /// When LSM merge filter found key in this map, it removes it from the set preventing memory overflow.
 ///
 struct GarbageCollector {
    garbage: Mutex<HashSet<Vec<u8>>>,
 }
 impl GarbageCollector {
    fn new() -> GarbageCollector {
        GarbageCollector {
            garbage: Mutex::new(HashSet::new()),
        }
    }
    /// Called by GC to mark version as delete
    fn mark_for_deletion(&self, key: &[u8]) {
        let mut garbage = self.garbage.lock().unwrap();
        garbage.insert(key.to_vec());
    }
    /// Called by LSM merge filter. If it finds key in the set, then
    /// it doesn't merge it and removes from this set.
    fn was_deleted(&self, key: &[u8]) -> bool {
        let key = key.to_vec();
        let mut garbage = self.garbage.lock().unwrap();
        garbage.remove(&key)
    }
 }
 pub struct RocksObjectStore {
    _conf: &'static PageServerConf,
    // RocksDB handle
    db: rocksdb::DB,
    gc: Arc<GarbageCollector>,
 }
 impl ObjectStore for RocksObjectStore {
    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>> {
        let val = self.db.get(StorageKey::ser(&StorageKey {
            obj_key: key.clone(),
            lsn,
        })?)?;
        if let Some(val) = val {
            Ok(val)
        } else {
            bail!("could not find page {:?}", key);
        }
    }
    fn get_next_key(&self, key: &ObjectKey) -> Result<Option<ObjectKey>> {
        let mut iter = self.db.raw_iterator();
        let search_key = StorageKey {
            obj_key: key.clone(),
            lsn: Lsn(0),
        };
        iter.seek(search_key.ser()?);
        if !iter.valid() {
            Ok(None)
        } else {
            let key = StorageKey::des(iter.key().unwrap())?;
            Ok(Some(key.obj_key.clone()))
        }
    }
    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()> {
        self.db.put(
            StorageKey::ser(&StorageKey {
                obj_key: key.clone(),
                lsn,
            })?,
            value,
        )?;
        Ok(())
    }
    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()> {
        self.gc.mark_for_deletion(&StorageKey::ser(&StorageKey {
            obj_key: key.clone(),
            lsn,
        })?);
        Ok(())
    }
    /// Iterate through page versions of given page, starting from the given LSN.
    /// The versions are walked in descending LSN order.
    fn object_versions<'a>(
        &'a self,
        key: &ObjectKey,
        lsn: Lsn,
    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>> {
        let iter = RocksObjectVersionIter::new(&self.db, key, lsn)?;
        Ok(Box::new(iter))
    }
    /// Iterate through all timeline objects
    fn list_objects<'a>(
        &'a self,
        timeline: ZTimelineId,
        lsn: Lsn,
    ) -> Result<Box<dyn Iterator<Item = ObjectTag> + 'a>> {
        let iter = RocksObjectIter::new(&self.db, timeline, lsn)?;
        Ok(Box::new(iter))
    }
    /// Get a list of all distinct relations in given tablespace and database.
    ///
    /// TODO: This implementation is very inefficient, it scans
    /// through all entries in the given database. In practice, this
    /// is used for CREATE DATABASE, and usually the template database is small.
    /// But if it's not, this will be slow.
    fn list_rels(
        &self,
        timelineid: ZTimelineId,
        spcnode: u32,
        dbnode: u32,
        lsn: Lsn,
    ) -> Result<HashSet<RelTag>> {
        // FIXME: This scans everything. Very slow
        let mut rels: HashSet<RelTag> = HashSet::new();
        let mut search_rel_tag = RelTag {
            spcnode,
            dbnode,
            relnode: 0,
            forknum: 0u8,
        };
        let mut iter = self.db.raw_iterator();
        loop {
            let search_key = StorageKey {
                obj_key: ObjectKey {
                    timeline: timelineid,
                    tag: ObjectTag::RelationMetadata(RelishTag::Relation(search_rel_tag)),
                },
                lsn: Lsn(0),
            };
            iter.seek(search_key.ser()?);
            if !iter.valid() {
                break;
            }
            let key = StorageKey::des(iter.key().unwrap())?;
            if let ObjectTag::RelationMetadata(RelishTag::Relation(rel_tag)) = key.obj_key.tag {
                if spcnode != 0 && rel_tag.spcnode != spcnode
                    || dbnode != 0 && rel_tag.dbnode != dbnode
                {
                    break;
                }
                if key.lsn <= lsn {
                    // visible in this snapshot
                    rels.insert(rel_tag);
                }
                search_rel_tag = rel_tag;
                // skip to next relation
                // FIXME: What if relnode is u32::MAX ?
                search_rel_tag.relnode += 1;
            } else {
                // no more relation metadata entries
                break;
            }
        }
        Ok(rels)
    }
    /// Get a list of all distinct NON-relations in timeline
    /// that are visible at given lsn.
    ///
    /// TODO: This implementation is very inefficient, it scans
    /// through all non-rel page versions in the system. In practice, this
    /// is used when initializing a new compute node, and the non-rel files
    /// are never very large nor change very frequently, so this will do for now.
    fn list_nonrels(&self, timelineid: ZTimelineId, lsn: Lsn) -> Result<HashSet<RelishTag>> {
        let mut rels: HashSet<RelishTag> = HashSet::new();
        let search_key = StorageKey {
            obj_key: ObjectKey {
                timeline: timelineid,
                tag: ObjectTag::Buffer(FIRST_NONREL_RELISH_TAG, 0),
            },
            lsn: Lsn(0),
        };
        let mut iter = self.db.raw_iterator();
        iter.seek(search_key.ser()?);
        while iter.valid() {
            let key = StorageKey::des(iter.key().unwrap())?;
            if key.obj_key.timeline != timelineid {
                // reached end of this timeline in the store
                break;
            }
            if let ObjectTag::Buffer(rel_tag, _blknum) = key.obj_key.tag {
                if key.lsn <= lsn {
                    // visible in this snapshot
                    rels.insert(rel_tag);
                }
            }
            // TODO: we could skip to next relation here like we do in list_rels(),
            // but hopefully there are not that many SLRU segments or other non-rel
            // entries for it to matter.
            iter.next();
        }
        Ok(rels)
    }
    /// Iterate through versions of all objects in a timeline.
    ///
    /// Returns objects in increasing key-version order.
    /// Returns all versions up to and including the specified LSN.
    fn objects<'a>(
        &'a self,
        timeline: ZTimelineId,
        lsn: Lsn,
    ) -> Result<Box<dyn Iterator<Item = Result<(ObjectTag, Lsn, Vec<u8>)>> + 'a>> {
        let start_key = StorageKey::timeline_start(timeline);
        let start_key_bytes = StorageKey::ser(&start_key)?;
        let iter = self.db.iterator(rocksdb::IteratorMode::From(
            &start_key_bytes,
            rocksdb::Direction::Forward,
        ));
        Ok(Box::new(RocksObjects {
            iter,
            timeline,
            lsn,
        }))
    }
    fn compact(&self) {
        self.db.compact_range::<&[u8], &[u8]>(None, None);
    }
 }
 impl RocksObjectStore {
    /// Open a RocksDB database.
    pub fn open(conf: &'static PageServerConf, tenantid: &ZTenantId) -> Result<RocksObjectStore> {
        let opts = Self::get_rocksdb_opts();
        let obj_store = Self::new(conf, opts, tenantid)?;
        Ok(obj_store)
    }
    /// Create a new, empty RocksDB database.
    pub fn create(conf: &'static PageServerConf, tenantid: &ZTenantId) -> Result<RocksObjectStore> {
        let path = conf.tenant_path(&tenantid).join("rocksdb-storage");
        std::fs::create_dir(&path)?;
        let mut opts = Self::get_rocksdb_opts();
        opts.create_if_missing(true);
        opts.set_error_if_exists(true);
        let obj_store = Self::new(conf, opts, tenantid)?;
        Ok(obj_store)
    }
    fn new(
        conf: &'static PageServerConf,
        mut opts: rocksdb::Options,
        tenantid: &ZTenantId,
    ) -> Result<RocksObjectStore> {
        let path = conf.tenant_path(&tenantid).join("rocksdb-storage");
        let gc = Arc::new(GarbageCollector::new());
        let gc_ref = gc.clone();
        opts.set_compaction_filter("ttl", move |_level: u32, key: &[u8], _val: &[u8]| {
            if gc_ref.was_deleted(key) {
                rocksdb::compaction_filter::Decision::Remove
            } else {
                rocksdb::compaction_filter::Decision::Keep
            }
        });
        let db = rocksdb::DB::open(&opts, &path)?;
        let obj_store = RocksObjectStore {
            _conf: conf,
            db,
            gc,
        };
        Ok(obj_store)
    }
    /// common options used by `open` and `create`
    fn get_rocksdb_opts() -> rocksdb::Options {
        let mut opts = rocksdb::Options::default();
        opts.set_use_fsync(true);
        opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
        opts
    }
 }
 ///
 /// Iterator for `object_versions`. Returns all page versions of a given block, in
 /// reverse LSN order.
 ///
 struct RocksObjectVersionIter<'a> {
    obj_key: ObjectKey,
    dbiter: rocksdb::DBRawIterator<'a>,
    first_call: bool,
 }
 impl<'a> RocksObjectVersionIter<'a> {
    fn new(
        db: &'a rocksdb::DB,
        obj_key: &ObjectKey,
        lsn: Lsn,
    ) -> Result<RocksObjectVersionIter<'a>> {
        let key = StorageKey {
            obj_key: obj_key.clone(),
            lsn,
        };
        let mut dbiter = db.raw_iterator();
        dbiter.seek_for_prev(StorageKey::ser(&key)?); // locate last entry
        Ok(RocksObjectVersionIter {
            first_call: true,
            obj_key: obj_key.clone(),
            dbiter,
        })
    }
 }
 impl<'a> Iterator for RocksObjectVersionIter<'a> {
    type Item = (Lsn, Vec<u8>);
    fn next(&mut self) -> std::option::Option<Self::Item> {
        if self.first_call {
            self.first_call = false;
        } else {
            self.dbiter.prev(); // walk backwards
        }
        if !self.dbiter.valid() {
            return None;
        }
        let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
        if key.obj_key.tag != self.obj_key.tag {
            return None;
        }
        let val = self.dbiter.value().unwrap();
        let result = val.to_vec();
        Some((key.lsn, result))
    }
 }
 struct RocksObjects<'r> {
    iter: rocksdb::DBIterator<'r>,
    timeline: ZTimelineId,
    lsn: Lsn,
 }
 impl<'r> Iterator for RocksObjects<'r> {
    // TODO consider returning Box<[u8]>
    type Item = Result<(ObjectTag, Lsn, Vec<u8>)>;
    fn next(&mut self) -> Option<Self::Item> {
        self.next_result().transpose()
    }
 }
 impl<'r> RocksObjects<'r> {
    fn next_result(&mut self) -> Result<Option<(ObjectTag, Lsn, Vec<u8>)>> {
        for (key_bytes, v) in &mut self.iter {
            let key = StorageKey::des(&key_bytes)?;
            if key.obj_key.timeline != self.timeline {
                return Ok(None);
            }
            if key.lsn > self.lsn {
                // TODO can speed up by seeking iterator
                continue;
            }
            return Ok(Some((key.obj_key.tag, key.lsn, v.to_vec())));
        }
        Ok(None)
    }
 }
 ///
 /// Iterator for `list_objects`. Returns all objects preceeding specified LSN
 ///
 struct RocksObjectIter<'a> {
    timeline: ZTimelineId,
    key: StorageKey,
    lsn: Lsn,
    dbiter: rocksdb::DBRawIterator<'a>,
 }
 impl<'a> RocksObjectIter<'a> {
    fn new(db: &'a rocksdb::DB, timeline: ZTimelineId, lsn: Lsn) -> Result<RocksObjectIter<'a>> {
        let key = StorageKey {
            obj_key: ObjectKey {
                timeline,
                tag: ObjectTag::FirstTag,
            },
            lsn: Lsn(0),
        };
        let dbiter = db.raw_iterator();
        Ok(RocksObjectIter {
            key,
            timeline,
            lsn,
            dbiter,
        })
    }
 }
 impl<'a> Iterator for RocksObjectIter<'a> {
    type Item = ObjectTag;
    fn next(&mut self) -> std::option::Option<Self::Item> {
        loop {
            self.dbiter.seek(StorageKey::ser(&self.key).unwrap());
            if !self.dbiter.valid() {
                return None;
            }
            let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
            if key.obj_key.timeline != self.timeline {
                // End of this timeline
                return None;
            }
            self.key = key.clone();
            self.key.lsn = Lsn(u64::MAX); // next seek should skip all versions
            if key.lsn <= self.lsn {
                // visible in this snapshot
                return Some(key.obj_key.tag);
            }
        }
    }
 }
@@ -0,0 +1,307 @@
 use crate::tui_event::{Event, Events};
 use crate::tui_logger::TuiLogger;
 use crate::tui_logger::TuiLoggerWidget;
 use lazy_static::lazy_static;
 use std::sync::Arc;
 use std::{error::Error, io};
 use termion::{event::Key, input::MouseTerminal, raw::IntoRawMode, screen::AlternateScreen};
 use tui::backend::TermionBackend;
 use tui::buffer::Buffer;
 use tui::layout::{Constraint, Direction, Layout, Rect};
 use tui::style::{Color, Modifier, Style};
 use tui::text::{Span, Spans, Text};
 use tui::widgets::{Block, BorderType, Borders, Paragraph, Widget};
 use tui::Terminal;
 use slog::Drain;
 lazy_static! {
    pub static ref PAGESERVICE_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
    pub static ref WALRECEIVER_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
    pub static ref WALREDO_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
    pub static ref CATCHALL_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
 }
 pub fn init_logging() -> slog_scope::GlobalLoggerGuard {
    let pageservice_drain =
        slog::Filter::new(PAGESERVICE_DRAIN.as_ref(), |record: &slog::Record| {
            if record.level().is_at_least(slog::Level::Debug)
                && record.module().starts_with("pageserver::page_service")
            {
                return true;
            }
            false
        })
        .fuse();
    let walredo_drain = slog::Filter::new(WALREDO_DRAIN.as_ref(), |record: &slog::Record| {
        if record.level().is_at_least(slog::Level::Debug)
            && record.module().starts_with("pageserver::walredo")
        {
            return true;
        }
        false
    })
    .fuse();
    let walreceiver_drain =
        slog::Filter::new(WALRECEIVER_DRAIN.as_ref(), |record: &slog::Record| {
            if record.level().is_at_least(slog::Level::Debug)
                && record.module().starts_with("pageserver::walreceiver")
            {
                return true;
            }
            false
        })
        .fuse();
    let catchall_drain = slog::Filter::new(CATCHALL_DRAIN.as_ref(), |record: &slog::Record| {
        if record.level().is_at_least(slog::Level::Info) {
            return true;
        }
        if record.level().is_at_least(slog::Level::Debug)
            && record.module().starts_with("pageserver")
        {
            return true;
        }
        false
    })
    .fuse();
    let drain = pageservice_drain;
    let drain = slog::Duplicate::new(drain, walreceiver_drain).fuse();
    let drain = slog::Duplicate::new(drain, walredo_drain).fuse();
    let drain = slog::Duplicate::new(drain, catchall_drain).fuse();
    let drain = slog_async::Async::new(drain).chan_size(1000).build().fuse();
    let drain = slog::Filter::new(drain, |record: &slog::Record| {
        if record.level().is_at_least(slog::Level::Info) {
            return true;
        }
        if record.level().is_at_least(slog::Level::Debug)
            && record.module().starts_with("pageserver")
        {
            return true;
        }
        false
    })
    .fuse();
    let logger = slog::Logger::root(drain, slog::o!());
    slog_scope::set_global_logger(logger)
 }
 pub fn ui_main() -> Result<(), Box<dyn Error>> {
    // Terminal initialization
    let stdout = io::stdout().into_raw_mode()?;
    let stdout = MouseTerminal::from(stdout);
    let stdout = AlternateScreen::from(stdout);
    let backend = TermionBackend::new(stdout);
    let mut terminal = Terminal::new(backend)?;
    // Setup event handlers
    let events = Events::new();
    loop {
        terminal.draw(|f| {
            let size = f.size();
            // +----------------+----------------+
            // |                |                |
            // |  top_top_left  | top_top_right  |
            // |                |                |
            // +----------------+----------------|
            // |                |                |
            // |  top_bot_left  | top_left_right |
            // |                |                |
            // +----------------+----------------+
            // |                                 |
            // |             bottom              |
            // |                                 |
            // +---------------------------------+
            let chunks = Layout::default()
                .direction(Direction::Vertical)
                .constraints([Constraint::Percentage(70), Constraint::Percentage(30)].as_ref())
                .split(size);
            let top_chunk = chunks[0];
            let bottom_chunk = chunks[1];
            let top_chunks = Layout::default()
                .direction(Direction::Horizontal)
                .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
                .split(top_chunk);
            let top_left_chunk = top_chunks[0];
            let top_right_chunk = top_chunks[1];
            let c = Layout::default()
                .direction(Direction::Vertical)
                .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
                .split(top_left_chunk);
            let top_top_left_chunk = c[0];
            let top_bot_left_chunk = c[1];
            let c = Layout::default()
                .direction(Direction::Vertical)
                .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
                .split(top_right_chunk);
            let top_top_right_chunk = c[0];
            let top_bot_right_chunk = c[1];
            f.render_widget(
                LogWidget::new(PAGESERVICE_DRAIN.as_ref(), "Page Service"),
                top_top_left_chunk,
            );
            f.render_widget(
                LogWidget::new(WALREDO_DRAIN.as_ref(), "WAL Redo"),
                top_bot_left_chunk,
            );
            f.render_widget(
                LogWidget::new(WALRECEIVER_DRAIN.as_ref(), "WAL Receiver"),
                top_top_right_chunk,
            );
            f.render_widget(MetricsWidget {}, top_bot_right_chunk);
            f.render_widget(
                LogWidget::new(CATCHALL_DRAIN.as_ref(), "All Log").show_module(true),
                bottom_chunk,
            );
        })?;
        // If ther user presses 'q', quit.
        // silence clippy's suggestion to rewrite this as an if-statement. Match
        // makes more sense as soon as we get another command than 'q'.
        #[allow(clippy::single_match)]
        #[allow(clippy::collapsible_match)]
        if let Event::Input(key) = events.next()? {
            match key {
                Key::Char('q') => {
                    break;
                }
                _ => (),
            }
        }
    }
    terminal.show_cursor().unwrap();
    terminal.clear().unwrap();
    Ok(())
 }
 #[allow(dead_code)]
 struct LogWidget<'a> {
    logger: &'a TuiLogger,
    title: &'a str,
    show_module: bool,
 }
 impl<'a> LogWidget<'a> {
    fn new(logger: &'a TuiLogger, title: &'a str) -> LogWidget<'a> {
        LogWidget {
            logger,
            title,
            show_module: false,
        }
    }
    fn show_module(mut self, b: bool) -> LogWidget<'a> {
        self.show_module = b;
        self
    }
 }
 impl<'a> Widget for LogWidget<'a> {
    fn render(self, area: Rect, buf: &mut Buffer) {
        let w = TuiLoggerWidget::default(self.logger)
            .block(
                Block::default()
                    .borders(Borders::ALL)
                    .title(self.title)
                    .border_type(BorderType::Rounded),
            )
            .show_module(true)
            .style_error(Style::default().fg(Color::Red))
            .style_warn(Style::default().fg(Color::Yellow))
            .style_info(Style::default().fg(Color::Green));
        w.render(area, buf);
    }
 }
 // Render a widget to show some metrics
 struct MetricsWidget {}
 fn _get_metric_u64(title: &str, value: u64) -> Spans {
    Spans::from(vec![
        Span::styled(format!("{:<20}", title), Style::default()),
        Span::raw(": "),
        Span::styled(
            value.to_string(),
            Style::default().add_modifier(Modifier::BOLD),
        ),
    ])
 }
 // This is not used since LSNs were removed from page cache stats.
 // Maybe it will be used in the future?
 fn _get_metric_str<'a>(title: &str, value: &'a str) -> Spans<'a> {
    Spans::from(vec![
        Span::styled(format!("{:<20}", title), Style::default()),
        Span::raw(": "),
        Span::styled(value, Style::default().add_modifier(Modifier::BOLD)),
    ])
 }
 impl tui::widgets::Widget for MetricsWidget {
    fn render(self, area: Rect, buf: &mut Buffer) {
        let block = Block::default()
            .borders(Borders::ALL)
            .title("Page Cache Metrics")
            .border_type(BorderType::Rounded);
        let inner_area = block.inner(area);
        block.render(area, buf);
        #[allow(unused_mut)]
        let mut lines: Vec<Spans> = Vec::new();
        // FIXME
        //let page_cache_stats = crate::page_cache::get_stats();
        // This is not used since LSNs were removed from page cache stats.
        // Maybe it will be used in the future?
        /*
        let lsnrange = format!(
            "{} - {}",
            page_cache_stats.first_valid_lsn, page_cache_stats.last_valid_lsn
        );
        let last_valid_recordlsn_str = page_cache_stats.last_record_lsn.to_string();
        lines.push(get_metric_str("Valid LSN range", &lsnrange));
        lines.push(get_metric_str("Last record LSN", &last_valid_recordlsn_str));
        */
        /*
        lines.push(get_metric_u64(
            "# of cache entries",
            page_cache_stats.num_entries,
        ));
        lines.push(get_metric_u64(
            "# of page images",
            page_cache_stats.num_page_images,
        ));
        lines.push(get_metric_u64(
            "# of WAL records",
            page_cache_stats.num_wal_records,
        ));
        lines.push(get_metric_u64(
            "# of GetPage@LSN calls",
            page_cache_stats.num_getpage_requests,
        ));
        */
        let text = Text::from(lines);
        Paragraph::new(text).render(inner_area, buf);
    }
 }
@@ -0,0 +1,96 @@
 use std::io;
 use std::sync::mpsc;
 use std::sync::{
    atomic::{AtomicBool, Ordering},
    Arc,
 };
 use std::thread;
 use std::time::Duration;
 use termion::event::Key;
 use termion::input::TermRead;
 pub enum Event<I> {
    Input(I),
    Tick,
 }
 /// A small event handler that wrap termion input and tick events. Each event
 /// type is handled in its own thread and returned to a common `Receiver`
 #[allow(dead_code)]
 pub struct Events {
    rx: mpsc::Receiver<Event<Key>>,
    input_handle: thread::JoinHandle<()>,
    ignore_exit_key: Arc<AtomicBool>,
    tick_handle: thread::JoinHandle<()>,
 }
 #[derive(Debug, Clone, Copy)]
 pub struct Config {
    pub exit_key: Key,
    pub tick_rate: Duration,
 }
 impl Default for Config {
    fn default() -> Config {
        Config {
            exit_key: Key::Char('q'),
            tick_rate: Duration::from_millis(250),
        }
    }
 }
 impl Events {
    pub fn new() -> Events {
        Events::with_config(Config::default())
    }
    pub fn with_config(config: Config) -> Events {
        let (tx, rx) = mpsc::channel();
        let ignore_exit_key = Arc::new(AtomicBool::new(false));
        let input_handle = {
            let tx = tx.clone();
            let ignore_exit_key = ignore_exit_key.clone();
            thread::spawn(move || {
                let stdin = io::stdin();
                for evt in stdin.keys() {
                    if let Ok(key) = evt {
                        if let Err(err) = tx.send(Event::Input(key)) {
                            eprintln!("{}", err);
                            return;
                        }
                        if !ignore_exit_key.load(Ordering::Relaxed) && key == config.exit_key {
                            return;
                        }
                    }
                }
            })
        };
        let tick_handle = {
            thread::spawn(move || loop {
                if tx.send(Event::Tick).is_err() {
                    break;
                }
                thread::sleep(config.tick_rate);
            })
        };
        Events {
            rx,
            input_handle,
            ignore_exit_key,
            tick_handle,
        }
    }
    pub fn next(&self) -> Result<Event<Key>, mpsc::RecvError> {
        self.rx.recv()
    }
    pub fn disable_exit_key(&mut self) {
        self.ignore_exit_key.store(true, Ordering::Relaxed);
    }
    pub fn enable_exit_key(&mut self) {
        self.ignore_exit_key.store(false, Ordering::Relaxed);
    }
 }
@@ -0,0 +1,199 @@
 //
 // A TUI Widget that displays log entries
 //
 // This is heavily inspired by gin66's tui_logger crate at https://github.com/gin66/tui-logger,
 // but I wrote this based on the 'slog' module, which simplified things a lot. tui-logger also
 // implemented the slog Drain trait, but it had a model of one global buffer for the records.
 // With this implementation, each TuiLogger is a separate ring buffer and separate slog Drain.
 // Also, I didn't do any of the "hot log" stuff that gin66's implementation had, you can use an
 // AsyncDrain to buffer and handle overflow if desired.
 //
 use chrono::offset::Local;
 use chrono::DateTime;
 use slog::{Drain, Level, OwnedKVList, Record};
 use slog_async::AsyncRecord;
 use std::collections::VecDeque;
 use std::sync::Mutex;
 use std::time::SystemTime;
 use tui::buffer::Buffer;
 use tui::layout::Rect;
 use tui::style::{Modifier, Style};
 use tui::text::{Span, Spans};
 use tui::widgets::{Block, Paragraph, Widget, Wrap};
 // Size of the log ring buffer, in # of records
 static BUFFER_SIZE: usize = 1000;
 pub struct TuiLogger {
    events: Mutex<VecDeque<(SystemTime, AsyncRecord)>>,
 }
 impl<'a> Default for TuiLogger {
    fn default() -> TuiLogger {
        TuiLogger {
            events: Mutex::new(VecDeque::with_capacity(BUFFER_SIZE)),
        }
    }
 }
 impl Drain for TuiLogger {
    type Ok = ();
    type Err = slog::Error;
    fn log(&self, record: &Record, values: &OwnedKVList) -> Result<Self::Ok, Self::Err> {
        let mut events = self.events.lock().unwrap();
        let now = SystemTime::now();
        let asyncrec = AsyncRecord::from(record, values);
        events.push_front((now, asyncrec));
        if events.len() > BUFFER_SIZE {
            events.pop_back();
        }
        Ok(())
    }
 }
 // TuiLoggerWidget renders a TuiLogger ring buffer
 pub struct TuiLoggerWidget<'b> {
    block: Option<Block<'b>>,
    /// Base style of the widget
    style: Style,
    /// Level based style
    style_error: Option<Style>,
    style_warn: Option<Style>,
    style_debug: Option<Style>,
    style_trace: Option<Style>,
    style_info: Option<Style>,
    show_module: bool,
    logger: &'b TuiLogger,
 }
 impl<'b> TuiLoggerWidget<'b> {
    pub fn default(logger: &'b TuiLogger) -> TuiLoggerWidget<'b> {
        TuiLoggerWidget {
            block: None,
            style: Default::default(),
            style_error: None,
            style_warn: None,
            style_debug: None,
            style_trace: None,
            style_info: None,
            show_module: true,
            logger,
        }
    }
 }
 impl<'b> TuiLoggerWidget<'b> {
    pub fn block(mut self, block: Block<'b>) -> TuiLoggerWidget<'b> {
        self.block = Some(block);
        self
    }
    #[allow(unused)]
    pub fn style(mut self, style: Style) -> TuiLoggerWidget<'b> {
        self.style = style;
        self
    }
    pub fn style_error(mut self, style: Style) -> TuiLoggerWidget<'b> {
        self.style_error = Some(style);
        self
    }
    pub fn style_warn(mut self, style: Style) -> TuiLoggerWidget<'b> {
        self.style_warn = Some(style);
        self
    }
    pub fn style_info(mut self, style: Style) -> TuiLoggerWidget<'b> {
        self.style_info = Some(style);
        self
    }
    #[allow(unused)]
    pub fn style_trace(mut self, style: Style) -> TuiLoggerWidget<'b> {
        self.style_trace = Some(style);
        self
    }
    #[allow(unused)]
    pub fn style_debug(mut self, style: Style) -> TuiLoggerWidget<'b> {
        self.style_debug = Some(style);
        self
    }
    pub fn show_module(mut self, b: bool) -> TuiLoggerWidget<'b> {
        self.show_module = b;
        self
    }
 }
 impl<'b> Widget for TuiLoggerWidget<'b> {
    fn render(mut self, area: Rect, buf: &mut Buffer) {
        buf.set_style(area, self.style);
        let list_area = match self.block.take() {
            Some(b) => {
                let inner_area = b.inner(area);
                b.render(area, buf);
                inner_area
            }
            None => area,
        };
        if list_area.width == 0 || list_area.height == 0 {
            return;
        }
        let la_height = list_area.height as usize;
        //
        // Iterate through the records in the buffer. The records are
        // pushed to the front, so the newest records come first.
        //
        let mut lines: Vec<Spans> = Vec::new();
        let style_msg = Style::default().add_modifier(Modifier::BOLD);
        {
            let events = self.logger.events.lock().unwrap();
            for evt in events.iter() {
                let (timestamp, rec) = evt;
                rec.as_record_values(|rec, _kwlist| {
                    let mut line: Vec<Span> = Vec::new();
                    let datetime: DateTime<Local> = timestamp.clone().into();
                    let ts = format!("{}", datetime.format("%H:%M:%S%.3f "));
                    line.push(Span::raw(ts));
                    let (lvl_style, txt, with_loc) = match rec.level() {
                        Level::Critical => (self.style_error, "CRIT ", true),
                        Level::Error => (self.style_error, "ERROR", true),
                        Level::Warning => (self.style_warn, "WARN ", true),
                        Level::Info => (self.style_info, "INFO ", false),
                        Level::Debug => (self.style_debug, "DEBUG", true),
                        Level::Trace => (self.style_trace, "TRACE", true),
                    };
                    line.push(Span::styled(txt, lvl_style.unwrap_or_default()));
                    if self.show_module {
                        line.push(Span::raw(" "));
                        line.push(Span::raw(rec.module()));
                    }
                    if with_loc {
                        let loc = format!(" {}:{}", rec.file(), rec.line());
                        line.push(Span::raw(loc));
                    }
                    let msg = format!(" {}", rec.msg());
                    line.push(Span::styled(msg, style_msg));
                    lines.push(Spans::from(line));
                });
                if lines.len() == la_height {
                    break;
                }
            }
        }
        lines.reverse();
        let text = tui::text::Text::from(lines);
        Paragraph::new(text)
            .wrap(Wrap { trim: true })
            .render(list_area, buf);
    }
 }
@@ -1,37 +1,62 @@
 //!
 //! WAL decoder. For each WAL record, it decodes the record to figure out which data blocks
 //! the record affects, to add the records to the page cache.
 //!
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use crc32c::*;
 use log::*;
-use postgres_ffi::pg_constants;
+use postgres_ffi::xlog_utils::XLogRecord;
-use postgres_ffi::xlog_utils::*;
+use postgres_ffi::*;
 use postgres_ffi::XLogLongPageHeaderData;
 use postgres_ffi::XLogPageHeaderData;
 use postgres_ffi::XLogRecord;
 use postgres_ffi::{Oid, TransactionId};
 use std::cmp::min;
 use std::str;
 use thiserror::Error;
 use zenith_utils::lsn::Lsn;
 pub type Oid = u32;
 pub type TransactionId = u32;
 pub type BlockNumber = u32;
 pub type OffsetNumber = u16;
 pub type MultiXactId = TransactionId;
 pub type MultiXactOffset = u32;
 pub type MultiXactStatus = u32;
 pub type TimeLineID = u32;
 pub type PgTime = i64;
 // From PostgreSQL headers
 #[repr(C)]
 #[derive(Debug)]
 pub struct XLogPageHeaderData {
    xlp_magic: u16,      /* magic value for correctness checks */
    xlp_info: u16,       /* flag bits, see below */
    xlp_tli: TimeLineID, /* TimeLineID of first record on page */
    xlp_pageaddr: u64,   /* XLOG address of this page */
    xlp_rem_len: u32,    /* total len of remaining data for record */
 }
 // FIXME: this assumes MAXIMUM_ALIGNOF 8. There are 4 padding bytes at end
 #[allow(non_upper_case_globals)]
 const SizeOfXLogShortPHD: usize = 2 + 2 + 4 + 8 + 4 + 4;
 #[repr(C)]
 #[derive(Debug)]
 pub struct XLogLongPageHeaderData {
    std: XLogPageHeaderData, /* standard header fields */
    xlp_sysid: u64,          /* system identifier from pg_control */
    xlp_seg_size: u32,       /* just as a cross-check */
    xlp_xlog_blcksz: u32,    /* just as a cross-check */
 }
 // FIXME: this assumes MAXIMUM_ALIGNOF 8.
 #[allow(non_upper_case_globals)]
 const SizeOfXLogLongPHD: usize = (2 + 2 + 4 + 8 + 4) + 4 + 8 + 4 + 4;
 #[allow(dead_code)]
 pub struct WalStreamDecoder {
    lsn: Lsn,
    startlsn: Lsn, // LSN where this record starts
    contlen: u32,
    padlen: u32,
    inputbuf: BytesMut,
    recordbuf: BytesMut,
-    crc_check: bool,
+    recordbuf: BytesMut,
 }
 #[derive(Error, Debug, Clone)]
@@ -46,22 +71,24 @@ pub struct WalDecodeError {
 // FIXME: This isn't a proper rust stream
 //
 impl WalStreamDecoder {
-    pub fn new(lsn: Lsn, crc_check: bool) -> WalStreamDecoder {
+    pub fn new(lsn: Lsn) -> WalStreamDecoder {
        WalStreamDecoder {
            lsn,
            startlsn: Lsn(0),
            contlen: 0,
            padlen: 0,
            inputbuf: BytesMut::new(),
            recordbuf: BytesMut::new(),
            crc_check,
        }
    }
-    pub fn available(&self) -> Lsn {
+    pub fn set_position(&mut self, lsn: Lsn) {
-        self.lsn + self.inputbuf.remaining() as u64
+        self.lsn = lsn;
        self.contlen = 0;
        self.padlen = 0;
        self.inputbuf.clear();
    }
    pub fn feed_bytes(&mut self, buf: &[u8]) {
@@ -72,7 +99,7 @@ impl WalStreamDecoder {
    /// decoder so far.
    ///
    /// Returns one of the following:
-    ///     Ok((Lsn, Bytes)): a tuple containing the LSN of next record, and the record itself
+    ///     Ok((u64, Bytes)): a tuple containing the LSN of next record, and the record itself
    ///     Ok(None): there is not enough data in the input buffer. Feed more by calling the `feed_bytes` function
    ///     Err(WalDecodeError): an error occured while decoding, meaning the input was invalid.
    ///
@@ -82,12 +109,11 @@ impl WalStreamDecoder {
            if self.lsn.segment_offset(pg_constants::WAL_SEGMENT_SIZE) == 0 {
                // parse long header
-                if self.inputbuf.remaining() < XLOG_SIZE_OF_XLOG_LONG_PHD {
+                if self.inputbuf.remaining() < SizeOfXLogLongPHD {
                    return Ok(None);
                }
-                let hdr = XLogLongPageHeaderData::from_bytes(&mut self.inputbuf);
+                let hdr = self.decode_XLogLongPageHeaderData();
                if hdr.std.xlp_pageaddr != self.lsn.0 {
                    return Err(WalDecodeError {
                        msg: "invalid xlog segment header".into(),
@@ -96,32 +122,24 @@ impl WalStreamDecoder {
                }
                // TODO: verify the remaining fields in the header
-                self.lsn += XLOG_SIZE_OF_XLOG_LONG_PHD as u64;
+                self.lsn += SizeOfXLogLongPHD as u64;
-                if !self.crc_check && self.contlen != hdr.std.xlp_rem_len {
+                continue;
                    self.contlen = hdr.std.xlp_rem_len; // skip continuation record
                }
            } else if self.lsn.block_offset() == 0 {
-                if self.inputbuf.remaining() < XLOG_SIZE_OF_XLOG_SHORT_PHD {
+                if self.inputbuf.remaining() < SizeOfXLogShortPHD {
                    return Ok(None);
                }
-                let hdr = XLogPageHeaderData::from_bytes(&mut self.inputbuf);
+                let hdr = self.decode_XLogPageHeaderData();
                if hdr.xlp_pageaddr != self.lsn.0 {
                    return Err(WalDecodeError {
-                        msg: format!(
+                        msg: "invalid xlog page header".into(),
                            "invalid xlog page header: xlp_pageaddr={} vs. lsn={}",
                            hdr.xlp_pageaddr, self.lsn
                        ),
                        lsn: self.lsn,
                    });
                }
                // TODO: verify the remaining fields in the header
-                self.lsn += XLOG_SIZE_OF_XLOG_SHORT_PHD as u64;
+                self.lsn += SizeOfXLogShortPHD as u64;
-                if !self.crc_check && self.contlen != hdr.xlp_rem_len {
+                continue;
                    self.contlen = hdr.xlp_rem_len; // skip continuation record
                }
            } else if self.padlen > 0 {
                if self.inputbuf.remaining() < self.padlen as usize {
                    return Ok(None);
@@ -139,8 +157,9 @@ impl WalStreamDecoder {
                }
                // read xl_tot_len FIXME: assumes little-endian
                self.startlsn = self.lsn;
                let xl_tot_len = self.inputbuf.get_u32_le();
-                if (xl_tot_len as usize) < XLOG_SIZE_OF_XLOG_RECORD {
+                if xl_tot_len < SizeOfXLogRecord {
                    return Err(WalDecodeError {
                        msg: format!("invalid xl_tot_len {}", xl_tot_len),
                        lsn: self.lsn,
@@ -153,6 +172,7 @@ impl WalStreamDecoder {
                self.recordbuf.put_u32_le(xl_tot_len);
                self.contlen = xl_tot_len - 4;
                continue;
            } else {
                // we're continuing a record, possibly from previous page.
                let pageleft = self.lsn.remaining_in_block() as u32;
@@ -174,10 +194,9 @@ impl WalStreamDecoder {
                    let recordbuf = recordbuf.freeze();
                    let mut buf = recordbuf.clone();
                    let xlogrec = XLogRecord::from_bytes(&mut buf);
                    // XLOG_SWITCH records are special. If we see one, we need to skip
                    // to the next WAL segment.
                    let xlogrec = XLogRecord::from_bytes(&mut buf);
                    if xlogrec.is_xlog_switch_record() {
                        trace!("saw xlog switch record at {}", self.lsn);
                        self.padlen =
@@ -187,29 +206,10 @@ impl WalStreamDecoder {
                        self.padlen = self.lsn.calc_padding(8u32) as u32;
                    }
-                    // Check record CRC
+                    let result = (self.lsn, recordbuf);
                    if self.crc_check {
                        let mut crc = crc32c_append(0, &recordbuf[XLOG_RECORD_CRC_OFFS + 4..]);
                        crc = crc32c_append(crc, &recordbuf[0..XLOG_RECORD_CRC_OFFS]);
                        if crc != xlogrec.xl_crc {
                            info!("WAL record crc mismatch n={}, buf.len()={}, lsn={}, rec={:?}, recordbuf={:?}",
 								  n, recordbuf.len(), self.lsn, xlogrec, recordbuf);
                            return Err(WalDecodeError {
                                msg: format!(
                                    "WAL record crc mismatch n={}, buf.len()={}, lsn={}, rec={:?}",
                                    n,
                                    buf.len(),
                                    self.lsn,
                                    xlogrec
                                ),
                                lsn: self.lsn,
                            });
                        }
                    }
                    let result = (self.lsn.align(), recordbuf);
                    return Ok(Some(result));
                }
                continue;
            }
        }
        // check record boundaries
@@ -218,6 +218,40 @@ impl WalStreamDecoder {
        // deal with xlog_switch records
    }
    #[allow(non_snake_case)]
    fn decode_XLogPageHeaderData(&mut self) -> XLogPageHeaderData {
        let buf = &mut self.inputbuf;
        // FIXME: Assume little-endian
        let hdr: XLogPageHeaderData = XLogPageHeaderData {
            xlp_magic: buf.get_u16_le(),
            xlp_info: buf.get_u16_le(),
            xlp_tli: buf.get_u32_le(),
            xlp_pageaddr: buf.get_u64_le(),
            xlp_rem_len: buf.get_u32_le(),
        };
        // 4 bytes of padding, on 64-bit systems
        buf.advance(4);
        // FIXME: check that hdr.xlp_rem_len matches self.contlen
        //println!("next xlog page (xlp_rem_len: {})", hdr.xlp_rem_len);
        hdr
    }
    #[allow(non_snake_case)]
    fn decode_XLogLongPageHeaderData(&mut self) -> XLogLongPageHeaderData {
        let hdr: XLogLongPageHeaderData = XLogLongPageHeaderData {
            std: self.decode_XLogPageHeaderData(),
            xlp_sysid: self.inputbuf.get_u64_le(),
            xlp_seg_size: self.inputbuf.get_u32_le(),
            xlp_xlog_blcksz: self.inputbuf.get_u32_le(),
        };
        hdr
    }
 }
 #[allow(dead_code)]
@@ -239,7 +273,7 @@ pub struct DecodedBkpBlock {
    /* Information on full-page image, if any */
    has_image: bool,       /* has image, even for consistency checking */
    pub apply_image: bool, /* has image that should be restored */
-    pub will_init: bool,   /* record doesn't need previous page version to apply */
+    pub will_init: bool,
    //char	   *bkp_image;
    hole_offset: u16,
    hole_length: u16,
@@ -275,8 +309,10 @@ impl DecodedBkpBlock {
    }
 }
 #[allow(non_upper_case_globals)]
 const SizeOfXLogRecord: u32 = 24;
 pub struct DecodedWALRecord {
    pub xl_xid: TransactionId,
    pub xl_info: u8,
    pub xl_rmid: u8,
    pub record: Bytes, // raw XLogRecord
@@ -320,7 +356,9 @@ pub struct XlSmgrTruncate {
 }
 impl XlSmgrTruncate {
-    pub fn decode(buf: &mut Bytes) -> XlSmgrTruncate {
+    pub fn decode(decoded: &DecodedWALRecord) -> XlSmgrTruncate {
        let mut buf = decoded.record.clone();
        buf.advance((SizeOfXLogRecord + 2) as usize);
        XlSmgrTruncate {
            blkno: buf.get_u32_le(),
            rnode: RelFileNode {
@@ -343,7 +381,9 @@ pub struct XlCreateDatabase {
 }
 impl XlCreateDatabase {
-    pub fn decode(buf: &mut Bytes) -> XlCreateDatabase {
+    pub fn decode(decoded: &DecodedWALRecord) -> XlCreateDatabase {
        let mut buf = decoded.record.clone();
        buf.advance((SizeOfXLogRecord + 2) as usize);
        XlCreateDatabase {
            db_id: buf.get_u32_le(),
            tablespace_id: buf.get_u32_le(),
@@ -429,121 +469,6 @@ impl XlHeapUpdate {
    }
 }
 ///
 /// Note: Parsing some fields is missing, because they're not needed.
 ///
 /// This is similar to the xl_xact_parsed_commit and
 /// xl_xact_parsed_abort structs in PostgreSQL, but we use the same
 /// struct for commits and aborts.
 ///
 #[derive(Debug)]
 pub struct XlXactParsedRecord {
    pub xid: TransactionId,
    pub info: u8,
    pub xact_time: TimestampTz,
    pub xinfo: u32,
    pub db_id: Oid, /* MyDatabaseId */
    pub ts_id: Oid, /* MyDatabaseTableSpace */
    pub subxacts: Vec<TransactionId>,
    pub xnodes: Vec<RelFileNode>,
 }
 impl XlXactParsedRecord {
    /// Decode a XLOG_XACT_COMMIT/ABORT/COMMIT_PREPARED/ABORT_PREPARED
    /// record. This should agree with the ParseCommitRecord and ParseAbortRecord
    /// functions in PostgreSQL (in src/backend/access/rmgr/xactdesc.c)
    pub fn decode(buf: &mut Bytes, mut xid: TransactionId, xl_info: u8) -> XlXactParsedRecord {
        let info = xl_info & pg_constants::XLOG_XACT_OPMASK;
        // The record starts with time of commit/abort
        let xact_time = buf.get_i64_le();
        let xinfo;
        if xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
            xinfo = buf.get_u32_le();
        } else {
            xinfo = 0;
        }
        let db_id;
        let ts_id;
        if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
            db_id = buf.get_u32_le();
            ts_id = buf.get_u32_le();
        } else {
            db_id = 0;
            ts_id = 0;
        }
        let mut subxacts = Vec::<TransactionId>::new();
        if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
            let nsubxacts = buf.get_i32_le();
            for _i in 0..nsubxacts {
                let subxact = buf.get_u32_le();
                subxacts.push(subxact);
            }
        }
        let mut xnodes = Vec::<RelFileNode>::new();
        if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
            let nrels = buf.get_i32_le();
            for _i in 0..nrels {
                let spcnode = buf.get_u32_le();
                let dbnode = buf.get_u32_le();
                let relnode = buf.get_u32_le();
                trace!(
                    "XLOG_XACT_COMMIT relfilenode {}/{}/{}",
                    spcnode,
                    dbnode,
                    relnode
                );
                xnodes.push(RelFileNode {
                    spcnode,
                    dbnode,
                    relnode,
                });
            }
        }
        if xinfo & pg_constants::XACT_XINFO_HAS_INVALS != 0 {
            let nmsgs = buf.get_i32_le();
            for _i in 0..nmsgs {
                let sizeof_shared_invalidation_message = 0;
                buf.advance(sizeof_shared_invalidation_message);
            }
        }
        if xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE != 0 {
            xid = buf.get_u32_le();
            trace!("XLOG_XACT_COMMIT-XACT_XINFO_HAS_TWOPHASE");
        }
        XlXactParsedRecord {
            xid,
            info,
            xact_time,
            xinfo,
            db_id,
            ts_id,
            subxacts,
            xnodes,
        }
    }
 }
 #[repr(C)]
 #[derive(Debug)]
 pub struct XlClogTruncate {
    pub pageno: u32,
    pub oldest_xid: TransactionId,
    pub oldest_xid_db: Oid,
 }
 impl XlClogTruncate {
    pub fn decode(buf: &mut Bytes) -> XlClogTruncate {
        XlClogTruncate {
            pageno: buf.get_u32_le(),
            oldest_xid: buf.get_u32_le(),
            oldest_xid_db: buf.get_u32_le(),
        }
    }
 }
 #[repr(C)]
 #[derive(Debug)]
 pub struct MultiXactMember {
@@ -590,14 +515,14 @@ impl XlMultiXactCreate {
 #[repr(C)]
 #[derive(Debug)]
 pub struct XlMultiXactTruncate {
-    pub oldest_multi_db: Oid,
+    oldest_multi_db: Oid,
    /* to-be-truncated range of multixact offsets */
-    pub start_trunc_off: MultiXactId, /* just for completeness' sake */
+    start_trunc_off: MultiXactId, /* just for completeness' sake */
-    pub end_trunc_off: MultiXactId,
+    end_trunc_off: MultiXactId,
    /* to-be-truncated range of multixact members */
-    pub start_trunc_memb: MultiXactOffset,
+    start_trunc_memb: MultiXactOffset,
-    pub end_trunc_memb: MultiXactOffset,
+    end_trunc_memb: MultiXactOffset,
 }
 impl XlMultiXactTruncate {
@@ -612,7 +537,8 @@ impl XlMultiXactTruncate {
    }
 }
-/// Main routine to decode a WAL record and figure out which blocks are modified
+//
 // Routines to decode a WAL record and figure out which blocks are modified
 //
 // See xlogrecord.h for details
 // The overall layout of an XLOG record is:
@@ -630,7 +556,7 @@ impl XlMultiXactTruncate {
 //      block data
 //      ...
 //      main data
-pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
+pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedWALRecord {
    let mut rnode_spcnode: u32 = 0;
    let mut rnode_dbnode: u32 = 0;
    let mut rnode_relnode: u32 = 0;
@@ -648,10 +574,15 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
        xlogrec.xl_rmid,
        xlogrec.xl_info
    );
    if xlogrec.xl_xid > checkpoint.nextXid.value as u32 {
        // TODO: handle XID wraparound
        checkpoint.nextXid = FullTransactionId {
            value: (checkpoint.nextXid.value & 0xFFFFFFFF00000000) | xlogrec.xl_xid as u64,
        };
    }
    let remaining = xlogrec.xl_tot_len - SizeOfXLogRecord;
-    let remaining: usize = xlogrec.xl_tot_len as usize - XLOG_SIZE_OF_XLOG_RECORD;
+    if buf.remaining() != remaining as usize {
    if buf.remaining() != remaining {
        //TODO error
    }
@@ -859,10 +790,194 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
        assert_eq!(buf.remaining(), main_data_len as usize);
    }
-    // 5. Handle a few special record types that modify blocks without registering
+    //5. Handle special CLOG and XACT records
-    // them with the standard mechanism.
+    if xlogrec.xl_rmid == pg_constants::RM_CLOG_ID {
-    if xlogrec.xl_rmid == pg_constants::RM_HEAP_ID {
+        let mut blk = DecodedBkpBlock::new();
-        let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
+        blk.forknum = pg_constants::PG_XACT_FORKNUM;
        blk.blkno = buf.get_i32_le() as u32;
        blk.will_init = true;
        trace!("RM_CLOG_ID updates block {}", blk.blkno);
        blocks.push(blk);
    } else if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
        let info = xlogrec.xl_info & pg_constants::XLOG_XACT_OPMASK;
        if info == pg_constants::XLOG_XACT_COMMIT || info == pg_constants::XLOG_XACT_COMMIT_PREPARED {
 			if info == pg_constants::XLOG_XACT_COMMIT {
 				let mut blk = DecodedBkpBlock::new();
 				blk.forknum = pg_constants::PG_XACT_FORKNUM;
 				blk.blkno = xlogrec.xl_xid / pg_constants::CLOG_XACTS_PER_PAGE;
 				trace!(
 					"XLOG_XACT_COMMIT xl_info {} xl_prev {:X}/{:X}  xid {} updates block {} main_data_len {}",
 					xlogrec.xl_info, (xlogrec.xl_prev >> 32),
 					xlogrec.xl_prev & 0xffffffff,
 					xlogrec.xl_xid,
 					blk.blkno,
 					main_data_len
 				);
 				blocks.push(blk);
 			}
            //parse commit record to extract subtrans entries
            // xl_xact_commit starts with time of commit
            let _xact_time = buf.get_i64_le();
            let mut xinfo = 0;
            if xlogrec.xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
                xinfo = buf.get_u32_le();
            }
            if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
                let _dbid = buf.get_u32_le();
                let _tsid = buf.get_u32_le();
            }
            if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
                let nsubxacts = buf.get_i32_le();
                let mut prev_blkno = u32::MAX;
                for _i in 0..nsubxacts {
                    let subxact = buf.get_u32_le();
                    let blkno = subxact / pg_constants::CLOG_XACTS_PER_PAGE;
                    if prev_blkno != blkno {
                        prev_blkno = blkno;
                        let mut blk = DecodedBkpBlock::new();
                        blk.forknum = pg_constants::PG_XACT_FORKNUM;
                        blk.blkno = blkno;
                        blocks.push(blk);
                    }
                }
            }
            if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
                let nrels = buf.get_i32_le();
                for _i in 0..nrels {
                    let spcnode = buf.get_u32_le();
                    let dbnode = buf.get_u32_le();
                    let relnode = buf.get_u32_le();
                    //TODO handle this too?
                    trace!(
                        "XLOG_XACT_COMMIT relfilenode {}/{}/{}",
                        spcnode,
                        dbnode,
                        relnode
                    );
                }
            }
            if xinfo & pg_constants::XACT_XINFO_HAS_INVALS != 0 {
                let nmsgs = buf.get_i32_le();
                for _i in 0..nmsgs {
                    let sizeof_shared_invalidation_message = 0;
                    buf.advance(sizeof_shared_invalidation_message);
                }
            }
            if xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE != 0 {
                let xid = buf.get_u32_le();
 				let mut blk = DecodedBkpBlock::new();
 				blk.forknum = pg_constants::PG_XACT_FORKNUM;
 				blk.blkno = xid / pg_constants::CLOG_XACTS_PER_PAGE;
 				blocks.push(blk);
                trace!("XLOG_XACT_COMMIT-XACT_XINFO_HAS_TWOPHASE");
                //TODO handle this to be able to restore pg_twophase on node start
            }
        } else if info == pg_constants::XLOG_XACT_ABORT || info == pg_constants::XLOG_XACT_ABORT_PREPARED {
 			if info == pg_constants::XLOG_XACT_ABORT {
 				let mut blk = DecodedBkpBlock::new();
 				blk.forknum = pg_constants::PG_XACT_FORKNUM;
 				blk.blkno = xlogrec.xl_xid / pg_constants::CLOG_XACTS_PER_PAGE;
 				trace!(
 					"XLOG_XACT_ABORT xl_info {} xl_prev {:X}/{:X} xid {} updates block {} main_data_len {}",
 					xlogrec.xl_info, (xlogrec.xl_prev >> 32),
 					xlogrec.xl_prev & 0xffffffff,
 					xlogrec.xl_xid,
 					blk.blkno,
 					main_data_len
 				);
 				blocks.push(blk);
 			}
            //parse abort record to extract subtrans entries
            // xl_xact_abort starts with time of commit
            let _xact_time = buf.get_i64_le();
            let mut xinfo = 0;
            if xlogrec.xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
                xinfo = buf.get_u32_le();
            }
            if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
                let _dbid = buf.get_u32_le();
                let _tsid = buf.get_u32_le();
            }
            if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
                let nsubxacts = buf.get_i32_le();
                let mut prev_blkno = u32::MAX;
                for _i in 0..nsubxacts {
                    let subxact = buf.get_u32_le();
                    let blkno = subxact / pg_constants::CLOG_XACTS_PER_PAGE;
                    if prev_blkno != blkno {
                        prev_blkno = blkno;
                        let mut blk = DecodedBkpBlock::new();
                        blk.forknum = pg_constants::PG_XACT_FORKNUM;
                        blk.blkno = blkno;
                        blocks.push(blk);
                    }
                }
            }
            if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
                let nrels = buf.get_i32_le();
                for _i in 0..nrels {
                    let spcnode = buf.get_u32_le();
                    let dbnode = buf.get_u32_le();
                    let relnode = buf.get_u32_le();
                    //TODO save these too
                    trace!(
                        "XLOG_XACT_ABORT relfilenode {}/{}/{}",
                        spcnode,
                        dbnode,
                        relnode
                    );
                }
            }
            if xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE != 0 {
                let xid = buf.get_u32_le();
 				let mut blk = DecodedBkpBlock::new();
 				blk.forknum = pg_constants::PG_XACT_FORKNUM;
 				blk.blkno = xid / pg_constants::CLOG_XACTS_PER_PAGE;
 				blocks.push(blk);
                trace!("XLOG_XACT_ABORT-XACT_XINFO_HAS_TWOPHASE");
            }
        } else if info == pg_constants::XLOG_XACT_PREPARE {
            let mut blk = DecodedBkpBlock::new();
            blk.forknum = pg_constants::PG_TWOPHASE_FORKNUM;
            blk.blkno = xlogrec.xl_xid;
            blk.will_init = true;
            blocks.push(blk);
 			info!("Prepare transaction {}", xlogrec.xl_xid);
        }
    } else if xlogrec.xl_rmid == pg_constants::RM_DBASE_ID {
        let info = xlogrec.xl_info & !pg_constants::XLR_INFO_MASK;
        if info == pg_constants::XLOG_DBASE_CREATE {
            //buf points to main_data
            let db_id = buf.get_u32_le();
            let tablespace_id = buf.get_u32_le();
            let src_db_id = buf.get_u32_le();
            let src_tablespace_id = buf.get_u32_le();
            trace!(
                "XLOG_DBASE_CREATE tablespace_id/db_id {}/{} src_db_id {}/{}",
                tablespace_id,
                db_id,
                src_tablespace_id,
                src_db_id
            );
            // in postgres it is implemented as copydir
            // we need to copy all pages in page_cache
        } else {
            trace!("XLOG_DBASE_DROP is not handled yet");
        }
    } else if xlogrec.xl_rmid == pg_constants::RM_TBLSPC_ID {
        let info = xlogrec.xl_info & !pg_constants::XLR_INFO_MASK;
        if info == pg_constants::XLOG_TBLSPC_CREATE {
            //buf points to main_data
            let ts_id = buf.get_u32_le();
            let ts_path = str::from_utf8(&buf).unwrap();
            trace!("XLOG_TBLSPC_CREATE ts_id {} ts_path {}", ts_id, ts_path);
        } else {
            trace!("XLOG_TBLSPC_DROP is not handled yet");
        }
    } else if xlogrec.xl_rmid == pg_constants::RM_HEAP_ID {
        let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
        let blkno = blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32;
        if info == pg_constants::XLOG_HEAP_INSERT {
            let xlrec = XlHeapInsert::decode(&mut buf);
@@ -916,7 +1031,7 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
            }
        }
    } else if xlogrec.xl_rmid == pg_constants::RM_HEAP2_ID {
-        let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
+        let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
        if info == pg_constants::XLOG_HEAP2_MULTI_INSERT {
            let xlrec = XlHeapMultiInsert::decode(&mut buf);
            if (xlrec.flags
@@ -934,10 +1049,105 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
                blocks.push(blk);
            }
        }
    } else if xlogrec.xl_rmid == pg_constants::RM_MULTIXACT_ID {
        let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
        if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE {
            let mut blk = DecodedBkpBlock::new();
            blk.forknum = pg_constants::PG_MXACT_OFFSETS_FORKNUM;
            blk.blkno = buf.get_u32_le();
            blk.will_init = true;
            blocks.push(blk);
        } else if info == pg_constants::XLOG_MULTIXACT_ZERO_MEM_PAGE {
            let mut blk = DecodedBkpBlock::new();
            blk.forknum = pg_constants::PG_MXACT_MEMBERS_FORKNUM;
            blk.blkno = buf.get_u32_le();
            blk.will_init = true;
            blocks.push(blk);
        } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
            let xlrec = XlMultiXactCreate::decode(&mut buf);
            // Update offset page
            let mut blk = DecodedBkpBlock::new();
            blk.blkno = xlrec.mid / pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32;
            blk.forknum = pg_constants::PG_MXACT_OFFSETS_FORKNUM;
            blocks.push(blk);
            let first_mbr_blkno = xlrec.moff / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
            let last_mbr_blkno =
                (xlrec.moff + xlrec.nmembers - 1) / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
            for blkno in first_mbr_blkno..=last_mbr_blkno {
                // Update members page
                let mut blk = DecodedBkpBlock::new();
                blk.forknum = pg_constants::PG_MXACT_MEMBERS_FORKNUM;
                blk.blkno = blkno;
                blocks.push(blk);
            }
            if xlrec.mid > checkpoint.nextMulti {
                checkpoint.nextMulti = xlrec.mid;
            }
            if xlrec.moff > checkpoint.nextMultiOffset {
                checkpoint.nextMultiOffset = xlrec.moff;
            }
            let max_xid = xlrec
                .members
                .iter()
                .fold(checkpoint.nextXid.value as u32, |acc, mbr| {
                    if mbr.xid > acc {
                        mbr.xid
                    } else {
                        acc
                    }
                });
            checkpoint.nextXid = FullTransactionId {
                value: (checkpoint.nextXid.value & 0xFFFFFFFF00000000) | max_xid as u64,
            };
        } else if info == pg_constants::XLOG_MULTIXACT_TRUNCATE_ID {
            let xlrec = XlMultiXactTruncate::decode(&mut buf);
            checkpoint.oldestXid = xlrec.end_trunc_off;
            checkpoint.oldestMultiDB = xlrec.oldest_multi_db;
            let first_off_blkno =
                xlrec.start_trunc_off / pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32;
            let last_off_blkno =
                xlrec.end_trunc_off / pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32;
            for blkno in first_off_blkno..last_off_blkno {
                let mut blk = DecodedBkpBlock::new();
                blk.forknum = pg_constants::PG_MXACT_OFFSETS_FORKNUM;
                blk.blkno = blkno;
                blk.will_init = true;
                blocks.push(blk);
            }
            let first_mbr_blkno =
                xlrec.start_trunc_memb / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
            let last_mbr_blkno =
                xlrec.end_trunc_memb / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
            for blkno in first_mbr_blkno..last_mbr_blkno {
                let mut blk = DecodedBkpBlock::new();
                blk.forknum = pg_constants::PG_MXACT_MEMBERS_FORKNUM;
                blk.blkno = blkno;
                blk.will_init = true;
                blocks.push(blk);
            }
        } else {
            assert!(false);
        }
    } else if xlogrec.xl_rmid == pg_constants::RM_RELMAP_ID {
        let xlrec = XlRelmapUpdate::decode(&mut buf);
        let mut blk = DecodedBkpBlock::new();
        blk.forknum = pg_constants::PG_FILENODEMAP_FORKNUM;
        blk.rnode_spcnode = xlrec.tsid;
        blk.rnode_dbnode = xlrec.dbid;
        blk.rnode_relnode = 0;
        blk.will_init = true;
        blocks.push(blk);
    } else if xlogrec.xl_rmid == pg_constants::RM_XLOG_ID {
        let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
        if info == pg_constants::XLOG_NEXTOID {
            let next_oid = buf.get_u32_le();
            if next_oid > checkpoint.nextOid {
                checkpoint.nextOid = next_oid;
            }
        }
    }
    DecodedWALRecord {
        xl_xid: xlogrec.xl_xid,
        xl_info: xlogrec.xl_info,
        xl_rmid: xlogrec.xl_rmid,
        record,
@@ -1,15 +1,17 @@
 //!
-//! WAL receiver connects to the WAL safekeeper service,
+//! WAL receiver
-//! streams WAL, decodes records and saves them in page cache.
+//!
 //! The WAL receiver connects to the WAL safekeeper service, and streams WAL.
 //! For each WAL record, it decodes the record to figure out which data blocks
 //! the record affects, and adds the records to the page cache.
 //!
 //! We keep one WAL receiver active per timeline.
 use crate::page_cache;
-use crate::relish::*;
+use crate::repository::*;
 use crate::restore_local_repo;
 use crate::waldecoder::*;
-use crate::{PageServerConf, RepositoryFormat};
+use crate::PageServerConf;
-use anyhow::{Error, Result};
+use crate::ZTimelineId;
 use anyhow::Error;
 use lazy_static::lazy_static;
 use log::*;
 use postgres::fallible_iterator::FallibleIterator;
@@ -19,17 +21,17 @@ use postgres_ffi::xlog_utils::*;
 use postgres_ffi::*;
 use postgres_protocol::message::backend::ReplicationMessage;
 use postgres_types::PgLsn;
 use std::cmp::{max, min};
 use std::collections::HashMap;
 use std::fs;
 use std::fs::{File, OpenOptions};
 use std::io::{Seek, SeekFrom, Write};
 use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Mutex;
 use std::thread;
 use std::thread::sleep;
 use std::time::{Duration, SystemTime};
 use zenith_utils::lsn::Lsn;
 use zenith_utils::zid::ZTenantId;
 use zenith_utils::zid::ZTimelineId;
 //
 // We keep one WAL Receiver active per timeline.
@@ -48,7 +50,6 @@ pub fn launch_wal_receiver(
    conf: &'static PageServerConf,
    timelineid: ZTimelineId,
    wal_producer_connstr: &str,
    tenantid: ZTenantId,
 ) {
    let mut receivers = WAL_RECEIVERS.lock().unwrap();
@@ -66,7 +67,7 @@ pub fn launch_wal_receiver(
            let _walreceiver_thread = thread::Builder::new()
                .name("WAL receiver thread".into())
                .spawn(move || {
-                    thread_main(conf, timelineid, &tenantid);
+                    thread_main(conf, timelineid);
                })
                .unwrap();
        }
@@ -87,7 +88,7 @@ fn get_wal_producer_connstr(timelineid: ZTimelineId) -> String {
 //
 // This is the entry point for the WAL receiver thread.
 //
-fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId, tenantid: &ZTenantId) {
+fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId) {
    info!(
        "WAL receiver thread started for timeline : '{}'",
        timelineid
@@ -101,7 +102,7 @@ fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId, tenantid:
        // Look up the current WAL producer address
        let wal_producer_connstr = get_wal_producer_connstr(timelineid);
-        let res = walreceiver_main(conf, timelineid, &wal_producer_connstr, tenantid);
+        let res = walreceiver_main(conf, timelineid, &wal_producer_connstr);
        if let Err(e) = res {
            info!(
@@ -114,36 +115,23 @@ fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId, tenantid:
 }
 fn walreceiver_main(
-    conf: &PageServerConf,
+    _conf: &PageServerConf,
    timelineid: ZTimelineId,
    wal_producer_connstr: &str,
    tenantid: &ZTenantId,
 ) -> Result<(), Error> {
    // Connect to the database in replication mode.
    info!("connecting to {:?}", wal_producer_connstr);
-    let connect_cfg = format!(
+    let connect_cfg = format!("{} replication=true", wal_producer_connstr);
        "{} application_name=pageserver replication=true",
        wal_producer_connstr
    );
    let mut rclient = Client::connect(&connect_cfg, NoTls)?;
    info!("connected!");
    // Immediately increment the gauge, then create a job to decrement it on thread exit.
    // One of the pros of `defer!` is that this will *most probably*
    // get called, even in presence of panics.
    let gauge = crate::LIVE_CONNECTIONS_COUNT.with_label_values(&["wal_receiver"]);
    gauge.inc();
    scopeguard::defer! {
        gauge.dec();
    }
    let identify = identify_system(&mut rclient)?;
    info!("{:?}", identify);
    let end_of_wal = Lsn::from(u64::from(identify.xlogpos));
    let mut caught_up = false;
-    let repository = page_cache::get_repository_for_tenant(tenantid)?;
+    let repository = page_cache::get_repository();
    let timeline = repository.get_timeline(timelineid).unwrap();
    //
@@ -152,13 +140,18 @@ fn walreceiver_main(
    // If we had previously received WAL up to some point in the middle of a WAL record, we
    // better start from the end of last full WAL record, not in the middle of one. Hence,
    // use 'last_record_lsn' rather than 'last_valid_lsn' here.
-    let mut last_rec_lsn = timeline.get_last_record_lsn();
+    let last_rec_lsn = timeline.get_last_record_lsn();
    let mut startpoint = last_rec_lsn;
    if startpoint == Lsn(0) {
        error!("No previous WAL position");
    }
    startpoint = Lsn::max(
        startpoint,
        Lsn(end_of_wal.0 & !(pg_constants::WAL_SEGMENT_SIZE as u64 - 1)),
    );
    // There might be some padding after the last full record, skip it.
    //
    // FIXME: It probably would be better to always start streaming from the beginning
@@ -176,67 +169,45 @@ fn walreceiver_main(
    let copy_stream = rclient.copy_both_simple(&query)?;
    let mut physical_stream = ReplicationIter::new(copy_stream);
-    let mut waldecoder = WalStreamDecoder::new(startpoint, true);
+    let mut waldecoder = WalStreamDecoder::new(startpoint);
    let checkpoint_bytes = timeline.get_page_at_lsn_nowait(RelishTag::Checkpoint, 0, startpoint)?;
    let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
    trace!("CheckPoint.nextXid = {}", checkpoint.nextXid.value);
    let mut checkpoint = CheckPoint::new(startpoint.0, identify.timeline);
    let checkpoint_tag = BufferTag::fork(pg_constants::PG_CHECKPOINT_FORKNUM);
    if let Some(checkpoint_bytes) = timeline.get_page_image(checkpoint_tag, Lsn(0))? {
        checkpoint = decode_checkpoint(checkpoint_bytes)?;
        trace!("CheckPoint.nextXid = {}", checkpoint.nextXid.value);
    } else {
        error!("No checkpoint record was found in reposistory");
    }
    while let Some(replication_message) = physical_stream.next()? {
-        let status_update = match replication_message {
+        match replication_message {
            ReplicationMessage::XLogData(xlog_data) => {
                // Pass the WAL data to the decoder, and see if we can decode
                // more records as a result.
                let data = xlog_data.data();
                let startlsn = Lsn::from(xlog_data.wal_start());
                let endlsn = startlsn + data.len() as u64;
-                let prev_last_rec_lsn = last_rec_lsn;
+
                write_wal_file(startlsn, timelineid, pg_constants::WAL_SEGMENT_SIZE, data)?;
                trace!("received XLogData between {} and {}", startlsn, endlsn);
                waldecoder.feed_bytes(data);
-                loop {
+                while let Some((lsn, recdata)) = waldecoder.poll_decode()? {
-                    match waldecoder.poll_decode() {
+                    let old_checkpoint_bytes = encode_checkpoint(checkpoint);
-                        Ok(Some((lsn, recdata))) => {
+                    let decoded = decode_wal_record(&mut checkpoint, recdata.clone());
-                            // Save old checkpoint value to compare with it after decoding WAL record
+                    timeline.save_decoded_record(decoded, recdata, lsn)?;
                            let old_checkpoint_bytes = checkpoint.encode();
                            let decoded = decode_wal_record(recdata.clone());
                            restore_local_repo::save_decoded_record(
                                &mut checkpoint,
                                &*timeline,
                                &decoded,
                                recdata,
                                lsn,
                            )?;
                            last_rec_lsn = lsn;
-                            let new_checkpoint_bytes = checkpoint.encode();
+                    let new_checkpoint_bytes = encode_checkpoint(checkpoint);
-                            // Check if checkpoint data was updated by save_decoded_record
+                    if new_checkpoint_bytes != old_checkpoint_bytes {
-                            if new_checkpoint_bytes != old_checkpoint_bytes {
+                        timeline.put_page_image(checkpoint_tag, Lsn(0), new_checkpoint_bytes);
                                timeline.put_page_image(
                                    RelishTag::Checkpoint,
                                    0,
                                    lsn,
                                    new_checkpoint_bytes,
                                    false,
                                )?;
                            }
                        }
                        Ok(None) => {
                            trace!(
                                "End of replication stream {}..{} at {}",
                                startlsn,
                                endlsn,
                                last_rec_lsn
                            );
                            break;
                        }
                        Err(e) => {
                            info!("Decode error {}", e);
                            return Err(e.into());
                        }
                    }
                    // Now that this record has been handled, let the page cache know that
                    // it is up-to-date to this LSN
                    timeline.advance_last_record_lsn(lsn);
                }
                // Update the last_valid LSN value in the page cache one more time. We updated
                // it in the loop above, between each WAL record, but we might have received
                // a partial record after the last completed record. Our page cache's value
@@ -245,52 +216,16 @@ fn walreceiver_main(
                // flush ptr.
                timeline.advance_last_valid_lsn(endlsn);
                // Somewhat arbitrarily, if we have at least 10 complete wal segments (16 MB each),
                // "checkpoint" the repository to flush all the changes from WAL we've processed
                // so far to disk. After this, we don't need the original WAL anymore, and it
                // can be removed. This is probably too aggressive for production, but it's useful
                // to expose bugs now.
                //
                // TODO: We don't actually dare to remove the WAL. It's useful for debugging,
                // and we might it for logical decoding other things in the future. Although
                // we should also be able to fetch it back from the WAL safekeepers or S3 if
                // needed.
                if prev_last_rec_lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE)
                    != last_rec_lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE)
                {
                    info!("switched segment {} to {}", prev_last_rec_lsn, last_rec_lsn);
                    let (oldest_segno, newest_segno) = find_wal_file_range(
                        conf,
                        &timelineid,
                        pg_constants::WAL_SEGMENT_SIZE,
                        last_rec_lsn,
                        tenantid,
                    )?;
                    if newest_segno - oldest_segno >= 10 {
                        // FIXME: The layered repository performs checkpointing in a separate thread, so this
                        // isn't needed anymore. Remove 'checkpoint' from the Timeline trait altogether?
                        if conf.repository_format == RepositoryFormat::RocksDb {
                            timeline.checkpoint()?;
                        }
                        // TODO: This is where we could remove WAL older than last_rec_lsn.
                        //remove_wal_files(timelineid, pg_constants::WAL_SEGMENT_SIZE, last_rec_lsn)?;
                    }
                }
                if !caught_up && endlsn >= end_of_wal {
                    info!("caught up at LSN {}", endlsn);
                    caught_up = true;
                }
                Some(endlsn)
            }
            ReplicationMessage::PrimaryKeepAlive(keepalive) => {
                let wal_end = keepalive.wal_end();
                let timestamp = keepalive.timestamp();
-                let reply_requested = keepalive.reply() != 0;
+                let reply_requested: bool = keepalive.reply() != 0;
                trace!(
                    "received PrimaryKeepAlive(wal_end: {}, timestamp: {:?} reply: {})",
@@ -298,73 +233,25 @@ fn walreceiver_main(
                    timestamp,
                    reply_requested,
                );
                if reply_requested {
-                    Some(timeline.get_last_valid_lsn())
+                    // TODO: More thought should go into what values are sent here.
-                } else {
+                    let last_lsn = PgLsn::from(u64::from(timeline.get_last_valid_lsn()));
-                    None
+                    let write_lsn = last_lsn;
                    let flush_lsn = last_lsn;
                    let apply_lsn = PgLsn::from(0);
                    let ts = SystemTime::now();
                    const NO_REPLY: u8 = 0u8;
                    physical_stream
                        .standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY)?;
                }
            }
-
+            _ => (),
            _ => None,
        };
        if let Some(last_lsn) = status_update {
            // TODO: More thought should go into what values are sent here.
            let last_lsn = PgLsn::from(u64::from(last_lsn));
            let write_lsn = last_lsn;
            let flush_lsn = last_lsn;
            let apply_lsn = PgLsn::from(0);
            let ts = SystemTime::now();
            const NO_REPLY: u8 = 0;
            physical_stream.standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY)?;
        }
    }
    Ok(())
 }
 fn find_wal_file_range(
    conf: &PageServerConf,
    timeline: &ZTimelineId,
    wal_seg_size: usize,
    written_upto: Lsn,
    tenant: &ZTenantId,
 ) -> Result<(u64, u64)> {
    let written_upto_segno = written_upto.segment_number(wal_seg_size);
    let mut oldest_segno = written_upto_segno;
    let mut newest_segno = written_upto_segno;
    // Scan the wal directory, and count how many WAL filed we could remove
    let wal_dir = conf.wal_dir_path(timeline, tenant);
    for entry in fs::read_dir(wal_dir)? {
        let entry = entry?;
        let path = entry.path();
        if path.is_dir() {
            continue;
        }
        let filename = path.file_name().unwrap().to_str().unwrap();
        if IsXLogFileName(filename) {
            let (segno, _tli) = XLogFromFileName(filename, wal_seg_size);
            if segno > written_upto_segno {
                // that's strange.
                warn!("there is a WAL file from future at {}", path.display());
                continue;
            }
            oldest_segno = min(oldest_segno, segno);
            newest_segno = max(newest_segno, segno);
        }
    }
    // FIXME: would be good to assert that there are no gaps in the WAL files
    Ok((oldest_segno, newest_segno))
 }
 /// Data returned from the postgres `IDENTIFY_SYSTEM` command
 ///
 /// See the [postgres docs] for more details.
@@ -411,3 +298,96 @@ pub fn identify_system(client: &mut Client) -> Result<IdentifySystem, Error> {
        Err(IdentifyError.into())
    }
 }
 fn write_wal_file(
    startpos: Lsn,
    timeline: ZTimelineId,
    wal_seg_size: usize,
    buf: &[u8],
 ) -> anyhow::Result<()> {
    let mut bytes_left: usize = buf.len();
    let mut bytes_written: usize = 0;
    let mut partial;
    let mut start_pos = startpos;
    const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ];
    let wal_dir = PathBuf::from(format!("timelines/{}/wal", timeline));
    /* Extract WAL location for this block */
    let mut xlogoff = start_pos.segment_offset(wal_seg_size);
    while bytes_left != 0 {
        let bytes_to_write;
        /*
         * If crossing a WAL boundary, only write up until we reach wal
         * segment size.
         */
        if xlogoff + bytes_left > wal_seg_size {
            bytes_to_write = wal_seg_size - xlogoff;
        } else {
            bytes_to_write = bytes_left;
        }
        /* Open file */
        let segno = start_pos.segment_number(wal_seg_size);
        let wal_file_name = XLogFileName(
            1, // FIXME: always use Postgres timeline 1
            segno,
            wal_seg_size,
        );
        let wal_file_path = wal_dir.join(wal_file_name.clone());
        let wal_file_partial_path = wal_dir.join(wal_file_name.clone() + ".partial");
        {
            let mut wal_file: File;
            /* Try to open already completed segment */
            if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_path) {
                wal_file = file;
                partial = false;
            } else if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_partial_path) {
                /* Try to open existed partial file */
                wal_file = file;
                partial = true;
            } else {
                /* Create and fill new partial file */
                partial = true;
                match OpenOptions::new()
                    .create(true)
                    .write(true)
                    .open(&wal_file_partial_path)
                {
                    Ok(mut file) => {
                        for _ in 0..(wal_seg_size / XLOG_BLCKSZ) {
                            file.write_all(&ZERO_BLOCK)?;
                        }
                        wal_file = file;
                    }
                    Err(e) => {
                        error!("Failed to open log file {:?}: {}", &wal_file_path, e);
                        return Err(e.into());
                    }
                }
            }
            wal_file.seek(SeekFrom::Start(xlogoff as u64))?;
            wal_file.write_all(&buf[bytes_written..(bytes_written + bytes_to_write)])?;
            // FIXME: Flush the file
            //wal_file.sync_all()?;
        }
        /* Write was successful, advance our position */
        bytes_written += bytes_to_write;
        bytes_left -= bytes_to_write;
        start_pos += bytes_to_write as u64;
        xlogoff += bytes_to_write;
        /* Did we reach the end of a WAL segment? */
        if start_pos.segment_offset(wal_seg_size) == 0 {
            xlogoff = 0;
            if partial {
                fs::rename(&wal_file_partial_path, &wal_file_path)?;
            }
        }
    }
    Ok(())
 }
@@ -1,11 +1,9 @@
 //!
-//! WAL redo. This service runs PostgreSQL in a special wal_redo mode
+//! WAL redo
 //! to apply given WAL records over an old page image and return new
 //! page image.
 //!
 //! We rely on Postgres to perform WAL redo for us. We launch a
 //! postgres process in special "wal redo" mode that's similar to
-//! single-user mode. We then pass the previous page image, if any,
+//! single-user mode. We then pass the the previous page image, if any,
 //! and all the WAL records we want to apply, to the postgres
 //! process. Then we get the page image back. Communication with the
 //! postgres process happens via stdin/stdout
@@ -13,23 +11,21 @@
 //! See src/backend/tcop/zenith_wal_redo.c for the other side of
 //! this communication.
 //!
-//! The Postgres process is assumed to be secure against malicious WAL
+//! TODO: Even though the postgres code runs in a separate process,
-//! records. It achieves it by dropping privileges before replaying
+//! it's not a secure sandbox.
 //! any WAL records, so that even if an attacker hijacks the Postgres
 //! process, he cannot escape out of it.
 //!
 use byteorder::{ByteOrder, LittleEndian};
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use lazy_static::lazy_static;
 use log::*;
-use serde::{Deserialize, Serialize};
+use std::assert;
 use std::cell::RefCell;
 use std::fs;
 use std::fs::OpenOptions;
 use std::io::prelude::*;
 use std::io::Error;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::process::Stdio;
 use std::sync::mpsc;
 use std::sync::Mutex;
 use std::time::Duration;
 use std::time::Instant;
@@ -37,31 +33,15 @@ use tokio::io::AsyncBufReadExt;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::process::{ChildStdin, ChildStdout, Command};
 use tokio::time::timeout;
 use zenith_metrics::{register_histogram, register_int_counter, Histogram, IntCounter};
 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;
 use zenith_utils::zid::ZTenantId;
-use crate::relish::*;
+use crate::repository::BufferTag;
 use crate::repository::WALRecord;
 use crate::waldecoder::XlXactParsedRecord;
 use crate::waldecoder::{MultiXactId, XlMultiXactCreate};
 use crate::PageServerConf;
 use postgres_ffi::nonrelfile_utils::transaction_id_set_status;
 use postgres_ffi::pg_constants;
-use postgres_ffi::XLogRecord;
+use postgres_ffi::xlog_utils::XLogRecord;
 ///
 /// `RelTag` + block number (`blknum`) gives us a unique id of the page in the cluster.
 ///
 /// In Postgres `BufferTag` structure is used for exactly the same purpose.
 /// [See more related comments here](https://github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/buf_internals.h#L91).
 ///
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Serialize, Deserialize)]
 pub struct BufferTag {
    pub rel: RelTag,
    pub blknum: u32,
 }
 ///
 /// WAL Redo Manager is responsible for replaying WAL records.
@@ -76,79 +56,40 @@ pub trait WalRedoManager: Send + Sync {
    /// the reords.
    fn request_redo(
        &self,
-        rel: RelishTag,
+        tag: BufferTag,
        blknum: u32,
        lsn: Lsn,
        base_img: Option<Bytes>,
        records: Vec<WALRecord>,
    ) -> Result<Bytes, WalRedoError>;
 }
 ///
 /// A dummy WAL Redo Manager implementation that doesn't allow replaying
 /// anything. Currently used during bootstrapping (zenith init), to create
 /// a Repository object without launching the real WAL redo process.
 ///
 pub struct DummyRedoManager {}
 impl crate::walredo::WalRedoManager for DummyRedoManager {
    fn request_redo(
        &self,
        _rel: RelishTag,
        _blknum: u32,
        _lsn: Lsn,
        _base_img: Option<Bytes>,
        _records: Vec<WALRecord>,
    ) -> Result<Bytes, WalRedoError> {
        Err(WalRedoError::InvalidState)
    }
 }
 static TIMEOUT: Duration = Duration::from_secs(20);
 // Metrics collected on WAL redo operations
 //
 // We collect the time spent in actual WAL redo ('redo'), and time waiting
 // for access to the postgres process ('wait') since there is only one for
 // each tenant.
 lazy_static! {
    static ref WAL_REDO_TIME: Histogram =
        register_histogram!("pageserver_wal_redo_time", "Time spent on WAL redo")
            .expect("failed to define a metric");
    static ref WAL_REDO_WAIT_TIME: Histogram = register_histogram!(
        "pageserver_wal_redo_wait_time",
        "Time spent waiting for access to the WAL redo process"
    )
    .expect("failed to define a metric");
    static ref WAL_REDO_RECORD_COUNTER: IntCounter = register_int_counter!(
        "pageserver_wal_records_replayed",
        "Number of WAL records replayed"
    )
    .unwrap();
 }
 ///
-/// This is the real implementation that uses a Postgres process to
+/// The implementation consists of two parts: PostgresRedoManager, and
-/// perform WAL replay. Only one thread can use the processs at a time,
+/// PostgresRedoManagerInternal. PostgresRedoManager is the public struct
-/// that is controlled by the Mutex. In the future, we might want to
+/// that can be used to send redo requests to the manager.
-/// launch a pool of processes to allow concurrent replay of multiple
+/// PostgresRedoManagerInternal is used by the manager thread itself.
 /// records.
 ///
 pub struct PostgresRedoManager {
-    tenantid: ZTenantId,
+    request_tx: Mutex<mpsc::Sender<WalRedoRequest>>,
 }
 struct PostgresRedoManagerInternal {
    conf: &'static PageServerConf,
-    runtime: tokio::runtime::Runtime,
+    request_rx: mpsc::Receiver<WalRedoRequest>,
    process: Mutex<Option<PostgresRedoProcess>>,
 }
 #[derive(Debug)]
 struct WalRedoRequest {
-    rel: RelishTag,
+    tag: BufferTag,
    blknum: u32,
    lsn: Lsn,
    base_img: Option<Bytes>,
    records: Vec<WALRecord>,
    response_channel: mpsc::Sender<Result<Bytes, WalRedoError>>,
 }
 /// An error happened in WAL redo
@@ -156,14 +97,42 @@ struct WalRedoRequest {
 pub enum WalRedoError {
    #[error(transparent)]
    IoError(#[from] std::io::Error),
    #[error("cannot perform WAL redo now")]
    InvalidState,
 }
 ///
 /// Public interface of WAL redo manager
 ///
 impl PostgresRedoManager {
    ///
    /// Create a new PostgresRedoManager.
    ///
    /// This launches a new thread to handle the requests.
    pub fn new(conf: &'static PageServerConf) -> PostgresRedoManager {
        let (tx, rx) = mpsc::channel();
        //
        // Launch the WAL redo thread
        //
        // Get mutable references to the values that we need to pass to the
        // thread.
        let request_rx = rx;
        // Currently, the join handle is not saved anywhere and we
        // won't try restart the thread if it dies.
        let _walredo_thread = std::thread::Builder::new()
            .name("WAL redo thread".into())
            .spawn(move || {
                let mut internal = PostgresRedoManagerInternal { conf, request_rx };
                internal.wal_redo_main();
            })
            .unwrap();
        PostgresRedoManager {
            request_tx: Mutex::new(tx),
        }
    }
 }
 impl WalRedoManager for PostgresRedoManager {
    ///
    /// Request the WAL redo manager to apply some WAL records
@@ -173,73 +142,62 @@ impl WalRedoManager for PostgresRedoManager {
    ///
    fn request_redo(
        &self,
-        rel: RelishTag,
+        tag: BufferTag,
        blknum: u32,
        lsn: Lsn,
        base_img: Option<Bytes>,
        records: Vec<WALRecord>,
    ) -> Result<Bytes, WalRedoError> {
-        let start_time;
+        // Create a channel where to receive the response
-        let lock_time;
+        let (tx, rx) = mpsc::channel::<Result<Bytes, WalRedoError>>();
        let end_time;
        let request = WalRedoRequest {
-            rel,
+            tag,
            blknum,
            lsn,
            base_img,
            records,
            response_channel: tx,
        };
-        start_time = Instant::now();
+        self.request_tx
-        let result = {
+            .lock()
-            let mut process_guard = self.process.lock().unwrap();
+            .unwrap()
-            lock_time = Instant::now();
+            .send(request)
            .expect("could not send WAL redo request");
-            // launch the WAL redo process on first use
+        rx.recv()
-            if process_guard.is_none() {
+            .expect("could not receive response to WAL redo request")
                let p = self
                    .runtime
                    .block_on(PostgresRedoProcess::launch(self.conf, &self.tenantid))?;
                *process_guard = Some(p);
            }
            let process = (*process_guard).as_ref().unwrap();
            self.runtime
                .block_on(self.handle_apply_request(&process, &request))
        };
        end_time = Instant::now();
        WAL_REDO_WAIT_TIME.observe(lock_time.duration_since(start_time).as_secs_f64());
        WAL_REDO_TIME.observe(end_time.duration_since(lock_time).as_secs_f64());
        result
    }
 }
 fn mx_offset_to_flags_offset(xid: MultiXactId) -> usize {
-    ((xid / pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP as u32) as u16
+    return ((xid / pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP as u32) as u16
        % pg_constants::MULTIXACT_MEMBERGROUPS_PER_PAGE
-        * pg_constants::MULTIXACT_MEMBERGROUP_SIZE) as usize
+        * pg_constants::MULTIXACT_MEMBERGROUP_SIZE) as usize;
 }
 fn mx_offset_to_flags_bitshift(xid: MultiXactId) -> u16 {
-    (xid as u16) % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP
+    return (xid as u16) % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP
-        * pg_constants::MXACT_MEMBER_BITS_PER_XACT
+        * pg_constants::MXACT_MEMBER_BITS_PER_XACT;
 }
 /* Location (byte offset within page) of TransactionId of given member */
 fn mx_offset_to_member_offset(xid: MultiXactId) -> usize {
-    mx_offset_to_flags_offset(xid)
+    return mx_offset_to_flags_offset(xid)
        + (pg_constants::MULTIXACT_FLAGBYTES_PER_GROUP
-            + (xid as u16 % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP) * 4) as usize
+            + (xid as u16 % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP) * 4)
            as usize;
 }
-impl PostgresRedoManager {
+///
-    ///
+/// WAL redo thread
-    /// Create a new PostgresRedoManager.
+///
-    ///
+impl PostgresRedoManagerInternal {
-    pub fn new(conf: &'static PageServerConf, tenantid: ZTenantId) -> PostgresRedoManager {
+    //
    // Main entry point for the WAL applicator thread.
    //
    fn wal_redo_main(&mut self) {
        info!("WAL redo thread started");
        // We block on waiting for requests on the walredo request channel, but
        // use async I/O to communicate with the child process. Initialize the
        // runtime for the async part.
@@ -248,12 +206,35 @@ impl PostgresRedoManager {
            .build()
            .unwrap();
-        // The actual process is launched lazily, on first request.
+        let process: PostgresRedoProcess;
-        PostgresRedoManager {
+
-            runtime,
+        // FIXME: We need a dummy Postgres cluster to run the process in. Currently, we
-            tenantid,
+        // just create one with constant name. That fails if you try to launch more than
-            conf,
+        // one WAL redo manager concurrently.
-            process: Mutex::new(None),
+        let datadir = self.conf.workdir.join("wal-redo-datadir");
        info!("launching WAL redo postgres process");
        process = runtime
            .block_on(PostgresRedoProcess::launch(&datadir))
            .unwrap();
        // Loop forever, handling requests as they come.
        loop {
            let request = self
                .request_rx
                .recv()
                .expect("WAL redo request channel was closed");
            let result = runtime.block_on(self.handle_apply_request(&process, &request));
            let result_ok = result.is_ok();
            // Send the result to the requester
            let _ = request.response_channel.send(result);
            if !result_ok {
                error!("wal-redo-postgres failed to apply request {:?}", request);
            }
        }
    }
@@ -265,8 +246,7 @@ impl PostgresRedoManager {
        process: &PostgresRedoProcess,
        request: &WalRedoRequest,
    ) -> Result<Bytes, WalRedoError> {
-        let rel = request.rel;
+        let tag = request.tag;
        let blknum = request.blknum;
        let lsn = request.lsn;
        let base_img = request.base_img.clone();
        let records = &request.records;
@@ -276,28 +256,17 @@ impl PostgresRedoManager {
        let start = Instant::now();
        let apply_result: Result<Bytes, Error>;
-        if let RelishTag::Relation(rel) = rel {
+        if tag.rel.forknum > pg_constants::INIT_FORKNUM {
            // Relational WAL records are applied using wal-redo-postgres
            let buf_tag = BufferTag { rel, blknum };
            apply_result = process.apply_wal_records(buf_tag, base_img, records).await;
        } else {
            // Non-relational WAL records are handled here, with custom code that has the
            // same effects as the corresponding Postgres WAL redo function.
            const ZERO_PAGE: [u8; 8192] = [0u8; 8192];
            let mut page = BytesMut::new();
            if let Some(fpi) = base_img {
                // If full-page image is provided, then use it...
                page.extend_from_slice(&fpi[..]);
            } else {
                // otherwise initialize page with zeros
                page.extend_from_slice(&ZERO_PAGE);
            }
            // Apply all collected WAL records
            for record in records {
                let mut buf = record.rec.clone();
                WAL_REDO_RECORD_COUNTER.inc();
                // 1. Parse XLogRecord struct
                // FIXME: refactor to avoid code duplication.
                let xlogrec = XLogRecord::from_bytes(&mut buf);
@@ -310,116 +279,180 @@ impl PostgresRedoManager {
                    buf.advance(skip);
                }
-                if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
+                if xlogrec.xl_rmid == pg_constants::RM_CLOG_ID {
-                    // Transaction manager stuff
+                    let info = xlogrec.xl_info & !pg_constants::XLR_INFO_MASK;
-                    let rec_segno = match rel {
+                    if info == pg_constants::CLOG_ZEROPAGE {
-                        RelishTag::Slru { slru, segno } => {
+                        page.copy_from_slice(&ZERO_PAGE);
-                            if slru != SlruKind::Clog {
+                    }
-                                panic!("Not valid XACT relish tag {:?}", rel);
+                } else if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
-                            }
+                    let info = xlogrec.xl_info & pg_constants::XLOG_XACT_OPMASK;
-                            segno
+                    let mut status = 0;
-                        }
+                    if info == pg_constants::XLOG_XACT_COMMIT || info == pg_constants::XLOG_XACT_COMMIT_PREPARED {
-                        _ => panic!("Not valid XACT relish tag {:?}", rel),
+                        status = pg_constants::TRANSACTION_STATUS_COMMITTED;
-                    };
+						if info == pg_constants::XLOG_XACT_COMMIT {
-                    let parsed_xact =
+							transaction_id_set_status(xlogrec.xl_xid, status, &mut page);
-                        XlXactParsedRecord::decode(&mut buf, xlogrec.xl_xid, xlogrec.xl_info);
+						}
-                    if parsed_xact.info == pg_constants::XLOG_XACT_COMMIT
+                        //handle subtrans
-                        || parsed_xact.info == pg_constants::XLOG_XACT_COMMIT_PREPARED
+                        let _xact_time = buf.get_i64_le();
-                    {
+                        let mut xinfo = 0;
-                        transaction_id_set_status(
+                        if xlogrec.xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
-                            parsed_xact.xid,
+                            xinfo = buf.get_u32_le();
-                            pg_constants::TRANSACTION_STATUS_COMMITTED,
+                            if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
-                            &mut page,
+                                let _dbid = buf.get_u32_le();
-                        );
+                                let _tsid = buf.get_u32_le();
                        for subxact in &parsed_xact.subxacts {
                            let pageno = *subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
                            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
                            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
                            // only update xids on the requested page
                            if rec_segno == segno && blknum == rpageno {
                                transaction_id_set_status(
                                    *subxact,
                                    pg_constants::TRANSACTION_STATUS_SUB_COMMITTED,
                                    &mut page,
                                );
                            }
                        }
-                    } else if parsed_xact.info == pg_constants::XLOG_XACT_ABORT
+
-                        || parsed_xact.info == pg_constants::XLOG_XACT_ABORT_PREPARED
+                        if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
-                    {
+                            let nsubxacts = buf.get_i32_le();
-                        transaction_id_set_status(
+                            for _i in 0..nsubxacts {
-                            parsed_xact.xid,
+                                let subxact = buf.get_u32_le();
-                            pg_constants::TRANSACTION_STATUS_ABORTED,
+                                let blkno = subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
-                            &mut page,
+                                // only update xids on the requested page
-                        );
+                                if tag.blknum == blkno {
-                        for subxact in &parsed_xact.subxacts {
+                                    status = pg_constants::TRANSACTION_STATUS_SUB_COMMITTED;
-                            let pageno = *subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
+                                    transaction_id_set_status(subxact, status, &mut page);
-                            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+                                }
                            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
                            // only update xids on the requested page
                            if rec_segno == segno && blknum == rpageno {
                                transaction_id_set_status(
                                    *subxact,
                                    pg_constants::TRANSACTION_STATUS_ABORTED,
                                    &mut page,
                                );
                            }
                        }
 						if info == pg_constants::XLOG_XACT_COMMIT_PREPARED {
 							if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
 								let nrels = buf.get_i32_le();
 								for _i in 0..nrels {
 									let spcnode = buf.get_u32_le();
 									let dbnode = buf.get_u32_le();
 									let relnode = buf.get_u32_le();
 									//TODO handle this too?
 									trace!(
 										"XLOG_XACT_COMMIT relfilenode {}/{}/{}",
 										spcnode,
 										dbnode,
 										relnode
 									);
 								}
 							}
 							if xinfo & pg_constants::XACT_XINFO_HAS_INVALS != 0 {
 								let nmsgs = buf.get_i32_le();
 								for _i in 0..nmsgs {
 									let sizeof_shared_invalidation_message = 0;
 									buf.advance(sizeof_shared_invalidation_message);
 								}
 							}
 							assert!((xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE) != 0);
 							let xid = buf.get_u32_le();
 							transaction_id_set_status(xid, status, &mut page);
 						}
                    } else if info == pg_constants::XLOG_XACT_ABORT || info == pg_constants::XLOG_XACT_ABORT_PREPARED {
                        status = pg_constants::TRANSACTION_STATUS_ABORTED;
 						if info == pg_constants::XLOG_XACT_ABORT {
 							transaction_id_set_status(xlogrec.xl_xid, status, &mut page);
 						}
                        //handle subtrans
                        let _xact_time = buf.get_i64_le();
                        let mut xinfo = 0;
                        if xlogrec.xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
                            xinfo = buf.get_u32_le();
                            if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
                                let _dbid = buf.get_u32_le();
                                let _tsid = buf.get_u32_le();
                            }
                        }
                        if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
                            let nsubxacts = buf.get_i32_le();
                            for _i in 0..nsubxacts {
                                let subxact = buf.get_u32_le();
                                let blkno = subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
                                // only update xids on the requested page
                                if tag.blknum == blkno {
                                    status = pg_constants::TRANSACTION_STATUS_ABORTED;
                                    transaction_id_set_status(subxact, status, &mut page);
                                }
                            }
                        }
 						if info == pg_constants::XLOG_XACT_ABORT_PREPARED {
 							if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
 								let nrels = buf.get_i32_le();
 								for _i in 0..nrels {
 									let spcnode = buf.get_u32_le();
 									let dbnode = buf.get_u32_le();
 									let relnode = buf.get_u32_le();
 									//TODO handle this too?
 									trace!(
 										"XLOG_XACT_COMMIT relfilenode {}/{}/{}",
 										spcnode,
 										dbnode,
 										relnode
 									);
 								}
 							}
 							if xinfo & pg_constants::XACT_XINFO_HAS_INVALS != 0 {
 								let nmsgs = buf.get_i32_le();
 								for _i in 0..nmsgs {
 									let sizeof_shared_invalidation_message = 0;
 									buf.advance(sizeof_shared_invalidation_message);
 								}
 							}
 							assert!((xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE) != 0);
 							let xid = buf.get_u32_le();
 							transaction_id_set_status(xid, status, &mut page);
 						}
                    } else if info == pg_constants::XLOG_XACT_PREPARE {
 						info!("Apply prepare {} record", xlogrec.xl_xid);
 						page.clear();
 						page.extend_from_slice(&buf[..]);
 					} else {
                        error!("handle_apply_request for RM_XACT_ID-{} NOT SUPPORTED YET. RETURN. lsn {} main_data_offset {}, rec.len {}",
                               status,
                               record.lsn,
                               record.main_data_offset, record.rec.len());
                    }
                } else if xlogrec.xl_rmid == pg_constants::RM_MULTIXACT_ID {
                    // Multixact operations
                    let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
-                    if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
+                    if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE {
                        page.copy_from_slice(&ZERO_PAGE);
                    } else if info == pg_constants::XLOG_MULTIXACT_ZERO_MEM_PAGE {
                        page.copy_from_slice(&ZERO_PAGE);
                    } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
                        let xlrec = XlMultiXactCreate::decode(&mut buf);
-                        if let RelishTag::Slru {
+                        if tag.rel.forknum == pg_constants::PG_MXACT_OFFSETS_FORKNUM {
-                            slru,
+                            let offs = (xlrec.mid % pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32
-                            segno: rec_segno,
+                                * 4) as usize;
-                        } = rel
+                            LittleEndian::write_u32(&mut page[offs..offs + 4], xlrec.moff);
                        {
                            if slru == SlruKind::MultiXactMembers {
                                for i in 0..xlrec.nmembers {
                                    let pageno =
                                        i / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
                                    let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
                                    let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
                                    if segno == rec_segno && rpageno == blknum {
                                        // update only target block
                                        let offset = xlrec.moff + i;
                                        let memberoff = mx_offset_to_member_offset(offset);
                                        let flagsoff = mx_offset_to_flags_offset(offset);
                                        let bshift = mx_offset_to_flags_bitshift(offset);
                                        let mut flagsval =
                                            LittleEndian::read_u32(&page[flagsoff..flagsoff + 4]);
                                        flagsval &= !(((1
                                            << pg_constants::MXACT_MEMBER_BITS_PER_XACT)
                                            - 1)
                                            << bshift);
                                        flagsval |= xlrec.members[i as usize].status << bshift;
                                        LittleEndian::write_u32(
                                            &mut page[flagsoff..flagsoff + 4],
                                            flagsval,
                                        );
                                        LittleEndian::write_u32(
                                            &mut page[memberoff..memberoff + 4],
                                            xlrec.members[i as usize].xid,
                                        );
                                    }
                                }
                            } else {
                                // Multixact offsets SLRU
                                let offs = (xlrec.mid
                                    % pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32
                                    * 4) as usize;
                                LittleEndian::write_u32(&mut page[offs..offs + 4], xlrec.moff);
                            }
                        } else {
-                            panic!();
+                            assert!(tag.rel.forknum == pg_constants::PG_MXACT_MEMBERS_FORKNUM);
                            for i in 0..xlrec.nmembers {
                                let blkno = i / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
                                if blkno == tag.blknum {
                                    // update only target block
                                    let offset = xlrec.moff + i;
                                    let memberoff = mx_offset_to_member_offset(offset);
                                    let flagsoff = mx_offset_to_flags_offset(offset);
                                    let bshift = mx_offset_to_flags_bitshift(offset);
                                    let mut flagsval =
                                        LittleEndian::read_u32(&page[flagsoff..flagsoff + 4]);
                                    flagsval &=
                                        !(((1 << pg_constants::MXACT_MEMBER_BITS_PER_XACT) - 1)
                                            << bshift);
                                    flagsval |= xlrec.members[i as usize].status << bshift;
                                    LittleEndian::write_u32(
                                        &mut page[flagsoff..flagsoff + 4],
                                        flagsval,
                                    );
                                    LittleEndian::write_u32(
                                        &mut page[memberoff..memberoff + 4],
                                        xlrec.members[i as usize].xid,
                                    );
                                }
                            }
                        }
                    } else if info == pg_constants::XLOG_MULTIXACT_TRUNCATE_ID {
                        // empty page image indicates that this SLRU page is truncated and can be removed by GC
                        page.clear();
                    } else {
-                        panic!();
+                        assert!(false);
                    }
                } else if xlogrec.xl_rmid == pg_constants::RM_RELMAP_ID {
                    // Relation map file has size 512 bytes
                    page.clear();
                    page.extend_from_slice(&buf[12..]); // skip xl_relmap_update
                    assert!(page.len() == 512); // size of pg_filenode.map
@@ -427,13 +460,15 @@ impl PostgresRedoManager {
            }
            apply_result = Ok::<Bytes, Error>(page.freeze());
        } else {
            apply_result = process.apply_wal_records(tag, base_img, records).await;
        }
        let duration = start.elapsed();
        let result: Result<Bytes, WalRedoError>;
-        debug!(
+        trace!(
            "applied {} WAL records in {} ms to reconstruct page image at LSN {}",
            nrecords,
            duration.as_millis(),
@@ -454,9 +489,6 @@ impl PostgresRedoManager {
    }
 }
 ///
 /// Handle to the Postgres WAL redo process
 ///
 struct PostgresRedoProcess {
    stdin: RefCell<ChildStdin>,
    stdout: RefCell<ChildStdout>,
@@ -466,15 +498,12 @@ impl PostgresRedoProcess {
    //
    // Start postgres binary in special WAL redo mode.
    //
-    async fn launch(
+    // Tests who run pageserver binary are setting proper PG_BIN_DIR
-        conf: &PageServerConf,
+    // and PG_LIB_DIR so that WalRedo would start right postgres.
        tenantid: &ZTenantId,
    ) -> Result<PostgresRedoProcess, Error> {
        // FIXME: We need a dummy Postgres cluster to run the process in. Currently, we
        // just create one with constant name. That fails if you try to launch more than
        // one WAL redo manager concurrently.
        let datadir = conf.tenant_path(&tenantid).join("wal-redo-datadir");
    // do that: We may later
    // switch to setting same things in pageserver config file.
    async fn launch(datadir: &Path) -> Result<PostgresRedoProcess, Error> {
        // Create empty data directory for wal-redo postgres, deleting old one first.
        if datadir.exists() {
            info!("directory {:?} exists, removing", &datadir);
@@ -483,12 +512,9 @@ impl PostgresRedoProcess {
            }
        }
        info!("running initdb in {:?}", datadir.display());
-        let initdb = Command::new(conf.pg_bin_dir().join("initdb"))
+        let initdb = Command::new("initdb")
            .args(&["-D", datadir.to_str().unwrap()])
            .arg("-N")
            .env_clear()
            .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
            .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
            .output()
            .await
            .expect("failed to execute initdb");
@@ -510,15 +536,12 @@ impl PostgresRedoProcess {
            config.write_all(b"zenith.wal_redo=on\n")?;
        }
        // Start postgres itself
-        let mut child = Command::new(conf.pg_bin_dir().join("postgres"))
+        let mut child = Command::new("postgres")
            .arg("--wal-redo")
            .stdin(Stdio::piped())
            .stderr(Stdio::piped())
            .stdout(Stdio::piped())
-            .env_clear()
+            .env("PGDATA", datadir)
            .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
            .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
            .env("PGDATA", &datadir)
            .spawn()
            .expect("postgres --wal-redo command failed to start");
@@ -598,8 +621,6 @@ impl PostgresRedoProcess {
            for rec in records.iter() {
                let r = rec.clone();
                WAL_REDO_RECORD_COUNTER.inc();
                stdin
                    .write_all(&build_apply_record_msg(r.lsn, r.rec))
                    .await?;
@@ -644,26 +665,9 @@ fn build_begin_redo_for_block_msg(tag: BufferTag) -> Bytes {
    buf.put_u8(b'B');
    buf.put_u32(len as u32);
    tag.pack(&mut buf);
-    // FIXME: this is a temporary hack that should go away when we refactor
+    assert!(buf.len() == 1 + len);
    // the postgres protocol serialization + handlers.
    //
    // BytesMut is a dynamic growable buffer, used a lot in tokio code but
    // not in the std library. To write to a BytesMut from a serde serializer,
    // we need to either:
    // - pre-allocate the required buffer space. This is annoying because we
    //   shouldn't care what the exact serialized size is-- that's the
    //   serializer's job.
    // - Or, we need to create a temporary "writer" (which implements the
    //   `Write` trait). It's a bit awkward, because the writer consumes the
    //   underlying BytesMut, and we need to extract it later with
    //   `into_inner`.
    let mut writer = buf.writer();
    tag.ser_into(&mut writer)
        .expect("serialize BufferTag should always succeed");
    let buf = writer.into_inner();
    debug_assert!(buf.len() == 1 + len);
    buf.freeze()
 }
@@ -676,13 +680,10 @@ fn build_push_page_msg(tag: BufferTag, base_img: Bytes) -> Bytes {
    buf.put_u8(b'P');
    buf.put_u32(len as u32);
-    let mut writer = buf.writer();
+    tag.pack(&mut buf);
    tag.ser_into(&mut writer)
        .expect("serialize BufferTag should always succeed");
    let mut buf = writer.into_inner();
    buf.put(base_img);
-    debug_assert!(buf.len() == 1 + len);
+    assert!(buf.len() == 1 + len);
    buf.freeze()
 }
@@ -696,7 +697,7 @@ fn build_apply_record_msg(endlsn: Lsn, rec: Bytes) -> Bytes {
    buf.put_u64(endlsn.0);
    buf.put(rec);
-    debug_assert!(buf.len() == 1 + len);
+    assert!(buf.len() == 1 + len);
    buf.freeze()
 }
@@ -707,12 +708,9 @@ fn build_get_page_msg(tag: BufferTag) -> Bytes {
    buf.put_u8(b'G');
    buf.put_u32(len as u32);
-    let mut writer = buf.writer();
+    tag.pack(&mut buf);
    tag.ser_into(&mut writer)
        .expect("serialize BufferTag should always succeed");
    let buf = writer.into_inner();
-    debug_assert!(buf.len() == 1 + len);
+    assert!(buf.len() == 1 + len);
    buf.freeze()
 }
@@ -17,10 +17,8 @@ crc32c = "0.6.0"
 hex = "0.4.3"
 lazy_static = "1.4"
 log = "0.4.14"
 memoffset = "0.6.2"
 thiserror = "1.0"
 workspace_hack = { path = "../workspace_hack" }
 zenith_utils = { path = "../zenith_utils" }
 [build-dependencies]
 bindgen = "0.57"
@@ -1,25 +1,3 @@
-This module contains utilities for working with PostgreSQL file
+This module contains utility functions for interacting with PostgreSQL
-formats. It's a collection of structs that are auto-generated from the
+file formats.
 PostgreSQL header files using bindgen, and Rust functions to read and
 manipulate them.
 There are also a bunch of constants in `pg_constants.rs` that are copied
 from various PostgreSQL headers, rather than auto-generated. They mostly
 should be auto-generated too, but that's a TODO.
 The PostgreSQL on-disk file format is not portable across different
 CPU architectures and operating systems. It is also subject to change
 in each major PostgreSQL version. Currently, this module is based on
 PostgreSQL v14, but in the future we will probably need a separate
 copy for each PostgreSQL version.
 To interact with the C structs, there is some unsafe code in this
 module. Do not copy-paste that to the rest of the codebase! Keep the
 amount of unsafe code to a minimum, and limited to this module only,
 and only where it's truly needed.
 TODO: Currently, there is also some code that deals with WAL records
 in pageserver/src/waldecoder.rs.  That should be moved into this
 module. The rest of the codebase should not have intimate knowledge of
 PostgreSQL file formats or WAL layout, that knowledge should be
 encapsulated in this module.
@@ -11,42 +11,29 @@ fn main() {
    // to bindgen, and lets you build up options for
    // the resulting bindings.
    let bindings = bindgen::Builder::default()
-        //
+        // The input header we would like to generate
-        // All the needed PostgreSQL headers are included from 'pg_control_ffi.h'
+        // bindings for.
        //
        .header("pg_control_ffi.h")
        //
        // Tell cargo to invalidate the built crate whenever any of the
        // included header files changed.
        //
        .parse_callbacks(Box::new(bindgen::CargoCallbacks))
        //
        // These are the types and constants that we want to generate bindings for
        //
        .whitelist_type("ControlFileData")
        .whitelist_type("CheckPoint")
        .whitelist_type("FullTransactionId")
        .whitelist_type("XLogRecord")
        .whitelist_type("XLogPageHeaderData")
        .whitelist_type("XLogLongPageHeaderData")
        .whitelist_var("XLOG_PAGE_MAGIC")
        .whitelist_var("PG_CONTROL_FILE_SIZE")
        .whitelist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
        .whitelist_type("DBState")
        //
        // Path the server include dir. It is in tmp_install/include/server, if you did
        // "configure --prefix=<path to tmp_install>". But if you used "configure --prefix=/",
        // and used DESTDIR to move it into tmp_install, then it's in
        // tmp_install/include/postgres/server
        // 'pg_config --includedir-server' would perhaps be the more proper way to find it,
        // but this will do for now.
        //
        .clang_arg("-I../tmp_install/include/server")
        .clang_arg("-I../tmp_install/include/postgresql/server")
        //
        // Finish the builder and generate the bindings.
        //
        .generate()
        // Unwrap the Result and panic on failure.
        .expect("Unable to generate bindings");
    // Write the bindings to the $OUT_DIR/bindings.rs file.
@@ -1,10 +1,4 @@
 /*
 * This header file is the input to bindgen. It includes all the
 * PostgreSQL headers that we need to auto-generate Rust structs
 * from. If you need to expose a new struct to Rust code, add the
 * header here, and whitelist the struct in the build.rs file.
 */
 #include "c.h"
 #include "catalog/pg_control.h"
 #include "access/xlog_internal.h"
 const uint32 PG_CONTROLFILEDATA_OFFSETOF_CRC = offsetof(ControlFileData, crc);
@@ -1,98 +0,0 @@
 # PostgreSQL Client Authentication Configuration File
 # ===================================================
 #
 # Refer to the "Client Authentication" section in the PostgreSQL
 # documentation for a complete description of this file.  A short
 # synopsis follows.
 #
 # This file controls: which hosts are allowed to connect, how clients
 # are authenticated, which PostgreSQL user names they can use, which
 # databases they can access.  Records take one of these forms:
 #
 # local         DATABASE  USER  METHOD  [OPTIONS]
 # host          DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
 # hostssl       DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
 # hostnossl     DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
 # hostgssenc    DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
 # hostnogssenc  DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
 #
 # (The uppercase items must be replaced by actual values.)
 #
 # The first field is the connection type:
 # - "local" is a Unix-domain socket
 # - "host" is a TCP/IP socket (encrypted or not)
 # - "hostssl" is a TCP/IP socket that is SSL-encrypted
 # - "hostnossl" is a TCP/IP socket that is not SSL-encrypted
 # - "hostgssenc" is a TCP/IP socket that is GSSAPI-encrypted
 # - "hostnogssenc" is a TCP/IP socket that is not GSSAPI-encrypted
 #
 # DATABASE can be "all", "sameuser", "samerole", "replication", a
 # database name, or a comma-separated list thereof. The "all"
 # keyword does not match "replication". Access to replication
 # must be enabled in a separate record (see example below).
 #
 # USER can be "all", a user name, a group name prefixed with "+", or a
 # comma-separated list thereof.  In both the DATABASE and USER fields
 # you can also write a file name prefixed with "@" to include names
 # from a separate file.
 #
 # ADDRESS specifies the set of hosts the record matches.  It can be a
 # host name, or it is made up of an IP address and a CIDR mask that is
 # an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that
 # specifies the number of significant bits in the mask.  A host name
 # that starts with a dot (.) matches a suffix of the actual host name.
 # Alternatively, you can write an IP address and netmask in separate
 # columns to specify the set of hosts.  Instead of a CIDR-address, you
 # can write "samehost" to match any of the server's own IP addresses,
 # or "samenet" to match any address in any subnet that the server is
 # directly connected to.
 #
 # METHOD can be "trust", "reject", "md5", "password", "scram-sha-256",
 # "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert".
 # Note that "password" sends passwords in clear text; "md5" or
 # "scram-sha-256" are preferred since they send encrypted passwords.
 #
 # OPTIONS are a set of options for the authentication in the format
 # NAME=VALUE.  The available options depend on the different
 # authentication methods -- refer to the "Client Authentication"
 # section in the documentation for a list of which options are
 # available for which authentication methods.
 #
 # Database and user names containing spaces, commas, quotes and other
 # special characters must be quoted.  Quoting one of the keywords
 # "all", "sameuser", "samerole" or "replication" makes the name lose
 # its special character, and just match a database or username with
 # that name.
 #
 # This file is read on server startup and when the server receives a
 # SIGHUP signal.  If you edit the file on a running system, you have to
 # SIGHUP the server for the changes to take effect, run "pg_ctl reload",
 # or execute "SELECT pg_reload_conf()".
 #
 # Put your actual configuration here
 # ----------------------------------
 #
 # If you want to allow non-local connections, you need to add more
 # "host" records.  In that case you will also need to make PostgreSQL
 # listen on a non-local interface via the listen_addresses
 # configuration parameter, or via the -i or -h command line switches.
 # CAUTION: Configuring the system for local "trust" authentication
 # allows any local user to connect as any PostgreSQL user, including
 # the database superuser.  If you do not trust all your local users,
 # use another authentication method.
 # TYPE  DATABASE        USER            ADDRESS                 METHOD
 # "local" is for Unix domain socket connections only
 local   all             all                                     trust
 # IPv4 local connections:
 host    all             all             127.0.0.1/32            trust
 # IPv6 local connections:
 host    all             all             ::1/128                 trust
 # Allow replication connections from localhost, by a user with the
 # replication privilege.
 local   replication     all                                     trust
 host    replication     all             127.0.0.1/32            trust
 host    replication     all             ::1/128                 trust
@@ -1,124 +0,0 @@
 //!
 //! Utilities for reading and writing the PostgreSQL control file.
 //!
 //! The PostgreSQL control file is one the first things that the PostgreSQL
 //! server reads when it starts up. It indicates whether the server was shut
 //! down cleanly, or if it crashed or was restored from online backup so that
 //! WAL recovery needs to be performed. It also contains a copy of the latest
 //! checkpoint record and its location in the WAL.
 //!
 //! The control file also contains fields for detecting whether the
 //! data directory is compatible with a postgres binary. That includes
 //! a version number, configuration options that can be set at
 //! compilation time like the block size, and the platform's alignment
 //! and endianess information. (The PostgreSQL on-disk file format is
 //! not portable across platforms.)
 //!
 //! The control file is stored in the PostgreSQL data directory, as
 //! `global/pg_control`. The data stored in it is designed to be smaller than
 //! 512 bytes, on the assumption that it can be updated atomically. The actual
 //! file is larger, 8192 bytes, but the rest of it is just filled with zeros.
 //!
 //! See src/include/catalog/pg_control.h in the PostgreSQL sources for more
 //! information. You can use PostgreSQL's pg_controldata utility to view its
 //! contents.
 //!
 use crate::{ControlFileData, PG_CONTROL_FILE_SIZE};
 use anyhow::{bail, Result};
 use bytes::{Bytes, BytesMut};
 /// Equivalent to sizeof(ControlFileData) in C
 const SIZEOF_CONTROLDATA: usize = std::mem::size_of::<ControlFileData>();
 impl ControlFileData {
    /// Compute the offset of the `crc` field within the `ControlFileData` struct.
    /// Equivalent to offsetof(ControlFileData, crc) in C.
    // Someday this can be const when the right compiler features land.
    fn pg_control_crc_offset() -> usize {
        memoffset::offset_of!(ControlFileData, crc)
    }
    ///
    /// Interpret a slice of bytes as a Postgres control file.
    ///
    pub fn decode(buf: &[u8]) -> Result<ControlFileData> {
        // Check that the slice has the expected size. The control file is
        // padded with zeros up to a 512 byte sector size, so accept a
        // larger size too, so that the caller can just the whole file
        // contents without knowing the exact size of the struct.
        if buf.len() < SIZEOF_CONTROLDATA {
            bail!("control file is too short");
        }
        // Compute the expected CRC of the content.
        let OFFSETOF_CRC = Self::pg_control_crc_offset();
        let expectedcrc = crc32c::crc32c(&buf[0..OFFSETOF_CRC]);
        // Convert the slice into an array of the right size, and use `transmute` to
        // reinterpret the raw bytes as a ControlFileData struct.
        //
        // NB: Ideally we would use 'zerocopy::FromBytes' for this, but bindgen doesn't
        // derive FromBytes for us. The safety of this depends on the same constraints
        // as for FromBytes, namely, all of its fields must implement FromBytes. That
        // includes the primitive integer types, like `u8`, `u16`, `u32`, `u64` and their
        // signed variants. But `bool` is not safe, because the contents of the high bits
        // in a rust bool are undefined. In practice, PostgreSQL uses 1 to represent
        // true and 0 for false, which is compatible with Rust bool, but let's try not to
        // depend on it.
        //
        // FIXME: ControlFileData does contain 'bool's at the moment.
        //
        // See https://github.com/zenithdb/zenith/issues/207 for discussion on the safety
        // of this.
        let mut b: [u8; SIZEOF_CONTROLDATA] = [0u8; SIZEOF_CONTROLDATA];
        b.copy_from_slice(&buf[0..SIZEOF_CONTROLDATA]);
        let controlfile: ControlFileData =
            unsafe { std::mem::transmute::<[u8; SIZEOF_CONTROLDATA], ControlFileData>(b) };
        // Check the CRC
        if expectedcrc != controlfile.crc {
            bail!(
                "invalid CRC in control file: expected {:08X}, was {:08X}",
                expectedcrc,
                controlfile.crc
            );
        }
        Ok(controlfile)
    }
    ///
    /// Convert a struct representing a Postgres control file into raw bytes.
    ///
    /// The CRC is recomputed to match the contents of the fields.
    pub fn encode(&self) -> Bytes {
        //
        // Use `transmute` to reinterpret struct as raw bytes.
        //
        // FIXME: This triggers undefined behavior, because the contents
        // of the padding bytes are undefined, and this leaks those
        // undefined bytes into the resulting array. The Rust code won't
        // care what's in those bytes, and PostgreSQL doesn't care
        // either. HOWEVER, it is a potential security issue, because the
        // bytes can contain arbitrary pieces of memory from the page
        // server. In the worst case, that could be private keys or
        // another tenant's data.
        //
        // See https://github.com/zenithdb/zenith/issues/207 for discussion.
        let b: [u8; SIZEOF_CONTROLDATA] =
            unsafe { std::mem::transmute::<ControlFileData, [u8; SIZEOF_CONTROLDATA]>(*self) };
        // Recompute the CRC
        let OFFSETOF_CRC = Self::pg_control_crc_offset();
        let newcrc = crc32c::crc32c(&b[0..OFFSETOF_CRC]);
        let mut buf = BytesMut::with_capacity(PG_CONTROL_FILE_SIZE as usize);
        buf.extend_from_slice(&b[0..OFFSETOF_CRC]);
        buf.extend_from_slice(&newcrc.to_ne_bytes());
        // Fill the rest of the control file with zeros.
        buf.resize(PG_CONTROL_FILE_SIZE as usize, 0);
        buf.into()
    }
 }
@@ -1,13 +1,112 @@
 #![allow(non_upper_case_globals)]
 #![allow(non_camel_case_types)]
 #![allow(non_snake_case)]
 // suppress warnings on rust 1.53 due to bindgen unit tests.
 // https://github.com/rust-lang/rust-bindgen/issues/1651
 #![allow(deref_nullptr)]
 include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
 pub mod controlfile_utils;
 pub mod nonrelfile_utils;
 pub mod pg_constants;
 pub mod relfile_utils;
 pub mod xlog_utils;
 use bytes::{Buf, Bytes, BytesMut};
 // sizeof(ControlFileData)
 const SIZEOF_CONTROLDATA: usize = std::mem::size_of::<ControlFileData>();
 const SIZEOF_CHECKPOINT: usize = std::mem::size_of::<CheckPoint>();
 const OFFSETOF_CRC: usize = PG_CONTROLFILEDATA_OFFSETOF_CRC as usize;
 impl ControlFileData {
    // Initialize an all-zeros ControlFileData struct
    pub fn new() -> ControlFileData {
        let controlfile: ControlFileData;
        let b = [0u8; SIZEOF_CONTROLDATA];
        controlfile =
            unsafe { std::mem::transmute::<[u8; SIZEOF_CONTROLDATA], ControlFileData>(b) };
        controlfile
    }
 }
 pub fn decode_pg_control(mut buf: Bytes) -> Result<ControlFileData, anyhow::Error> {
    let mut b: [u8; SIZEOF_CONTROLDATA] = [0u8; SIZEOF_CONTROLDATA];
    buf.copy_to_slice(&mut b);
    let controlfile: ControlFileData;
    // TODO: verify CRC
    let mut data_without_crc: [u8; OFFSETOF_CRC] = [0u8; OFFSETOF_CRC];
    data_without_crc.copy_from_slice(&b[0..OFFSETOF_CRC]);
    let expectedcrc = crc32c::crc32c(&data_without_crc);
    controlfile = unsafe { std::mem::transmute::<[u8; SIZEOF_CONTROLDATA], ControlFileData>(b) };
    if expectedcrc != controlfile.crc {
        anyhow::bail!(
            "invalid CRC in control file: expected {:08X}, was {:08X}",
            expectedcrc,
            controlfile.crc
        );
    }
    Ok(controlfile)
 }
 pub fn encode_pg_control(controlfile: ControlFileData) -> Bytes {
    let b: [u8; SIZEOF_CONTROLDATA];
    b = unsafe { std::mem::transmute::<ControlFileData, [u8; SIZEOF_CONTROLDATA]>(controlfile) };
    // Recompute the CRC
    let mut data_without_crc: [u8; OFFSETOF_CRC] = [0u8; OFFSETOF_CRC];
    data_without_crc.copy_from_slice(&b[0..OFFSETOF_CRC]);
    let newcrc = crc32c::crc32c(&data_without_crc);
    let mut buf = BytesMut::with_capacity(PG_CONTROL_FILE_SIZE as usize);
    buf.extend_from_slice(&b[0..OFFSETOF_CRC]);
    buf.extend_from_slice(&newcrc.to_ne_bytes());
    // Fill the rest of the control file with zeros.
    buf.resize(PG_CONTROL_FILE_SIZE as usize, 0);
    buf.into()
 }
 pub fn encode_checkpoint(checkpoint: CheckPoint) -> Bytes {
    let b: [u8; SIZEOF_CHECKPOINT];
    b = unsafe { std::mem::transmute::<CheckPoint, [u8; SIZEOF_CHECKPOINT]>(checkpoint) };
    return Bytes::copy_from_slice(&b[..]);
 }
 pub fn decode_checkpoint(mut buf: Bytes) -> Result<CheckPoint, anyhow::Error> {
    let mut b = [0u8; SIZEOF_CHECKPOINT];
    buf.copy_to_slice(&mut b);
    let checkpoint: CheckPoint;
    checkpoint = unsafe { std::mem::transmute::<[u8; SIZEOF_CHECKPOINT], CheckPoint>(b) };
    Ok(checkpoint)
 }
 impl CheckPoint {
    pub fn new(lsn: u64, timeline: u32) -> CheckPoint {
        CheckPoint {
            redo: lsn,
            ThisTimeLineID: timeline,
            PrevTimeLineID: timeline,
            fullPageWrites: true, // TODO: get actual value of full_page_writes
            nextXid: FullTransactionId {
                value: pg_constants::FIRST_NORMAL_TRANSACTION_ID as u64,
            }, // TODO: handle epoch?
            nextOid: pg_constants::FIRST_BOOTSTRAP_OBJECT_ID,
            nextMulti: 1,
            nextMultiOffset: 0,
            oldestXid: pg_constants::FIRST_NORMAL_TRANSACTION_ID,
            oldestXidDB: 0,
            oldestMulti: 1,
            oldestMultiDB: 0,
            time: 0,
            oldestCommitTsXid: 0,
            newestCommitTsXid: 0,
            oldestActiveXid: pg_constants::INVALID_TRANSACTION_ID,
        }
    }
 }
@@ -28,5 +28,5 @@ pub fn transaction_id_get_status(xid: u32, page: &[u8]) -> u8 {
    let bshift: u8 =
        ((xid % pg_constants::CLOG_XACTS_PER_BYTE) * pg_constants::CLOG_BITS_PER_XACT as u32) as u8;
-    ((page[byteno] >> bshift) & pg_constants::CLOG_XACT_BITMASK) as u8
+    return ((page[byteno] >> bshift) & pg_constants::CLOG_XACT_BITMASK) as u8;
 }
@@ -1,11 +1,6 @@
 //!
 //! Misc constants, copied from PostgreSQL headers.
 //!
 //! TODO: These probably should be auto-generated using bindgen,
 //! rather than copied by hand. Although on the other hand, it's nice
 //! to have them all here in one place, and have the ability to add
 //! comments on them.
 //!
 //
 // From pg_tablespace_d.h
@@ -20,11 +15,18 @@ pub const MAIN_FORKNUM: u8 = 0;
 pub const FSM_FORKNUM: u8 = 1;
 pub const VISIBILITYMAP_FORKNUM: u8 = 2;
 pub const INIT_FORKNUM: u8 = 3;
 // Special values for non-rel files' tags (Zenith-specific)
 //Special values for non-rel files' tags
 pub const PG_CONTROLFILE_FORKNUM: u8 = 42;
 pub const PG_FILENODEMAP_FORKNUM: u8 = 43;
 pub const PG_XACT_FORKNUM: u8 = 44;
 pub const PG_MXACT_OFFSETS_FORKNUM: u8 = 45;
 pub const PG_MXACT_MEMBERS_FORKNUM: u8 = 46;
 pub const PG_TWOPHASE_FORKNUM: u8 = 47;
 pub const PG_CHECKPOINT_FORKNUM: u8 = 48;
 // From storage_xlog.h
 pub const SMGR_TRUNCATE_HEAP: u32 = 0x0001;
 pub const SMGR_TRUNCATE_VM: u32 = 0x0002;
 pub const SMGR_TRUNCATE_FSM: u32 = 0x0004;
 // from pg_config.h. These can be changed with configure options --with-blocksize=BLOCKSIZE and
 // --with-segsize=SEGSIZE, but assume the defaults for now.
@@ -46,6 +48,7 @@ pub const SIZE_OF_PAGE_HEADER: u16 = 24;
 pub const BITS_PER_HEAPBLOCK: u16 = 2;
 pub const HEAPBLOCKS_PER_PAGE: u16 = (BLCKSZ - SIZE_OF_PAGE_HEADER) * 8 / BITS_PER_HEAPBLOCK;
 pub const TRANSACTION_STATUS_IN_PROGRESS: u8 = 0x00;
 pub const TRANSACTION_STATUS_COMMITTED: u8 = 0x01;
 pub const TRANSACTION_STATUS_ABORTED: u8 = 0x02;
 pub const TRANSACTION_STATUS_SUB_COMMITTED: u8 = 0x03;
@@ -62,11 +65,9 @@ pub const XLOG_XACT_ABORT_PREPARED: u8 = 0x40;
 // From srlu.h
 pub const SLRU_PAGES_PER_SEGMENT: u32 = 32;
 pub const SLRU_SEG_SIZE: usize = BLCKSZ as usize * SLRU_PAGES_PER_SEGMENT as usize;
 /* mask for filtering opcodes out of xl_info */
 pub const XLOG_XACT_OPMASK: u8 = 0x70;
 pub const XLOG_HEAP_OPMASK: u8 = 0x70;
 /* does this record have a 'xinfo' field or not */
 pub const XLOG_XACT_HAS_INFO: u8 = 0x80;
@@ -87,12 +88,8 @@ pub const XACT_XINFO_HAS_TWOPHASE: u32 = 1u32 << 4;
 pub const XLOG_NEXTOID: u8 = 0x30;
 pub const XLOG_SWITCH: u8 = 0x40;
 pub const XLOG_SMGR_TRUNCATE: u8 = 0x20;
 pub const DB_SHUTDOWNED: u32 = 1;
 // From multixact.h
 pub const FIRST_MULTIXACT_ID: u32 = 1;
 pub const MAX_MULTIXACT_ID: u32 = 0xFFFFFFFF;
 pub const XLOG_MULTIXACT_ZERO_OFF_PAGE: u8 = 0x00;
 pub const XLOG_MULTIXACT_ZERO_MEM_PAGE: u8 = 0x10;
 pub const XLOG_MULTIXACT_CREATE_ID: u8 = 0x20;
@@ -179,46 +176,3 @@ pub const FIRST_NORMAL_OBJECT_ID: u32 = 16384;
 /* FIXME: pageserver should request wal_seg_size from compute node */
 pub const WAL_SEGMENT_SIZE: usize = 16 * 1024 * 1024;
 pub const XLOG_BLCKSZ: usize = 8192;
 pub const XLOG_CHECKPOINT_SHUTDOWN: u8 = 0x00;
 pub const XLOG_CHECKPOINT_ONLINE: u8 = 0x10;
 pub const XLP_LONG_HEADER: u16 = 0x0002;
 pub const PG_MAJORVERSION: &'static str = "14";
 // List of subdirectories inside pgdata.
 // Copied from src/bin/initdb/initdb.c
 pub const PGDATA_SUBDIRS: [&'static str; 22] = [
    "global",
    "pg_wal/archive_status",
    "pg_commit_ts",
    "pg_dynshmem",
    "pg_notify",
    "pg_serial",
    "pg_snapshots",
    "pg_subtrans",
    "pg_twophase",
    "pg_multixact",
    "pg_multixact/members",
    "pg_multixact/offsets",
    "base",
    "base/1",
    "pg_replslot",
    "pg_tblspc",
    "pg_stat",
    "pg_stat_tmp",
    "pg_xact",
    "pg_logical",
    "pg_logical/snapshots",
    "pg_logical/mappings",
 ];
 pub const PGDATA_SPECIAL_FILES: [&'static str; 4] = [
    "pg_hba.conf",
    "pg_ident.conf",
    "postgresql.conf",
    "postgresql.auto.conf",
 ];
 pub static PG_HBA: &'static str = include_str!("../samples/pg_hba.conf");
@@ -38,6 +38,16 @@ pub fn forknumber_to_name(forknum: u8) -> Option<&'static str> {
        pg_constants::FSM_FORKNUM => Some("fsm"),
        pg_constants::VISIBILITYMAP_FORKNUM => Some("vm"),
        pg_constants::INIT_FORKNUM => Some("init"),
        // These should not appear in WAL records, but we use them internally,
        // and need to be prepared to print them out in log messages and such
        pg_constants::PG_CONTROLFILE_FORKNUM => Some("controlfile"),
        pg_constants::PG_FILENODEMAP_FORKNUM => Some("filenodemap"),
        pg_constants::PG_XACT_FORKNUM => Some("xact"),
        pg_constants::PG_MXACT_OFFSETS_FORKNUM => Some("mxact_offsets"),
        pg_constants::PG_MXACT_MEMBERS_FORKNUM => Some("mxact_members"),
        pg_constants::PG_TWOPHASE_FORKNUM => Some("twophase"),
        _ => Some("UNKNOWN FORKNUM"),
    }
 }
@@ -8,17 +8,8 @@
 //
 use crate::pg_constants;
 use crate::CheckPoint;
 use crate::ControlFileData;
 use crate::FullTransactionId;
 use crate::XLogLongPageHeaderData;
 use crate::XLogPageHeaderData;
 use crate::XLogRecord;
 use crate::XLOG_PAGE_MAGIC;
 use byteorder::{ByteOrder, LittleEndian};
 use bytes::{Buf, Bytes};
 use bytes::{BufMut, BytesMut};
 use crc32c::*;
 use log::*;
 use std::cmp::min;
@@ -30,22 +21,19 @@ use std::time::SystemTime;
 pub const XLOG_FNAME_LEN: usize = 24;
 pub const XLOG_BLCKSZ: usize = 8192;
 pub const XLP_FIRST_IS_CONTRECORD: u16 = 0x0001;
 pub const XLOG_PAGE_MAGIC: u16 = 0xD109;
 pub const XLP_REM_LEN_OFFS: usize = 2 + 2 + 4 + 8;
 pub const XLOG_SIZE_OF_XLOG_SHORT_PHD: usize = XLP_REM_LEN_OFFS + 4 + 4;
 pub const XLOG_SIZE_OF_XLOG_LONG_PHD: usize = XLOG_SIZE_OF_XLOG_SHORT_PHD + 8 + 4 + 4;
 pub const XLOG_RECORD_CRC_OFFS: usize = 4 + 4 + 8 + 1 + 1 + 2;
 pub const XLOG_SIZE_OF_XLOG_RECORD: usize = XLOG_RECORD_CRC_OFFS + 4;
 pub const MAX_SEND_SIZE: usize = XLOG_BLCKSZ * 16;
 pub const XLOG_SIZE_OF_XLOG_SHORT_PHD: usize = std::mem::size_of::<XLogPageHeaderData>();
 pub const XLOG_SIZE_OF_XLOG_LONG_PHD: usize = std::mem::size_of::<XLogLongPageHeaderData>();
 pub const XLOG_SIZE_OF_XLOG_RECORD: usize = std::mem::size_of::<XLogRecord>();
 pub const SIZE_OF_XLOG_RECORD_DATA_HEADER_SHORT: usize = 1 * 2;
 pub type XLogRecPtr = u64;
 pub type TimeLineID = u32;
-pub type TimestampTz = i64;
+pub type TimestampTz = u64;
 pub type XLogSegNo = u64;
 const XID_CHECKPOINT_INTERVAL: u32 = 1024;
 #[allow(non_snake_case)]
 pub fn XLogSegmentsPerXLogId(wal_segsz_bytes: usize) -> XLogSegNo {
    (0x100000000u64 / wal_segsz_bytes as u64) as XLogSegNo
@@ -95,9 +83,9 @@ pub fn get_current_timestamp() -> TimestampTz {
    const USECS_PER_SEC: u64 = 1000000;
    match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) {
        Ok(n) => {
-            ((n.as_secs() - ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY))
+            (n.as_secs() - ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY))
                * USECS_PER_SEC
-                + n.subsec_micros() as u64) as i64
+                + n.subsec_micros() as u64
        }
        Err(_) => panic!("SystemTime before UNIX EPOCH!"),
    }
@@ -108,23 +96,17 @@ fn find_end_of_wal_segment(
    segno: XLogSegNo,
    tli: TimeLineID,
    wal_seg_size: usize,
    is_partial: bool,
    rec_offs: &mut usize,
    rec_hdr: &mut [u8; XLOG_SIZE_OF_XLOG_RECORD],
    crc: &mut u32,
    check_contrec: bool,
 ) -> u32 {
    let mut offs: usize = 0;
    let mut contlen: usize = 0;
    let mut wal_crc: u32 = 0;
    let mut crc: u32 = 0;
    let mut rec_offs: usize = 0;
    let mut buf = [0u8; XLOG_BLCKSZ];
    let file_name = XLogFileName(tli, segno, wal_seg_size);
    let mut last_valid_rec_pos: usize = 0;
-    let file_path = data_dir.join(if is_partial {
+    let mut file = File::open(data_dir.join(file_name.clone() + ".partial")).unwrap();
-        file_name.clone() + ".partial"
+    let mut rec_hdr = [0u8; XLOG_RECORD_CRC_OFFS];
    } else {
        file_name
    });
    let mut file = File::open(&file_path).unwrap();
    while offs < wal_seg_size {
        if offs % XLOG_BLCKSZ == 0 {
@@ -138,34 +120,14 @@ fn find_end_of_wal_segment(
            let xlp_magic = LittleEndian::read_u16(&buf[0..2]);
            let xlp_info = LittleEndian::read_u16(&buf[2..4]);
            let xlp_rem_len = LittleEndian::read_u32(&buf[XLP_REM_LEN_OFFS..XLP_REM_LEN_OFFS + 4]);
-            if xlp_magic != XLOG_PAGE_MAGIC as u16 {
+            if xlp_magic != XLOG_PAGE_MAGIC {
-                info!("Invalid WAL file {:?} magic {}", &file_path, xlp_magic);
+                info!("Invalid WAL file {}.partial magic {}", file_name, xlp_magic);
                break;
            }
            if offs == 0 {
                offs = XLOG_SIZE_OF_XLOG_LONG_PHD;
                if (xlp_info & XLP_FIRST_IS_CONTRECORD) != 0 {
-                    if check_contrec {
+                    offs += ((xlp_rem_len + 7) & !7) as usize;
                        let xl_tot_len = LittleEndian::read_u32(&rec_hdr[0..4]) as usize;
                        contlen = xlp_rem_len as usize;
                        if *rec_offs + contlen < xl_tot_len
                            || (*rec_offs + contlen != xl_tot_len
                                && contlen != XLOG_BLCKSZ - XLOG_SIZE_OF_XLOG_LONG_PHD)
                        {
                            info!(
                                "Corrupted continuation record: offs={}, contlen={}, xl_tot_len={}",
                                *rec_offs, contlen, xl_tot_len
                            );
                            return 0;
                        }
                    } else {
                        offs += ((xlp_rem_len + 7) & !7) as usize;
                    }
                } else if *rec_offs != 0 {
                    // There is incompleted page at previous segment but no cont record:
                    // it means that current segment is not valid and we have to return back.
                    info!("CONTRECORD flag is missed in page header");
                    return 0;
                }
            } else {
                offs += XLOG_SIZE_OF_XLOG_SHORT_PHD;
@@ -176,8 +138,9 @@ fn find_end_of_wal_segment(
            if xl_tot_len == 0 {
                break;
            }
            last_valid_rec_pos = offs;
            offs += 4;
-            *rec_offs = 4;
+            rec_offs = 4;
            contlen = xl_tot_len - 4;
            rec_hdr[0..4].copy_from_slice(&buf[page_offs..page_offs + 4]);
        } else {
@@ -187,33 +150,34 @@ fn find_end_of_wal_segment(
            // read the rest of the record, or as much as fits on this page.
            let n = min(contlen, pageleft);
-            let mut hdr_len: usize = 0;
+            if rec_offs < XLOG_RECORD_CRC_OFFS {
-            if *rec_offs < XLOG_SIZE_OF_XLOG_RECORD {
+                let len = min(XLOG_RECORD_CRC_OFFS - rec_offs, n);
-                // copy header
+                rec_hdr[rec_offs..rec_offs + len].copy_from_slice(&buf[page_offs..page_offs + len]);
                hdr_len = min(XLOG_SIZE_OF_XLOG_RECORD - *rec_offs, n);
                rec_hdr[*rec_offs..*rec_offs + hdr_len]
                    .copy_from_slice(&buf[page_offs..page_offs + hdr_len]);
            }
-            *crc = crc32c_append(*crc, &buf[page_offs + hdr_len..page_offs + n]);
+            if rec_offs <= XLOG_RECORD_CRC_OFFS && rec_offs + n >= XLOG_SIZE_OF_XLOG_RECORD {
-            *rec_offs += n;
+                let crc_offs = page_offs - rec_offs + XLOG_RECORD_CRC_OFFS;
                wal_crc = LittleEndian::read_u32(&buf[crc_offs..crc_offs + 4]);
                crc = crc32c_append(0, &buf[crc_offs + 4..page_offs + n]);
                crc = !crc;
            } else {
                crc ^= 0xFFFFFFFFu32;
                crc = crc32c_append(crc, &buf[page_offs..page_offs + n]);
                crc = !crc;
            }
            rec_offs += n;
            offs += n;
            contlen -= n;
            if contlen == 0 {
-                *crc = crc32c_append(*crc, &rec_hdr[0..XLOG_RECORD_CRC_OFFS]);
+                crc = !crc;
                crc = crc32c_append(crc, &rec_hdr);
                offs = (offs + 7) & !7; // pad on 8 bytes boundary */
-                let wal_crc = LittleEndian::read_u32(
+                if crc == wal_crc {
                    &rec_hdr[XLOG_RECORD_CRC_OFFS..XLOG_RECORD_CRC_OFFS + 4],
                );
                if *crc == wal_crc {
                    last_valid_rec_pos = offs;
                    // Reset rec_offs and crc for start of new record
                    *rec_offs = 0;
                    *crc = 0;
                } else {
                    info!(
-                        "CRC mismatch {} vs {} at offset {} lsn {}",
+                        "CRC mismatch {} vs {} at {}",
-                        *crc, wal_crc, offs, last_valid_rec_pos
+                        crc, wal_crc, last_valid_rec_pos
                    );
                    break;
                }
@@ -235,171 +199,51 @@ pub fn find_end_of_wal(
    let mut high_tli: TimeLineID = 0;
    let mut high_ispartial = false;
-    for entry in fs::read_dir(data_dir).unwrap().flatten() {
+    for entry in fs::read_dir(data_dir).unwrap() {
-        let ispartial: bool;
+        if let Ok(entry) = entry {
-        let entry_name = entry.file_name();
+            let ispartial: bool;
-        let fname = entry_name.to_str().unwrap();
+            let entry_name = entry.file_name();
-        /*
+            let fname = entry_name.to_str().unwrap();
-         * Check if the filename looks like an xlog file, or a .partial file.
+            /*
-         */
+             * Check if the filename looks like an xlog file, or a .partial file.
-        if IsXLogFileName(fname) {
+             */
-            ispartial = false;
+            if IsXLogFileName(fname) {
-        } else if IsPartialXLogFileName(fname) {
+                ispartial = false;
-            ispartial = true;
+            } else if IsPartialXLogFileName(fname) {
-        } else {
+                ispartial = true;
-            continue;
+            } else {
-        }
+                continue;
-        let (segno, tli) = XLogFromFileName(fname, wal_seg_size);
+            }
-        if !ispartial && entry.metadata().unwrap().len() != wal_seg_size as u64 {
+            let (segno, tli) = XLogFromFileName(fname, wal_seg_size);
-            continue;
+            if !ispartial && entry.metadata().unwrap().len() != wal_seg_size as u64 {
-        }
+                continue;
-        if segno > high_segno
+            }
-            || (segno == high_segno && tli > high_tli)
+            if segno > high_segno
-            || (segno == high_segno && tli == high_tli && high_ispartial && !ispartial)
+                || (segno == high_segno && tli > high_tli)
-        {
+                || (segno == high_segno && tli == high_tli && high_ispartial && !ispartial)
-            high_segno = segno;
+            {
-            high_tli = tli;
+                high_segno = segno;
-            high_ispartial = ispartial;
+                high_tli = tli;
                high_ispartial = ispartial;
            }
        }
    }
    if high_segno > 0 {
        let mut high_offs = 0;
-        if precise {
+        /*
-            let mut crc: u32 = 0;
+         * Move the starting pointer to the start of the next segment, if the
-            let mut rec_offs: usize = 0;
+         * highest one we saw was completed.
-            let mut rec_hdr = [0u8; XLOG_SIZE_OF_XLOG_RECORD];
+         */
-            let wal_dir = data_dir.join("pg_wal");
+        if !high_ispartial {
-
+            high_segno += 1;
-            /*
+        } else if precise {
-             * To be able to calculate CRC of records crossing segment boundary,
+            /* otherwise locate last record in last partial segment */
-             * we need to parse previous segments.
+            high_offs = find_end_of_wal_segment(data_dir, high_segno, high_tli, wal_seg_size);
             * So first traverse segments in backward direction to locate record start
             * and then traverse forward, accumulating CRC.
             */
            let mut prev_segno = high_segno - 1;
            let mut prev_offs: u32 = 0;
            while prev_segno > 1 {
                // TOFO: first segment constains dummy checkpoint record at the beginning
                prev_offs = find_end_of_wal_segment(
                    data_dir,
                    prev_segno,
                    high_tli,
                    wal_seg_size,
                    false,
                    &mut rec_offs,
                    &mut rec_hdr,
                    &mut crc,
                    false,
                );
                if prev_offs != 0 {
                    break;
                }
                prev_segno -= 1;
            }
            if prev_offs != 0 {
                // found start of WAL record
                let first_segno = prev_segno;
                let first_offs = prev_offs;
                while prev_segno + 1 < high_segno {
                    // now traverse record in forward direction, accumulating CRC
                    prev_segno += 1;
                    prev_offs = find_end_of_wal_segment(
                        data_dir,
                        prev_segno,
                        high_tli,
                        wal_seg_size,
                        false,
                        &mut rec_offs,
                        &mut rec_hdr,
                        &mut crc,
                        true,
                    );
                    if prev_offs == 0 {
                        info!("Segment {} is corrupted", prev_segno,);
                        break;
                    }
                }
                if prev_offs != 0 {
                    high_offs = find_end_of_wal_segment(
                        data_dir,
                        high_segno,
                        high_tli,
                        wal_seg_size,
                        high_ispartial,
                        &mut rec_offs,
                        &mut rec_hdr,
                        &mut crc,
                        true,
                    );
                }
                if high_offs == 0 {
                    // If last segment contais no valid records, then return back
                    info!("Last WAL segment {} contains no valid record, truncate WAL till {} segment",
 						  high_segno, first_segno);
                    // Remove last segments containing corrupted WAL record
                    for segno in first_segno + 1..high_segno {
                        let file_name = XLogFileName(high_tli, segno, wal_seg_size);
                        let file_path = wal_dir.join(file_name);
                        if let Err(e) = fs::remove_file(&file_path) {
                            info!("Failed to remove file {:?}: {}", &file_path, e);
                        }
                    }
                    let file_name = XLogFileName(high_tli, high_segno, wal_seg_size);
                    let file_path = if high_ispartial {
                        wal_dir.join(file_name.clone() + ".partial")
                    } else {
                        wal_dir.join(file_name.clone())
                    };
                    if let Err(e) = fs::remove_file(&file_path) {
                        info!("Failed to remove file {:?}: {}", &file_path, e);
                    }
                    high_ispartial = false; // previous segment should not be partial
                    high_segno = first_segno;
                    high_offs = first_offs;
                }
            } else {
                // failed to locate previous segment
                assert!(prev_segno <= 1);
                high_offs = find_end_of_wal_segment(
                    data_dir,
                    high_segno,
                    high_tli,
                    wal_seg_size,
                    high_ispartial,
                    &mut rec_offs,
                    &mut rec_hdr,
                    &mut crc,
                    false,
                );
            }
            // If last segment is not marked as partial, it means that next segment
            // was not written. Let's make this segment partial once again.
            if !high_ispartial {
                let file_name = XLogFileName(high_tli, high_segno, wal_seg_size);
                if let Err(e) = fs::rename(
                    wal_dir.join(file_name.clone()),
                    wal_dir.join(file_name.clone() + ".partial"),
                ) {
                    info!(
                        "Failed to rename {} to {}.partial: {}",
                        &file_name, &file_name, e
                    );
                }
            }
        } else {
            /*
             * Move the starting pointer to the start of the next segment, if the
             * highest one we saw was completed.
             */
            if !high_ispartial {
                high_segno += 1;
            }
        }
        let high_ptr = XLogSegNoOffsetToRecPtr(high_segno, high_offs, wal_seg_size);
        return (high_ptr, high_tli);
    }
-    (0, 1) // First timeline is 1
+    (0, 0)
 }
 pub fn main() {
@@ -415,6 +259,21 @@ pub fn main() {
    );
 }
 //
 // Xlog record parsing routines
 // TODO move here other related code from waldecoder.rs
 //
 #[repr(C)]
 #[derive(Debug)]
 pub struct XLogRecord {
    pub xl_tot_len: u32,
    pub xl_xid: u32,
    pub xl_prev: u64,
    pub xl_info: u8,
    pub xl_rmid: u8,
    pub xl_crc: u32,
 }
 impl XLogRecord {
    pub fn from_bytes(buf: &mut Bytes) -> XLogRecord {
        XLogRecord {
@@ -430,217 +289,8 @@ impl XLogRecord {
        }
    }
    pub fn encode(&self) -> Bytes {
        let b: [u8; XLOG_SIZE_OF_XLOG_RECORD];
        b = unsafe { std::mem::transmute::<XLogRecord, [u8; XLOG_SIZE_OF_XLOG_RECORD]>(*self) };
        Bytes::copy_from_slice(&b[..])
    }
    // Is this record an XLOG_SWITCH record? They need some special processing,
    pub fn is_xlog_switch_record(&self) -> bool {
        self.xl_info == pg_constants::XLOG_SWITCH && self.xl_rmid == pg_constants::RM_XLOG_ID
    }
 }
 impl XLogPageHeaderData {
    pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogPageHeaderData {
        let hdr: XLogPageHeaderData = XLogPageHeaderData {
            xlp_magic: buf.get_u16_le(),
            xlp_info: buf.get_u16_le(),
            xlp_tli: buf.get_u32_le(),
            xlp_pageaddr: buf.get_u64_le(),
            xlp_rem_len: buf.get_u32_le(),
        };
        buf.get_u32_le(); //padding
        hdr
    }
 }
 impl XLogLongPageHeaderData {
    pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogLongPageHeaderData {
        XLogLongPageHeaderData {
            std: XLogPageHeaderData::from_bytes(buf),
            xlp_sysid: buf.get_u64_le(),
            xlp_seg_size: buf.get_u32_le(),
            xlp_xlog_blcksz: buf.get_u32_le(),
        }
    }
    pub fn encode(&self) -> Bytes {
        let b: [u8; XLOG_SIZE_OF_XLOG_LONG_PHD];
        b = unsafe {
            std::mem::transmute::<XLogLongPageHeaderData, [u8; XLOG_SIZE_OF_XLOG_LONG_PHD]>(*self)
        };
        Bytes::copy_from_slice(&b[..])
    }
 }
 pub const SIZEOF_CHECKPOINT: usize = std::mem::size_of::<CheckPoint>();
 impl CheckPoint {
    pub fn encode(&self) -> Bytes {
        let b: [u8; SIZEOF_CHECKPOINT];
        b = unsafe { std::mem::transmute::<CheckPoint, [u8; SIZEOF_CHECKPOINT]>(*self) };
        Bytes::copy_from_slice(&b[..])
    }
    pub fn decode(buf: &[u8]) -> Result<CheckPoint, anyhow::Error> {
        let mut b = [0u8; SIZEOF_CHECKPOINT];
        b.copy_from_slice(&buf[0..SIZEOF_CHECKPOINT]);
        let checkpoint: CheckPoint;
        checkpoint = unsafe { std::mem::transmute::<[u8; SIZEOF_CHECKPOINT], CheckPoint>(b) };
        Ok(checkpoint)
    }
    // Update next XID based on provided new_xid and stored epoch.
    // Next XID should be greater than new_xid.
    // Also take in account 32-bit wrap-around.
    pub fn update_next_xid(&mut self, xid: u32) {
        let xid = xid.wrapping_add(XID_CHECKPOINT_INTERVAL - 1) & !(XID_CHECKPOINT_INTERVAL - 1);
        let full_xid = self.nextXid.value;
        let new_xid = std::cmp::max(xid + 1, pg_constants::FIRST_NORMAL_TRANSACTION_ID);
        let old_xid = full_xid as u32;
        if new_xid.wrapping_sub(old_xid) as i32 > 0 {
            let mut epoch = full_xid >> 32;
            if new_xid < old_xid {
                // wrap-around
                epoch += 1;
            }
            self.nextXid = FullTransactionId {
                value: (epoch << 32) | new_xid as u64,
            };
        }
    }
 }
 //
 // Generate new WAL segment with single XLOG_CHECKPOINT_SHUTDOWN record.
 // We need this segment to start compute node.
 // In order to minimize changes in Postgres core, we prefer to
 // provide WAL segment from which is can extract checkpoint record in standard way,
 // rather then implement some alternative mechanism.
 //
 pub fn generate_wal_segment(pg_control: &ControlFileData) -> Bytes {
    let mut seg_buf = BytesMut::with_capacity(pg_constants::WAL_SEGMENT_SIZE as usize);
    let hdr = XLogLongPageHeaderData {
        std: {
            XLogPageHeaderData {
                xlp_magic: XLOG_PAGE_MAGIC as u16,
                xlp_info: pg_constants::XLP_LONG_HEADER,
                xlp_tli: 1, // FIXME: always use Postgres timeline 1
                xlp_pageaddr: pg_control.checkPoint - XLOG_SIZE_OF_XLOG_LONG_PHD as u64,
                xlp_rem_len: 0,
            }
        },
        xlp_sysid: pg_control.system_identifier,
        xlp_seg_size: pg_constants::WAL_SEGMENT_SIZE as u32,
        xlp_xlog_blcksz: XLOG_BLCKSZ as u32,
    };
    let hdr_bytes = hdr.encode();
    seg_buf.extend_from_slice(&hdr_bytes);
    let rec_hdr = XLogRecord {
        xl_tot_len: (XLOG_SIZE_OF_XLOG_RECORD
            + SIZE_OF_XLOG_RECORD_DATA_HEADER_SHORT
            + SIZEOF_CHECKPOINT) as u32,
        xl_xid: 0, //0 is for InvalidTransactionId
        xl_prev: 0,
        xl_info: pg_constants::XLOG_CHECKPOINT_SHUTDOWN,
        xl_rmid: pg_constants::RM_XLOG_ID,
        xl_crc: 0,
    };
    let mut rec_shord_hdr_bytes = BytesMut::new();
    rec_shord_hdr_bytes.put_u8(pg_constants::XLR_BLOCK_ID_DATA_SHORT);
    rec_shord_hdr_bytes.put_u8(SIZEOF_CHECKPOINT as u8);
    let rec_bytes = rec_hdr.encode();
    let checkpoint_bytes = pg_control.checkPointCopy.encode();
    //calculate record checksum
    let mut crc = 0;
    crc = crc32c_append(crc, &rec_shord_hdr_bytes[..]);
    crc = crc32c_append(crc, &checkpoint_bytes[..]);
    crc = crc32c_append(crc, &rec_bytes[0..XLOG_RECORD_CRC_OFFS]);
    seg_buf.extend_from_slice(&rec_bytes[0..XLOG_RECORD_CRC_OFFS]);
    seg_buf.put_u32_le(crc);
    seg_buf.extend_from_slice(&rec_shord_hdr_bytes);
    seg_buf.extend_from_slice(&checkpoint_bytes);
    //zero out the rest of the file
    seg_buf.resize(pg_constants::WAL_SEGMENT_SIZE, 0);
    seg_buf.freeze()
 }
 #[cfg(test)]
 mod tests {
    use super::*;
    use regex::Regex;
    use std::{env, process::Command, str::FromStr};
    use zenith_utils::lsn::Lsn;
    // Run find_end_of_wal against file in test_wal dir
    // Ensure that it finds last record correctly
    #[test]
    pub fn test_find_end_of_wal() {
        // 1. Run initdb to generate some WAL
        let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
        let data_dir = top_path.join("test_output/test_find_end_of_wal");
        let initdb_path = top_path.join("tmp_install/bin/initdb");
        let lib_path = top_path.join("tmp_install/lib");
        if data_dir.exists() {
            fs::remove_dir_all(&data_dir).unwrap();
        }
        println!("Using initdb from '{}'", initdb_path.display());
        println!("Data directory '{}'", data_dir.display());
        let initdb_output = Command::new(initdb_path)
            .args(&["-D", data_dir.to_str().unwrap()])
            .arg("--no-instructions")
            .arg("--no-sync")
            .env_clear()
            .env("LD_LIBRARY_PATH", &lib_path)
            .env("DYLD_LIBRARY_PATH", &lib_path)
            .output()
            .unwrap();
        assert!(initdb_output.status.success());
        // 2. Pick WAL generated by initdb
        let wal_dir = data_dir.join("pg_wal");
        let wal_seg_size = 16 * 1024 * 1024;
        // 3. Check end_of_wal on non-partial WAL segment (we treat it as fully populated)
        let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true);
        let wal_end = Lsn(wal_end);
        println!("wal_end={}, tli={}", wal_end, tli);
        assert_eq!(wal_end, "0/1699D10".parse::<Lsn>().unwrap());
        // 4. Get the actual end of WAL by pg_waldump
        let waldump_path = top_path.join("tmp_install/bin/pg_waldump");
        let waldump_output = Command::new(waldump_path)
            .arg(wal_dir.join("000000010000000000000001"))
            .env_clear()
            .env("LD_LIBRARY_PATH", &lib_path)
            .env("DYLD_LIBRARY_PATH", &lib_path)
            .output()
            .unwrap();
        let waldump_output = std::str::from_utf8(&waldump_output.stderr).unwrap();
        println!("waldump_output = '{}'", &waldump_output);
        let re = Regex::new(r"invalid record length at (.+):").unwrap();
        let caps = re.captures(&waldump_output).unwrap();
        let waldump_wal_end = Lsn::from_str(caps.get(1).unwrap().as_str()).unwrap();
        // 5. Rename file to partial to actually find last valid lsn
        fs::rename(
            wal_dir.join("000000010000000000000001"),
            wal_dir.join("000000010000000000000001.partial"),
        )
        .unwrap();
        let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true);
        let wal_end = Lsn(wal_end);
        println!("wal_end={}, tli={}", wal_end, tli);
        assert_eq!(wal_end, waldump_wal_end);
    }
 }
@@ -1,21 +0,0 @@
 [package]
 name = "proxy"
 version = "0.1.0"
 authors = ["Stas Kelvich <stas.kelvich@gmail.com>"]
 edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [dependencies]
 anyhow = "1.0"
 bytes = { version = "1.0.1", features = ['serde'] }
 md5 = "0.7.0"
 rand = "0.8.3"
 hex = "0.4.3"
 serde = "1"
 serde_json = "1"
 tokio = { version = "1.7.1", features = ["full"] }
 tokio-postgres = "0.7.2"
 clap = "2.33.0"
 zenith_utils = { path = "../zenith_utils" }
@@ -1,92 +0,0 @@
 use anyhow::{bail, Result};
 use serde::{Deserialize, Serialize};
 use std::{
    collections::HashMap,
    net::{IpAddr, SocketAddr},
 };
 pub struct CPlaneApi {
    // address: SocketAddr,
 }
 #[derive(Serialize, Deserialize)]
 pub struct DatabaseInfo {
    pub host: IpAddr, // TODO: allow host name here too
    pub port: u16,
    pub dbname: String,
    pub user: String,
    pub password: String,
 }
 impl DatabaseInfo {
    pub fn socket_addr(&self) -> SocketAddr {
        SocketAddr::new(self.host, self.port)
    }
    pub fn conn_string(&self) -> String {
        format!(
            "dbname={} user={} password={}",
            self.dbname, self.user, self.password
        )
    }
 }
 // mock cplane api
 impl CPlaneApi {
    pub fn new(_address: &SocketAddr) -> CPlaneApi {
        CPlaneApi {
            // address: address.clone(),
        }
    }
    pub fn check_auth(&self, user: &str, md5_response: &[u8], salt: &[u8; 4]) -> Result<()> {
        // passwords for both is "mypass"
        let auth_map: HashMap<_, &str> = vec![
            ("stas@zenith", "716ee6e1c4a9364d66285452c47402b1"),
            ("stas2@zenith", "3996f75df64c16a8bfaf01301b61d582"),
        ]
        .into_iter()
        .collect();
        let stored_hash = auth_map
            .get(&user)
            .ok_or_else(|| anyhow::Error::msg("user not found"))?;
        let salted_stored_hash = format!(
            "md5{:x}",
            md5::compute([stored_hash.as_bytes(), salt].concat())
        );
        let received_hash = std::str::from_utf8(&md5_response)?;
        println!(
            "auth: {} rh={} sh={} ssh={} {:?}",
            user, received_hash, stored_hash, salted_stored_hash, salt
        );
        if received_hash == salted_stored_hash {
            Ok(())
        } else {
            bail!("Auth failed")
        }
    }
    pub fn get_database_uri(&self, _user: &str, _database: &str) -> Result<DatabaseInfo> {
        Ok(DatabaseInfo {
            host: "127.0.0.1".parse()?,
            port: 5432,
            dbname: "stas".to_string(),
            user: "stas".to_string(),
            password: "mypass".to_string(),
        })
    }
    // pub fn create_database(&self, _user: &String, _database: &String) -> Result<DatabaseInfo> {
    //     Ok(DatabaseInfo {
    //         host: "127.0.0.1".parse()?,
    //         port: 5432,
    //         dbname: "stas".to_string(),
    //         user: "stas".to_string(),
    //         password: "mypass".to_string(),
    //     })
    // }
 }
@@ -1,106 +0,0 @@
 ///
 /// Postgres protocol proxy/router.
 ///
 /// This service listens psql port and can check auth via external service
 /// (control plane API in our case) and can create new databases and accounts
 /// in somewhat transparent manner (again via communication with control plane API).
 ///
 use std::{
    collections::HashMap,
    net::{SocketAddr, TcpListener},
    sync::{mpsc, Mutex},
    thread,
 };
 use clap::{App, Arg};
 use cplane_api::DatabaseInfo;
 mod cplane_api;
 mod mgmt;
 mod proxy;
 pub struct ProxyConf {
    /// main entrypoint for users to connect to
    pub proxy_address: SocketAddr,
    /// http management endpoint. Upon user account creation control plane
    /// will notify us here, so that we can 'unfreeze' user session.
    pub mgmt_address: SocketAddr,
    /// send unauthenticated users to this URI
    pub redirect_uri: String,
    /// control plane address where we would check auth.
    pub cplane_address: SocketAddr,
 }
 pub struct ProxyState {
    pub conf: ProxyConf,
    pub waiters: Mutex<HashMap<String, mpsc::Sender<anyhow::Result<DatabaseInfo>>>>,
 }
 fn main() -> anyhow::Result<()> {
    let arg_matches = App::new("Zenith proxy/router")
        .arg(
            Arg::with_name("proxy")
                .short("p")
                .long("proxy")
                .takes_value(true)
                .help("listen for incoming client connections on ip:port")
                .default_value("127.0.0.1:4432"),
        )
        .arg(
            Arg::with_name("mgmt")
                .short("m")
                .long("mgmt")
                .takes_value(true)
                .help("listen for management callback connection on ip:port")
                .default_value("127.0.0.1:7000"),
        )
        .arg(
            Arg::with_name("uri")
                .short("u")
                .long("uri")
                .takes_value(true)
                .help("redirect unauthenticated users to given uri")
                .default_value("http://localhost:3000/psql_session/"),
        )
        .get_matches();
    let conf = ProxyConf {
        proxy_address: arg_matches.value_of("proxy").unwrap().parse()?,
        mgmt_address: arg_matches.value_of("mgmt").unwrap().parse()?,
        redirect_uri: arg_matches.value_of("uri").unwrap().parse()?,
        cplane_address: "127.0.0.1:3000".parse()?,
    };
    let state = ProxyState {
        conf,
        waiters: Mutex::new(HashMap::new()),
    };
    let state: &'static ProxyState = Box::leak(Box::new(state));
    // Check that we can bind to address before further initialization
    println!("Starting proxy on {}", state.conf.proxy_address);
    let pageserver_listener = TcpListener::bind(state.conf.proxy_address)?;
    println!("Starting mgmt on {}", state.conf.mgmt_address);
    let mgmt_listener = TcpListener::bind(state.conf.mgmt_address)?;
    let threads = vec![
        // Spawn a thread to listen for connections. It will spawn further threads
        // for each connection.
        thread::Builder::new()
            .name("Proxy thread".into())
            .spawn(move || proxy::thread_main(&state, pageserver_listener))?,
        thread::Builder::new()
            .name("Mgmt thread".into())
            .spawn(move || mgmt::thread_main(&state, mgmt_listener))?,
    ];
    for t in threads.into_iter() {
        t.join().unwrap()?;
    }
    Ok(())
 }
@@ -1,111 +0,0 @@
 use std::{
    net::{TcpListener, TcpStream},
    thread,
 };
 use anyhow::bail;
 use bytes::Bytes;
 use serde::{Deserialize, Serialize};
 use zenith_utils::{
    postgres_backend::{self, query_from_cstring, AuthType, PostgresBackend},
    pq_proto::{BeMessage, SINGLE_COL_ROWDESC},
 };
 use crate::{cplane_api::DatabaseInfo, ProxyState};
 ///
 /// Main proxy listener loop.
 ///
 /// Listens for connections, and launches a new handler thread for each.
 ///
 pub fn thread_main(state: &'static ProxyState, listener: TcpListener) -> anyhow::Result<()> {
    loop {
        let (socket, peer_addr) = listener.accept()?;
        println!("accepted connection from {}", peer_addr);
        socket.set_nodelay(true).unwrap();
        thread::spawn(move || {
            if let Err(err) = mgmt_conn_main(state, socket) {
                println!("error: {}", err);
            }
        });
    }
 }
 pub fn mgmt_conn_main(state: &'static ProxyState, socket: TcpStream) -> anyhow::Result<()> {
    let mut conn_handler = MgmtHandler { state };
    let pgbackend = PostgresBackend::new(socket, AuthType::Trust)?;
    pgbackend.run(&mut conn_handler)
 }
 struct MgmtHandler {
    state: &'static ProxyState,
 }
 /// Serialized examples:
 // {
 //     "session_id": "71d6d03e6d93d99a",
 //     "result": {
 //         "Success": {
 //             "host": "127.0.0.1",
 //             "port": 5432,
 //             "dbname": "stas",
 //             "user": "stas"
 //             "password": "mypass"
 //         }
 //     }
 // }
 // {
 //     "session_id": "71d6d03e6d93d99a",
 //     "result": {
 //         "Failure": "oops"
 //     }
 // }
 #[derive(Serialize, Deserialize)]
 pub struct PsqlSessionResponse {
    session_id: String,
    result: PsqlSessionResult,
 }
 #[derive(Serialize, Deserialize)]
 pub enum PsqlSessionResult {
    Success(DatabaseInfo),
    Failure(String),
 }
 impl postgres_backend::Handler for MgmtHandler {
    fn process_query(
        &mut self,
        pgb: &mut PostgresBackend,
        query_string: Bytes,
    ) -> anyhow::Result<()> {
        let query_string = query_from_cstring(query_string);
        println!("Got mgmt query: '{}'", std::str::from_utf8(&query_string)?);
        let resp: PsqlSessionResponse = serde_json::from_slice(&query_string)?;
        let waiters = self.state.waiters.lock().unwrap();
        let sender = waiters
            .get(&resp.session_id)
            .ok_or_else(|| anyhow::Error::msg("psql_session_id is not found"))?;
        match resp.result {
            PsqlSessionResult::Success(db_info) => {
                sender.send(Ok(db_info))?;
                pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
                    .write_message_noflush(&BeMessage::DataRow(&[Some(b"ok")]))?
                    .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
                pgb.flush()?;
                Ok(())
            }
            PsqlSessionResult::Failure(message) => {
                sender.send(Err(anyhow::Error::msg(message.clone())))?;
                bail!("psql session request failed: {}", message)
            }
        }
    }
 }
@@ -1,256 +0,0 @@
 use crate::cplane_api::CPlaneApi;
 use crate::cplane_api::DatabaseInfo;
 use crate::ProxyState;
 use anyhow::bail;
 use tokio_postgres::NoTls;
 use rand::Rng;
 use std::sync::mpsc::channel;
 use std::thread;
 use tokio::io::AsyncWriteExt;
 use zenith_utils::postgres_backend::{PostgresBackend, ProtoState};
 use zenith_utils::pq_proto::*;
 use zenith_utils::{postgres_backend, pq_proto::BeMessage};
 ///
 /// Main proxy listener loop.
 ///
 /// Listens for connections, and launches a new handler thread for each.
 ///
 pub fn thread_main(
    state: &'static ProxyState,
    listener: std::net::TcpListener,
 ) -> anyhow::Result<()> {
    loop {
        let (socket, peer_addr) = listener.accept()?;
        println!("accepted connection from {}", peer_addr);
        socket.set_nodelay(true).unwrap();
        thread::spawn(move || {
            if let Err(err) = proxy_conn_main(state, socket) {
                println!("error: {}", err);
            }
        });
    }
 }
 // XXX: clean up fields
 struct ProxyConnection {
    state: &'static ProxyState,
    cplane: CPlaneApi,
    user: String,
    database: String,
    pgb: PostgresBackend,
    md5_salt: [u8; 4],
    psql_session_id: String,
 }
 pub fn proxy_conn_main(
    state: &'static ProxyState,
    socket: std::net::TcpStream,
 ) -> anyhow::Result<()> {
    let mut conn = ProxyConnection {
        state,
        cplane: CPlaneApi::new(&state.conf.cplane_address),
        user: "".into(),
        database: "".into(),
        pgb: PostgresBackend::new(socket, postgres_backend::AuthType::MD5)?,
        md5_salt: [0u8; 4],
        psql_session_id: "".into(),
    };
    // Check StartupMessage
    // This will set conn.existing_user and we can decide on next actions
    conn.handle_startup()?;
    // both scenarious here should end up producing database connection string
    let db_info = if conn.is_existing_user() {
        conn.handle_existing_user()?
    } else {
        conn.handle_new_user()?
    };
    // ok, proxy pass user connection to database_uri
    let runtime = tokio::runtime::Builder::new_current_thread()
        .enable_all()
        .build()
        .unwrap();
    let _ = runtime.block_on(proxy_pass(conn.pgb, db_info))?;
    println!("proxy_conn_main done;");
    Ok(())
 }
 impl ProxyConnection {
    fn is_existing_user(&self) -> bool {
        self.user.ends_with("@zenith")
    }
    fn handle_startup(&mut self) -> anyhow::Result<()> {
        loop {
            let msg = self.pgb.read_message()?;
            println!("got message {:?}", msg);
            match msg {
                Some(FeMessage::StartupMessage(m)) => {
                    println!("got startup message {:?}", m);
                    match m.kind {
                        StartupRequestCode::NegotiateGss | StartupRequestCode::NegotiateSsl => {
                            println!("SSL requested");
                            self.pgb.write_message(&BeMessage::Negotiate)?;
                        }
                        StartupRequestCode::Normal => {
                            self.user = m
                                .params
                                .get("user")
                                .ok_or_else(|| {
                                    anyhow::Error::msg("user is required in startup packet")
                                })?
                                .into();
                            self.database = m
                                .params
                                .get("database")
                                .ok_or_else(|| {
                                    anyhow::Error::msg("database is required in startup packet")
                                })?
                                .into();
                            break;
                        }
                        StartupRequestCode::Cancel => break,
                    }
                }
                None => {
                    bail!("connection closed")
                }
                unexpected => {
                    bail!("unexpected message type : {:?}", unexpected)
                }
            }
        }
        Ok(())
    }
    fn handle_existing_user(&mut self) -> anyhow::Result<DatabaseInfo> {
        // ask password
        rand::thread_rng().fill(&mut self.md5_salt);
        self.pgb
            .write_message(&BeMessage::AuthenticationMD5Password(&self.md5_salt))?;
        self.pgb.state = ProtoState::Authentication; // XXX
        // check password
        println!("handle_existing_user");
        let msg = self.pgb.read_message()?;
        println!("got message {:?}", msg);
        if let Some(FeMessage::PasswordMessage(m)) = msg {
            println!("got password message '{:?}'", m);
            assert!(self.is_existing_user());
            let (_trailing_null, md5_response) = m
                .split_last()
                .ok_or_else(|| anyhow::Error::msg("unexpected password message"))?;
            if let Err(e) = self.check_auth_md5(md5_response) {
                self.pgb
                    .write_message(&BeMessage::ErrorResponse(format!("{}", e)))?;
                bail!("auth failed: {}", e);
            } else {
                self.pgb
                    .write_message_noflush(&BeMessage::AuthenticationOk)?;
                self.pgb
                    .write_message_noflush(&BeMessage::ParameterStatus)?;
                self.pgb.write_message(&BeMessage::ReadyForQuery)?;
            }
        }
        // ok, we are authorized
        self.cplane.get_database_uri(&self.user, &self.database)
    }
    fn handle_new_user(&mut self) -> anyhow::Result<DatabaseInfo> {
        let mut psql_session_id_buf = [0u8; 8];
        rand::thread_rng().fill(&mut psql_session_id_buf);
        self.psql_session_id = hex::encode(psql_session_id_buf);
        let hello_message = format!("☀️  Welcome to Zenith!
 To proceed with database creation open following link:
    {}{}
 It needed to be done once and we will send you '.pgpass' file which will allow you to access or create
 databases without opening the browser.
 ", self.state.conf.redirect_uri,self.psql_session_id);
        self.pgb
            .write_message_noflush(&BeMessage::AuthenticationOk)?;
        self.pgb
            .write_message_noflush(&BeMessage::ParameterStatus)?;
        self.pgb
            .write_message(&BeMessage::NoticeResponse(hello_message))?;
        // await for database creation
        let (tx, rx) = channel::<anyhow::Result<DatabaseInfo>>();
        let _ = self
            .state
            .waiters
            .lock()
            .unwrap()
            .insert(self.psql_session_id.clone(), tx);
        // Wait for web console response
        // XXX: respond with error to client
        let dbinfo = rx.recv()??;
        self.pgb.write_message_noflush(&BeMessage::NoticeResponse(
            "Connecting to database.".to_string(),
        ))?;
        self.pgb.write_message(&BeMessage::ReadyForQuery)?;
        Ok(dbinfo)
    }
    fn check_auth_md5(&self, md5_response: &[u8]) -> anyhow::Result<()> {
        assert!(self.is_existing_user());
        self.cplane
            .check_auth(self.user.as_str(), md5_response, &self.md5_salt)
    }
 }
 async fn proxy_pass(pgb: PostgresBackend, db_info: DatabaseInfo) -> anyhow::Result<()> {
    let mut socket = tokio::net::TcpStream::connect(db_info.socket_addr()).await?;
    let config = db_info.conn_string().parse::<tokio_postgres::Config>()?;
    let _ = config.connect_raw(&mut socket, NoTls).await?;
    println!("Connected to pg, proxying");
    let incoming_std = pgb.into_stream();
    incoming_std.set_nonblocking(true)?;
    let mut incoming_conn = tokio::net::TcpStream::from_std(incoming_std)?;
    let (mut ri, mut wi) = incoming_conn.split();
    let (mut ro, mut wo) = socket.split();
    let client_to_server = async {
        tokio::io::copy(&mut ri, &mut wo).await?;
        wo.shutdown().await
    };
    let server_to_client = async {
        tokio::io::copy(&mut ro, &mut wi).await?;
        wi.shutdown().await
    };
    tokio::try_join!(client_to_server, server_to_client)?;
    Ok(())
 }
@@ -1,11 +0,0 @@
 #!/bin/bash
 # If you save this in your path under the name "cargo-zclippy" (or whatever
 # name you like), then you can run it as "cargo zclippy" from the shell prompt.
 #
 # If your text editor has rust-analyzer integration, you can also use this new
 # command as a replacement for "cargo check" or "cargo clippy" and see clippy
 # warnings and errors right in the editor.
 # In vscode, this setting is Rust-analyzer>Check On Save:Command
 cargo clippy "${@:2}" -- -A clippy::new_without_default -A clippy::manual_range_contains -A clippy::comparison_chain
@@ -1,19 +0,0 @@
 [[source]]
 url = "https://pypi.python.org/simple"
 verify_ssl = true
 name = "pypi"
 [packages]
 pytest = ">=6.0.0"
 psycopg2 = "*"
 typing-extensions = "*"
 pyjwt = {extras = ["crypto"], version = "*"}
 [dev-packages]
 yapf = "*"
 flake8 = "*"
 mypy = "*"
 [requires]
 # we need at least 3.6, but pipenv doesn't allow to say this directly
 python_version = "3"
@@ -1,287 +0,0 @@
 {
    "_meta": {
        "hash": {
            "sha256": "f60a966726bcc19670402ad3fa57396b5dacf0a027544418ceb7cc0d42d94a52"
        },
        "pipfile-spec": 6,
        "requires": {
            "python_version": "3"
        },
        "sources": [
            {
                "name": "pypi",
                "url": "https://pypi.python.org/simple",
                "verify_ssl": true
            }
        ]
    },
    "default": {
        "attrs": {
            "hashes": [
                "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1",
                "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"
            ],
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
            "version": "==21.2.0"
        },
        "cffi": {
            "hashes": [
                "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d",
                "sha256:0c0591bee64e438883b0c92a7bed78f6290d40bf02e54c5bf0978eaf36061771",
                "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872",
                "sha256:22b9c3c320171c108e903d61a3723b51e37aaa8c81255b5e7ce102775bd01e2c",
                "sha256:26bb2549b72708c833f5abe62b756176022a7b9a7f689b571e74c8478ead51dc",
                "sha256:33791e8a2dc2953f28b8d8d300dde42dd929ac28f974c4b4c6272cb2955cb762",
                "sha256:3c8d896becff2fa653dc4438b54a5a25a971d1f4110b32bd3068db3722c80202",
                "sha256:4373612d59c404baeb7cbd788a18b2b2a8331abcc84c3ba40051fcd18b17a4d5",
                "sha256:487d63e1454627c8e47dd230025780e91869cfba4c753a74fda196a1f6ad6548",
                "sha256:48916e459c54c4a70e52745639f1db524542140433599e13911b2f329834276a",
                "sha256:4922cd707b25e623b902c86188aca466d3620892db76c0bdd7b99a3d5e61d35f",
                "sha256:55af55e32ae468e9946f741a5d51f9896da6b9bf0bbdd326843fec05c730eb20",
                "sha256:57e555a9feb4a8460415f1aac331a2dc833b1115284f7ded7278b54afc5bd218",
                "sha256:5d4b68e216fc65e9fe4f524c177b54964af043dde734807586cf5435af84045c",
                "sha256:64fda793737bc4037521d4899be780534b9aea552eb673b9833b01f945904c2e",
                "sha256:6d6169cb3c6c2ad50db5b868db6491a790300ade1ed5d1da29289d73bbe40b56",
                "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224",
                "sha256:80b06212075346b5546b0417b9f2bf467fea3bfe7352f781ffc05a8ab24ba14a",
                "sha256:818014c754cd3dba7229c0f5884396264d51ffb87ec86e927ef0be140bfdb0d2",
                "sha256:8eb687582ed7cd8c4bdbff3df6c0da443eb89c3c72e6e5dcdd9c81729712791a",
                "sha256:99f27fefe34c37ba9875f224a8f36e31d744d8083e00f520f133cab79ad5e819",
                "sha256:9f3e33c28cd39d1b655ed1ba7247133b6f7fc16fa16887b120c0c670e35ce346",
                "sha256:a8661b2ce9694ca01c529bfa204dbb144b275a31685a075ce123f12331be790b",
                "sha256:a9da7010cec5a12193d1af9872a00888f396aba3dc79186604a09ea3ee7c029e",
                "sha256:aedb15f0a5a5949ecb129a82b72b19df97bbbca024081ed2ef88bd5c0a610534",
                "sha256:b315d709717a99f4b27b59b021e6207c64620790ca3e0bde636a6c7f14618abb",
                "sha256:ba6f2b3f452e150945d58f4badd92310449876c4c954836cfb1803bdd7b422f0",
                "sha256:c33d18eb6e6bc36f09d793c0dc58b0211fccc6ae5149b808da4a62660678b156",
                "sha256:c9a875ce9d7fe32887784274dd533c57909b7b1dcadcc128a2ac21331a9765dd",
                "sha256:c9e005e9bd57bc987764c32a1bee4364c44fdc11a3cc20a40b93b444984f2b87",
                "sha256:d2ad4d668a5c0645d281dcd17aff2be3212bc109b33814bbb15c4939f44181cc",
                "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195",
                "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33",
                "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f",
                "sha256:e9dc245e3ac69c92ee4c167fbdd7428ec1956d4e754223124991ef29eb57a09d",
                "sha256:eb687a11f0a7a1839719edd80f41e459cc5366857ecbed383ff376c4e3cc6afd",
                "sha256:eb9e2a346c5238a30a746893f23a9535e700f8192a68c07c0258e7ece6ff3728",
                "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7",
                "sha256:f0010c6f9d1a4011e429109fda55a225921e3206e7f62a0c22a35344bfd13cca",
                "sha256:f0c5d1acbfca6ebdd6b1e3eded8d261affb6ddcf2186205518f1428b8569bb99",
                "sha256:f10afb1004f102c7868ebfe91c28f4a712227fe4cb24974350ace1f90e1febbf",
                "sha256:f174135f5609428cc6e1b9090f9268f5c8935fddb1b25ccb8255a2d50de6789e",
                "sha256:f3ebe6e73c319340830a9b2825d32eb6d8475c1dac020b4f0aa774ee3b898d1c",
                "sha256:f627688813d0a4140153ff532537fbe4afea5a3dffce1f9deb7f91f848a832b5",
                "sha256:fd4305f86f53dfd8cd3522269ed7fc34856a8ee3709a5e28b2836b2db9d4cd69"
            ],
            "version": "==1.14.6"
        },
        "cryptography": {
            "hashes": [
                "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d",
                "sha256:1e056c28420c072c5e3cb36e2b23ee55e260cb04eee08f702e0edfec3fb51959",
                "sha256:240f5c21aef0b73f40bb9f78d2caff73186700bf1bc6b94285699aff98cc16c6",
                "sha256:26965837447f9c82f1855e0bc8bc4fb910240b6e0d16a664bb722df3b5b06873",
                "sha256:37340614f8a5d2fb9aeea67fd159bfe4f5f4ed535b1090ce8ec428b2f15a11f2",
                "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713",
                "sha256:3d8427734c781ea5f1b41d6589c293089704d4759e34597dce91014ac125aad1",
                "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177",
                "sha256:8e56e16617872b0957d1c9742a3f94b43533447fd78321514abbe7db216aa250",
                "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca",
                "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d",
                "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9"
            ],
            "version": "==3.4.7"
        },
        "iniconfig": {
            "hashes": [
                "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
                "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
            ],
            "version": "==1.1.1"
        },
        "packaging": {
            "hashes": [
                "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7",
                "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"
            ],
            "markers": "python_version >= '3.6'",
            "version": "==21.0"
        },
        "pluggy": {
            "hashes": [
                "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
                "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
            ],
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
            "version": "==0.13.1"
        },
        "psycopg2": {
            "hashes": [
                "sha256:079d97fc22de90da1d370c90583659a9f9a6ee4007355f5825e5f1c70dffc1fa",
                "sha256:2087013c159a73e09713294a44d0c8008204d06326006b7f652bef5ace66eebb",
                "sha256:2c992196719fadda59f72d44603ee1a2fdcc67de097eea38d41c7ad9ad246e62",
                "sha256:7640e1e4d72444ef012e275e7b53204d7fab341fb22bc76057ede22fe6860b25",
                "sha256:7f91312f065df517187134cce8e395ab37f5b601a42446bdc0f0d51773621854",
                "sha256:830c8e8dddab6b6716a4bf73a09910c7954a92f40cf1d1e702fb93c8a919cc56",
                "sha256:89409d369f4882c47f7ea20c42c5046879ce22c1e4ea20ef3b00a4dfc0a7f188",
                "sha256:bf35a25f1aaa8a3781195595577fcbb59934856ee46b4f252f56ad12b8043bcf",
                "sha256:de5303a6f1d0a7a34b9d40e4d3bef684ccc44a49bbe3eb85e3c0bffb4a131b7c"
            ],
            "index": "pypi",
            "version": "==2.9.1"
        },
        "py": {
            "hashes": [
                "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3",
                "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"
            ],
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
            "version": "==1.10.0"
        },
        "pycparser": {
            "hashes": [
                "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",
                "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"
            ],
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
            "version": "==2.20"
        },
        "pyjwt": {
            "extras": [
                "crypto"
            ],
            "hashes": [
                "sha256:934d73fbba91b0483d3857d1aff50e96b2a892384ee2c17417ed3203f173fca1",
                "sha256:fba44e7898bbca160a2b2b501f492824fc8382485d3a6f11ba5d0c1937ce6130"
            ],
            "index": "pypi",
            "version": "==2.1.0"
        },
        "pyparsing": {
            "hashes": [
                "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
                "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
            ],
            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
            "version": "==2.4.7"
        },
        "pytest": {
            "hashes": [
                "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b",
                "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890"
            ],
            "index": "pypi",
            "version": "==6.2.4"
        },
        "toml": {
            "hashes": [
                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
            ],
            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
            "version": "==0.10.2"
        },
        "typing-extensions": {
            "hashes": [
                "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497",
                "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342",
                "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"
            ],
            "index": "pypi",
            "version": "==3.10.0.0"
        }
    },
    "develop": {
        "flake8": {
            "hashes": [
                "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b",
                "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"
            ],
            "index": "pypi",
            "version": "==3.9.2"
        },
        "mccabe": {
            "hashes": [
                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
                "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
            ],
            "version": "==0.6.1"
        },
        "mypy": {
            "hashes": [
                "sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9",
                "sha256:0aadfb2d3935988ec3815952e44058a3100499f5be5b28c34ac9d79f002a4a9a",
                "sha256:119bed3832d961f3a880787bf621634ba042cb8dc850a7429f643508eeac97b9",
                "sha256:1a85e280d4d217150ce8cb1a6dddffd14e753a4e0c3cf90baabb32cefa41b59e",
                "sha256:3c4b8ca36877fc75339253721f69603a9c7fdb5d4d5a95a1a1b899d8b86a4de2",
                "sha256:3e382b29f8e0ccf19a2df2b29a167591245df90c0b5a2542249873b5c1d78212",
                "sha256:42c266ced41b65ed40a282c575705325fa7991af370036d3f134518336636f5b",
                "sha256:53fd2eb27a8ee2892614370896956af2ff61254c275aaee4c230ae771cadd885",
                "sha256:704098302473cb31a218f1775a873b376b30b4c18229421e9e9dc8916fd16150",
                "sha256:7df1ead20c81371ccd6091fa3e2878559b5c4d4caadaf1a484cf88d93ca06703",
                "sha256:866c41f28cee548475f146aa4d39a51cf3b6a84246969f3759cb3e9c742fc072",
                "sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457",
                "sha256:adaeee09bfde366d2c13fe6093a7df5df83c9a2ba98638c7d76b010694db760e",
                "sha256:b6fb13123aeef4a3abbcfd7e71773ff3ff1526a7d3dc538f3929a49b42be03f0",
                "sha256:b94e4b785e304a04ea0828759172a15add27088520dc7e49ceade7834275bedb",
                "sha256:c0df2d30ed496a08de5daed2a9ea807d07c21ae0ab23acf541ab88c24b26ab97",
                "sha256:c6c2602dffb74867498f86e6129fd52a2770c48b7cd3ece77ada4fa38f94eba8",
                "sha256:ceb6e0a6e27fb364fb3853389607cf7eb3a126ad335790fa1e14ed02fba50811",
                "sha256:d9dd839eb0dc1bbe866a288ba3c1afc33a202015d2ad83b31e875b5905a079b6",
                "sha256:e4dab234478e3bd3ce83bac4193b2ecd9cf94e720ddd95ce69840273bf44f6de",
                "sha256:ec4e0cd079db280b6bdabdc807047ff3e199f334050db5cbb91ba3e959a67504",
                "sha256:ecd2c3fe726758037234c93df7e98deb257fd15c24c9180dacf1ef829da5f921",
                "sha256:ef565033fa5a958e62796867b1df10c40263ea9ded87164d67572834e57a174d"
            ],
            "index": "pypi",
            "version": "==0.910"
        },
        "mypy-extensions": {
            "hashes": [
                "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
                "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
            ],
            "version": "==0.4.3"
        },
        "pycodestyle": {
            "hashes": [
                "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068",
                "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"
            ],
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
            "version": "==2.7.0"
        },
        "pyflakes": {
            "hashes": [
                "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3",
                "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"
            ],
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
            "version": "==2.3.1"
        },
        "toml": {
            "hashes": [
                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
            ],
            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
            "version": "==0.10.2"
        },
        "typing-extensions": {
            "hashes": [
                "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497",
                "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342",
                "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"
            ],
            "index": "pypi",
            "version": "==3.10.0.0"
        },
        "yapf": {
            "hashes": [
                "sha256:408fb9a2b254c302f49db83c59f9aa0b4b0fd0ec25be3a5c51181327922ff63d",
                "sha256:e3a234ba8455fe201eaa649cdac872d590089a18b661e39bbac7020978dd9c2e"
            ],
            "index": "pypi",
            "version": "==0.31.0"
        }
    }
 }
@@ -4,10 +4,13 @@ This directory contains integration tests.
 Prerequisites:
 - Python 3.6 or later
- Dependencies: install them via `pipenv install`. Note that Debian/Ubuntu
+- Python packages: pytest, psycopg2
-  packages are stale, as it commonly happens, so manual installation is not
+    - pytest 6.0 is required.
-  recommended.
+    - __NOTE: `apt install` on Debian/Ubuntu won't work.__
-  Run `pipenv shell` to activate the venv.
+      They ship a much older version of pytest (and sometimes rename it to
      `pytest-3`.)
    - Install using something like this:
        - `pip3 install pytest psycopg2` (Debian or Ubuntu)
 - Zenith and Postgres binaries
    - See the root README.md for build directions
    - Tests can be run from the git tree; or see the environment variables
@@ -69,8 +72,7 @@ The tests make heavy use of pytest fixtures. You can read about how they work he
 Essentially, this means that each time you see a fixture named as an input parameter, the function with that name will be run and passed as a parameter to the function.
 So this code:
-
+```
 ```python
 def test_something(zenith_cli, pg_bin):
    pass
 ```
@@ -78,11 +80,9 @@ def test_something(zenith_cli, pg_bin):
 ... will run the fixtures called `zenith_cli` and `pg_bin` and deliver those results to the test function.
 Fixtures can't be imported using the normal python syntax. Instead, use this:
-
+```
 ```python
 pytest_plugins = ("fixtures.something")
 ```
 That will make all the fixtures in the `fixtures/something.py` file available.
 Anything that's likely to be used in multiple tests should be built into a fixture.
@@ -90,15 +90,3 @@ Anything that's likely to be used in multiple tests should be built into a fixtu
 Note that fixtures can clean up after themselves if they use the `yield` syntax.
 Cleanup will happen even if the test fails (raises an unhandled exception).
 Python destructors, e.g. `__del__()` aren't recommended for cleanup.
 ### Code quality
 Before submitting a patch, please consider:
 * Writing a couple of docstrings to clarify the reasoning behind a new test.
 * Running `flake8` (or a linter of your choice, e.g. `pycodestyle`) and fixing possible defects, if any.
 * Formatting the code with `yapf -r -i .` (TODO: implement an opt-in pre-commit hook for that).
 * (Optional) Typechecking the code with `mypy .`. Currently this mostly affects `fixtures/zenith_fixtures.py`.
 The tools can be installed with `pipenv install --dev`.
@@ -1,117 +0,0 @@
 from contextlib import closing
 from pathlib import Path
 from uuid import uuid4
 from dataclasses import dataclass
 import jwt
 import psycopg2
 from fixtures.zenith_fixtures import Postgres, ZenithCli, ZenithPageserver
 import pytest
@pytest.fixture
 def pageserver_auth_enabled(zenith_cli: ZenithCli):
    with ZenithPageserver(zenith_cli).init(enable_auth=True).start() as ps:
        # For convenience in tests, create a branch from the freshly-initialized cluster.
        zenith_cli.run(["branch", "empty", "main"])
        yield ps
@dataclass
 class AuthKeys:
    pub: bytes
    priv: bytes
    def generate_management_token(self):
        token = jwt.encode({"scope": "pageserverapi"}, self.priv, algorithm="RS256")
        # jwt.encode can return 'bytes' or 'str', depending on Python version or type
        # hinting or something (not sure what). If it returned 'bytes', convert it to 'str'
        # explicitly.
        if isinstance(token, bytes):
            token = token.decode()
        return token
    def generate_tenant_token(self, tenant_id):
        token = jwt.encode({"scope": "tenant", "tenant_id": tenant_id}, self.priv, algorithm="RS256")
        if isinstance(token, bytes):
            token = token.decode()
        return token
@pytest.fixture
 def auth_keys(repo_dir: str) -> AuthKeys:
    # TODO probably this should be specified in cli config and used in tests for single source of truth
    pub = (Path(repo_dir) / 'auth_public_key.pem').read_bytes()
    priv = (Path(repo_dir) / 'auth_private_key.pem').read_bytes()
    return AuthKeys(pub=pub, priv=priv)
 def test_pageserver_auth(pageserver_auth_enabled: ZenithPageserver, auth_keys: AuthKeys):
    ps = pageserver_auth_enabled
    tenant_token = auth_keys.generate_tenant_token(ps.initial_tenant)
    invalid_tenant_token = auth_keys.generate_tenant_token(uuid4().hex)
    management_token = auth_keys.generate_management_token()
    # this does not invoke auth check and only decodes jwt and checks it for validity
    # check both tokens
    ps.safe_psql("status", password=tenant_token)
    ps.safe_psql("status", password=management_token)
    # tenant can create branches
    ps.safe_psql(f"branch_create {ps.initial_tenant} new1 main", password=tenant_token)
    # console can create branches for tenant
    ps.safe_psql(f"branch_create {ps.initial_tenant} new2 main", password=management_token)
    # fail to create branch using token with different tenantid
    with pytest.raises(psycopg2.DatabaseError, match='Tenant id mismatch. Permission denied'):
        ps.safe_psql(f"branch_create {ps.initial_tenant} new2 main", password=invalid_tenant_token)
    # create tenant using management token
    ps.safe_psql(f"tenant_create {uuid4().hex}", password=management_token)
    # fail to create tenant using tenant token
    with pytest.raises(psycopg2.DatabaseError, match='Attempt to access management api with tenant scope. Permission denied'):
        ps.safe_psql(f"tenant_create {uuid4().hex}", password=tenant_token)
@pytest.mark.parametrize('with_wal_acceptors', [False, True])
 def test_compute_auth_to_pageserver(
    zenith_cli: ZenithCli,
    wa_factory,
    pageserver_auth_enabled: ZenithPageserver,
    repo_dir: str,
    with_wal_acceptors: bool,
    auth_keys: AuthKeys,
 ):
    ps = pageserver_auth_enabled
    # since we are in progress of refactoring protocols between compute safekeeper and page server
    # use hardcoded management token in safekeeper
    management_token = auth_keys.generate_management_token()
    branch = f"test_compute_auth_to_pageserver{with_wal_acceptors}"
    zenith_cli.run(["branch", branch, "empty"])
    if with_wal_acceptors:
        wa_factory.start_n_new(3, management_token)
    with Postgres(
        zenith_cli=zenith_cli,
        repo_dir=repo_dir,
        tenant_id=ps.initial_tenant,
        port=55432, # FIXME port distribution is hardcoded in tests and in cli
    ).create_start(
        branch,
        wal_acceptors=wa_factory.get_connstrs() if with_wal_acceptors else None,
    ) as pg:
        with closing(pg.connect()) as conn:
            with conn.cursor() as cur:
                # we rely upon autocommit after each statement
                # as waiting for acceptors happens there
                cur.execute('CREATE TABLE t(key int primary key, value text)')
                cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
                cur.execute('SELECT sum(key) FROM t')
                assert cur.fetchone() == (5000050000, )
@@ -1,76 +1,67 @@
-from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+import pytest
-
+import getpass
 import psycopg2
 pytest_plugins = ("fixtures.zenith_fixtures")
 #
 # Create a couple of branches off the main branch, at a historical point in time.
 #
-def test_branch_behind(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+def test_branch_behind(zenith_cli, pageserver, postgres, pg_bin):
    # Branch at the point where only 100 rows were inserted
-    zenith_cli.run(["branch", "test_branch_behind", "empty"])
+    zenith_cli.run(["branch", "test_branch_behind", "empty"]);
    pgmain = postgres.create_start('test_branch_behind')
    print("postgres is running on 'test_branch_behind' branch")
-    main_pg_conn = pgmain.connect()
+    main_pg_conn = psycopg2.connect(pgmain.connstr());
    main_pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
    main_cur = main_pg_conn.cursor()
    # Create table, and insert the first 100 rows
-    main_cur.execute('CREATE TABLE foo (t text)')
+    main_cur.execute('CREATE TABLE foo (t text)');
-    main_cur.execute('''
+    main_cur.execute("INSERT INTO foo SELECT 'long string to consume some space' || g FROM generate_series(1, 100) g");
-        INSERT INTO foo
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
            SELECT 'long string to consume some space' || g
            FROM generate_series(1, 100) g
    ''')
    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
    lsn_a = main_cur.fetchone()[0]
    print('LSN after 100 rows: ' + lsn_a)
    # Insert some more rows. (This generates enough WAL to fill a few segments.)
-    main_cur.execute('''
+    main_cur.execute("INSERT INTO foo SELECT 'long string to consume some space' || g FROM generate_series(1, 100000) g");
-        INSERT INTO foo
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
            SELECT 'long string to consume some space' || g
            FROM generate_series(1, 100000) g
    ''')
    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
    lsn_b = main_cur.fetchone()[0]
    print('LSN after 100100 rows: ' + lsn_b)
    # Branch at the point where only 100 rows were inserted
-    zenith_cli.run(["branch", "test_branch_behind_hundred", "test_branch_behind@" + lsn_a])
+    zenith_cli.run(["branch", "test_branch_behind_hundred", "test_branch_behind@"+lsn_a]);
    # Insert many more rows. This generates enough WAL to fill a few segments.
-    main_cur.execute('''
+    main_cur.execute("INSERT INTO foo SELECT 'long string to consume some space' || g FROM generate_series(1, 100000) g");
-        INSERT INTO foo
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
            SELECT 'long string to consume some space' || g
            FROM generate_series(1, 100000) g
    ''')
    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
-    main_cur.execute('SELECT pg_current_wal_insert_lsn()')
+    main_cur.execute('SELECT pg_current_wal_insert_lsn()');
    lsn_c = main_cur.fetchone()[0]
    print('LSN after 200100 rows: ' + lsn_c)
    # Branch at the point where only 200 rows were inserted
-    zenith_cli.run(["branch", "test_branch_behind_more", "test_branch_behind@" + lsn_b])
+    zenith_cli.run(["branch", "test_branch_behind_more", "test_branch_behind@"+lsn_b]);
    pg_hundred = postgres.create_start("test_branch_behind_hundred")
    pg_more = postgres.create_start("test_branch_behind_more")
    # On the 'hundred' branch, we should see only 100 rows
-    hundred_pg_conn = pg_hundred.connect()
+    hundred_pg_conn = psycopg2.connect(pg_hundred.connstr())
    hundred_pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
    hundred_cur = hundred_pg_conn.cursor()
-    hundred_cur.execute('SELECT count(*) FROM foo')
+    hundred_cur.execute('SELECT count(*) FROM foo');
-    assert hundred_cur.fetchone() == (100, )
+    assert(hundred_cur.fetchone()[0] == 100);
    # On the 'more' branch, we should see 100200 rows
-    more_pg_conn = pg_more.connect()
+    more_pg_conn = psycopg2.connect(pg_more.connstr())
    more_pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
    more_cur = more_pg_conn.cursor()
-    more_cur.execute('SELECT count(*) FROM foo')
+    more_cur.execute('SELECT count(*) FROM foo');
-    assert more_cur.fetchone() == (100100, )
+    assert(more_cur.fetchone()[0] == 100100);
    # All the rows are visible on the main branch
-    main_cur.execute('SELECT count(*) FROM foo')
+    main_cur.execute('SELECT count(*) FROM foo');
-    assert main_cur.fetchone() == (200100, )
+    assert(main_cur.fetchone()[0] == 200100);
@@ -1,72 +0,0 @@
 import time
 import os
 from contextlib import closing
 from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
 pytest_plugins = ("fixtures.zenith_fixtures")
 #
 # Test compute node start after clog truncation
 #
 def test_clog_truncate(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
    # Create a branch for us
    zenith_cli.run(["branch", "test_clog_truncate", "empty"])
    # set agressive autovacuum to make sure that truncation will happen
    config = [
        'autovacuum_max_workers=10', 'autovacuum_vacuum_threshold=0',
        'autovacuum_vacuum_insert_threshold=0', 'autovacuum_vacuum_cost_delay=0',
        'autovacuum_vacuum_cost_limit=10000', 'autovacuum_naptime =1s',
        'autovacuum_freeze_max_age=100000'
    ]
    pg = postgres.create_start('test_clog_truncate', config_lines=config)
    print('postgres is running on test_clog_truncate branch')
    # Install extension containing function needed for test
    pg.safe_psql('CREATE EXTENSION zenith_test_utils')
    # Consume many xids to advance clog
    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
            cur.execute('select test_consume_xids(1000*1000*10);')
            print('xids consumed')
            # call a checkpoint to trigger TruncateSubtrans
            cur.execute('CHECKPOINT;')
            # ensure WAL flush
            cur.execute('select txid_current()')
            print(cur.fetchone())
    # wait for autovacuum to truncate the pg_xact
    # XXX Is it worth to add a timeout here?
    pg_xact_0000_path = os.path.join(pg.pg_xact_dir_path(), '0000')
    print("pg_xact_0000_path = " + pg_xact_0000_path)
    while os.path.isfile(pg_xact_0000_path):
        print("file exists. wait for truncation. " "pg_xact_0000_path = " + pg_xact_0000_path)
        time.sleep(5)
    # checkpoint to advance latest lsn
    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
            cur.execute('CHECKPOINT;')
            cur.execute('select pg_current_wal_insert_lsn()')
            lsn_after_truncation = cur.fetchone()[0]
    # create new branch after clog truncation and start a compute node on it
    print('create branch at lsn_after_truncation ' + lsn_after_truncation)
    zenith_cli.run(
        ["branch", "test_clog_truncate_new", "test_clog_truncate@" + lsn_after_truncation])
    pg2 = postgres.create_start('test_clog_truncate_new')
    print('postgres is running on test_clog_truncate_new branch')
    # check that new node doesn't contain truncated segment
    pg_xact_0000_path_new = os.path.join(pg2.pg_xact_dir_path(), '0000')
    print("pg_xact_0000_path_new = " + pg_xact_0000_path_new)
    assert os.path.isfile(pg_xact_0000_path_new) is False
@@ -1,6 +1,7 @@
-from contextlib import closing
+import pytest
-
+import os
-from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+import getpass
 import psycopg2
 pytest_plugins = ("fixtures.zenith_fixtures")
@@ -8,24 +9,22 @@ pytest_plugins = ("fixtures.zenith_fixtures")
 #
 # Test starting Postgres with custom options
 #
-def test_config(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+def test_config(zenith_cli, pageserver, postgres, pg_bin):
    # Create a branch for us
-    zenith_cli.run(["branch", "test_config", "empty"])
+    zenith_cli.run(["branch", "test_config", "empty"]);
    # change config
-    pg = postgres.create_start('test_config', config_lines=['log_min_messages=debug1'])
+    pg = postgres.create_start('test_config', ['log_min_messages=debug1'])
    print('postgres is running on test_config branch')
-    with closing(pg.connect()) as conn:
+    pg_conn = psycopg2.connect(pg.connstr())
-        with conn.cursor() as cur:
+    pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
-            cur.execute('''
+    cur = pg_conn.cursor()
                SELECT setting
                FROM pg_settings
                WHERE
                    source != 'default'
                    AND source != 'override'
                    AND name = 'log_min_messages'
            ''')
-            # check that config change was applied
+    #check that config change was applied
-            assert cur.fetchone() == ('debug1', )
+    cur.execute('SELECT name, setting from pg_settings WHERE source!=%s and source!=%s', ("default","override",))
    for record in cur:
        if record[0] == 'log_min_messages':
            assert(record[1] == 'debug1')
    pg_conn.close()
@@ -1,38 +1,37 @@
-from contextlib import closing
+import pytest
-from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory, ZenithCli
+import getpass
 import psycopg2
 pytest_plugins = ("fixtures.zenith_fixtures")
 #
 # Test CREATE DATABASE when there have been relmapper changes
 #
-def test_createdb(
+def test_createdb(zenith_cli, pageserver, postgres, pg_bin):
-    zenith_cli: ZenithCli,
+    zenith_cli.run(["branch", "test_createdb", "empty"]);
    pageserver: ZenithPageserver,
    postgres: PostgresFactory,
    pg_bin,
 ):
    zenith_cli.run(["branch", "test_createdb", "empty"])
    pg = postgres.create_start('test_createdb')
    print("postgres is running on 'test_createdb' branch")
-    with closing(pg.connect()) as conn:
+    conn = psycopg2.connect(pg.connstr());
-        with conn.cursor() as cur:
+    conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
-            # Cause a 'relmapper' change in the original branch
+    cur = conn.cursor()
            cur.execute('VACUUM FULL pg_class')
-            cur.execute('CREATE DATABASE foodb')
+    # Cause a 'relmapper' change in the original branch
    cur.execute('VACUUM FULL pg_class');
-            cur.execute('SELECT pg_current_wal_insert_lsn()')
+    cur.execute('CREATE DATABASE foodb');
-            lsn = cur.fetchone()[0]
+
    cur.execute('SELECT pg_current_wal_insert_lsn()');
    lsn = cur.fetchone()[0]
    conn.close();
    # Create a branch
-    zenith_cli.run(["branch", "test_createdb2", "test_createdb@" + lsn])
+    zenith_cli.run(["branch", "test_createdb2", "test_createdb@"+lsn]);
    pg2 = postgres.create_start('test_createdb2')
    # Test that you can connect to the new database on both branches
-    for db in (pg, pg2):
+    conn = psycopg2.connect(pg.connstr('foodb'));
-        db.connect(dbname='foodb').close()
+    conn2 = psycopg2.connect(pg2.connstr('foodb'));
@@ -1,33 +0,0 @@
 from contextlib import closing
 from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
 pytest_plugins = ("fixtures.zenith_fixtures")
 #
 # Test CREATE USER to check shared catalog restore
 #
 def test_createuser(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
    zenith_cli.run(["branch", "test_createuser", "empty"])
    pg = postgres.create_start('test_createuser')
    print("postgres is running on 'test_createuser' branch")
    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
            # Cause a 'relmapper' change in the original branch
            cur.execute('CREATE USER testuser with password %s', ('testpwd', ))
            cur.execute('CHECKPOINT')
            cur.execute('SELECT pg_current_wal_insert_lsn()')
            lsn = cur.fetchone()[0]
    # Create a branch
    zenith_cli.run(["branch", "test_createuser2", "test_createuser@" + lsn])
    pg2 = postgres.create_start('test_createuser2')
    # Test that you can connect to new branch as a new user
    assert pg2.safe_psql('select current_user', username='testuser') == [('testuser', )]
@@ -1,105 +0,0 @@
 import pytest
 from contextlib import closing
 from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
 import psycopg2.extras
 pytest_plugins = ("fixtures.zenith_fixtures")
 #
 # Test Garbage Collection of old page versions.
 #
 # This test is pretty tightly coupled with the current implementation of page version storage
 # and garbage collection in object_repository.rs.
 #
@pytest.mark.skip(reason=""""
    Current GC test is flaky and overly strict. Since we are migrating to the layered repo format
    with different GC implementation let's just silence this test for now. This test only
    works with the RocksDB implementation.
 """)
 def test_gc(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
    zenith_cli.run(["branch", "test_gc", "empty"])
    pg = postgres.create_start('test_gc')
    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
            with closing(pageserver.connect()) as psconn:
                with psconn.cursor(cursor_factory = psycopg2.extras.DictCursor) as pscur:
                    # Get the timeline ID of our branch. We need it for the 'do_gc' command
                    cur.execute("SHOW zenith.zenith_timeline")
                    timeline = cur.fetchone()[0]
                    # Create a test table
                    cur.execute("CREATE TABLE foo(x integer)")
                    # Run GC, to clear out any old page versions left behind in the catalogs by
                    # the CREATE TABLE command. We want to have a clean slate with no garbage
                    # before running the actual tests below, otherwise the counts won't match
                    # what we expect.
                    print("Running GC before test")
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
                    # remember the number of relations
                    n_relations = row['n_relations']
                    assert n_relations > 0
                    # Insert a row. The first insert will also create a metadata entry for the
                    # relation, with size == 1 block. Hence, bump up the expected relation count.
                    n_relations += 1
                    print("Inserting one row and running GC")
                    cur.execute("INSERT INTO foo VALUES (1)")
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
                    assert row['n_relations'] == n_relations
                    assert row['dropped'] == 0
                    assert row['truncated'] == 31
                    assert row['deleted'] == 4
                    # Insert two more rows and run GC.
                    print("Inserting two more rows and running GC")
                    cur.execute("INSERT INTO foo VALUES (2)")
                    cur.execute("INSERT INTO foo VALUES (3)")
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
                    assert row['n_relations'] == n_relations
                    assert row['dropped'] == 0
                    assert row['truncated'] == 31
                    assert row['deleted'] == 4
                    # Insert one more row. It creates one more page version, but doesn't affect the
                    # relation size.
                    print("Inserting one more row")
                    cur.execute("INSERT INTO foo VALUES (3)")
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
                    assert row['n_relations'] == n_relations
                    assert row['dropped'] == 0
                    assert row['truncated'] == 31
                    assert row['deleted'] == 2
                    # Run GC again, with no changes in the database. Should not remove anything.
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
                    assert row['n_relations'] == n_relations
                    assert row['dropped'] == 0
                    assert row['truncated'] == 31
                    assert row['deleted'] == 0
                    #
                    # Test DROP TABLE checks that relation data and metadata was deleted by GC from object storage
                    #
                    cur.execute("DROP TABLE foo")
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
                    # Each relation fork is counted separately, hence 3.
                    assert row['dropped'] == 3
@@ -1,65 +0,0 @@
 from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
 pytest_plugins = ("fixtures.zenith_fixtures")
 #
 # Test multixact state after branching
 # Now this test is very minimalistic -
 # it only checks next_multixact_id field in restored pg_control,
 # since we don't have functions to check multixact internals.
 #
 def test_multixact(pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin, zenith_cli, base_dir):
    # Create a branch for us
    zenith_cli.run(["branch", "test_multixact", "empty"])
    pg = postgres.create_start('test_multixact')
    print("postgres is running on 'test_multixact' branch")
    pg_conn = pg.connect()
    cur = pg_conn.cursor()
    cur.execute('''
        CREATE TABLE t1(i int primary key);
        INSERT INTO t1 select * from generate_series(1, 100);
    ''')
    cur.execute('SELECT next_multixact_id FROM pg_control_checkpoint()')
    next_multixact_id_old = cur.fetchone()[0]
    # Lock entries in parallel connections to set multixact
    nclients = 3
    connections = []
    for i in range(nclients):
        # Do not turn on autocommit. We want to hold the key-share locks.
        conn = pg.connect(autocommit=False)
        conn.cursor().execute('select * from t1 for key share')
        connections.append(conn)
    # We should have a multixact now. We can close the connections.
    for c in connections:
        c.close()
    # force wal flush
    cur.execute('checkpoint')
    cur.execute('SELECT next_multixact_id, pg_current_wal_flush_lsn() FROM pg_control_checkpoint()')
    res = cur.fetchone()
    next_multixact_id = res[0]
    lsn = res[1]
    # Ensure that we did lock some tuples
    assert int(next_multixact_id) > int(next_multixact_id_old)
    # Branch at this point
    zenith_cli.run(["branch", "test_multixact_new", "test_multixact@" + lsn])
    pg_new = postgres.create_start('test_multixact_new')
    print("postgres is running on 'test_multixact_new' branch")
    pg_new_conn = pg_new.connect()
    cur_new = pg_new_conn.cursor()
    cur_new.execute('SELECT next_multixact_id FROM pg_control_checkpoint()')
    next_multixact_id_new = cur_new.fetchone()[0]
    # Check that we restored pg_controlfile correctly
    assert next_multixact_id_new == next_multixact_id
@@ -1,27 +1,29 @@
 import json
 import uuid
 import pytest
 import psycopg2
-from fixtures.zenith_fixtures import ZenithPageserver
+import getpass
 import json
 pytest_plugins = ("fixtures.zenith_fixtures")
 def test_status(pageserver):
-    assert pageserver.safe_psql('status') == [
+    pg_conn = psycopg2.connect(pageserver.connstr())
-        ('hello world', ),
+    pg_conn.autocommit = True
-    ]
+    cur = pg_conn.cursor()
    cur.execute('status;')
    assert cur.fetchone() == ('hello world',)
    pg_conn.close()
 def test_branch_list(pageserver, zenith_cli):
 def test_branch_list(pageserver: ZenithPageserver, zenith_cli):
    # Create a branch for us
-    zenith_cli.run(["branch", "test_branch_list_main", "empty"])
+    zenith_cli.run(["branch", "test_branch_list_main", "empty"]);
-    conn = pageserver.connect()
+    page_server_conn = psycopg2.connect(pageserver.connstr())
-    cur = conn.cursor()
+    page_server_conn.autocommit = True
    page_server_cur = page_server_conn.cursor()
-    cur.execute(f'branch_list {pageserver.initial_tenant}')
+    page_server_cur.execute('branch_list;')
-    branches = json.loads(cur.fetchone()[0])
+    branches = json.loads(page_server_cur.fetchone()[0])
    # Filter out branches created by other tests
    branches = [x for x in branches if x['name'].startswith('test_branch_list')]
@@ -36,8 +38,8 @@ def test_branch_list(pageserver: ZenithPageserver, zenith_cli):
    zenith_cli.run(['branch', 'test_branch_list_experimental', 'test_branch_list_main'])
    zenith_cli.run(['pg', 'create', 'test_branch_list_experimental'])
-    cur.execute(f'branch_list {pageserver.initial_tenant}')
+    page_server_cur.execute('branch_list;')
-    new_branches = json.loads(cur.fetchone()[0])
+    new_branches = json.loads(page_server_cur.fetchone()[0])
    # Filter out branches created by other tests
    new_branches = [x for x in new_branches if x['name'].startswith('test_branch_list')]
    assert len(new_branches) == 2
@@ -49,28 +51,4 @@ def test_branch_list(pageserver: ZenithPageserver, zenith_cli):
    # TODO: do the LSNs have to match here?
    assert new_branches[1] == branches[0]
-    conn.close()
+    page_server_conn.close()
 def test_tenant_list(pageserver: ZenithPageserver, zenith_cli):
    res = zenith_cli.run(["tenant", "list"])
    res.check_returncode()
    tenants = res.stdout.splitlines()
    assert tenants == [pageserver.initial_tenant]
    conn = pageserver.connect()
    cur = conn.cursor()
    # check same tenant cannot be created twice
    with pytest.raises(psycopg2.DatabaseError, match=f'tenant {pageserver.initial_tenant} already exists'):
        cur.execute(f'tenant_create {pageserver.initial_tenant}')
    # create one more tenant
    tenant1 = uuid.uuid4().hex
    cur.execute(f'tenant_create {tenant1}')
    cur.execute('tenant_list')
    # compare tenants list
    new_tenants = sorted(json.loads(cur.fetchone()[0]))
    assert sorted([pageserver.initial_tenant, tenant1]) == new_tenants
@@ -1,66 +0,0 @@
 import pytest
 import random
 import time
 from contextlib import closing
 from multiprocessing import Process, Value
 from fixtures.zenith_fixtures import WalAcceptorFactory, ZenithPageserver, PostgresFactory
 pytest_plugins = ("fixtures.zenith_fixtures")
 # Check that dead minority doesn't prevent the commits: execute insert n_inserts
 # times, with fault_probability chance of getting a wal acceptor down or up
 # along the way. 2 of 3 are always alive, so the work keeps going.
 def test_pageserver_restart(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory: WalAcceptorFactory):
    # One safekeeper is enough for this test.
    wa_factory.start_n_new(1)
    zenith_cli.run(["branch", "test_pageserver_restart", "empty"])
    pg = postgres.create_start('test_pageserver_restart',
                               wal_acceptors=wa_factory.get_connstrs())
    pg_conn = pg.connect()
    cur = pg_conn.cursor()
    # Create table, and insert some rows. Make it big enough that it doesn't fit in
    # shared_buffers, otherwise the SELECT after restart will just return answer
    # from shared_buffers without hitting the page server, which defeats the point
    # of this test.
    cur.execute('CREATE TABLE foo (t text)')
    cur.execute('''
        INSERT INTO foo
            SELECT 'long string to consume some space' || g
            FROM generate_series(1, 100000) g
    ''')
    # Verify that the table is larger than shared_buffers
    cur.execute('''
        select setting::int * pg_size_bytes(unit) as shared_buffers, pg_relation_size('foo') as tbl_ize
        from pg_settings where name = 'shared_buffers'
    ''')
    row = cur.fetchone()
    print("shared_buffers is {}, table size {}", row[0], row[1]);
    assert int(row[0]) < int(row[1])
    # Stop and restart pageserver. This is a more or less graceful shutdown, although
    # the page server doesn't currently have a shutdown routine so there's no difference
    # between stopping and crashing.
    pageserver.stop();
    pageserver.start();
    # Stopping the pageserver breaks the connection from the postgres backend to
    # the page server, and causes the next query on the connection to fail. Start a new
    # postgres connection too, to avoid that error. (Ideally, the compute node would
    # handle that and retry internally, without propagating the error to the user, but
    # currently it doesn't...)
    pg_conn = pg.connect()
    cur = pg_conn.cursor()
    cur.execute("SELECT count(*) FROM foo")
    assert cur.fetchone() == (100000, )
    # Stop the page server by force, and restart it
    pageserver.stop();
    pageserver.start();
@@ -1,16 +1,17 @@
-from fixtures.zenith_fixtures import PostgresFactory
+import pytest
 pytest_plugins = ("fixtures.zenith_fixtures")
-def test_pgbench(postgres: PostgresFactory, pg_bin, zenith_cli):
+def test_pgbench(pageserver, postgres, pg_bin, zenith_cli):
    # Create a branch for us
-    zenith_cli.run(["branch", "test_pgbench", "empty"])
+    zenith_cli.run(["branch", "test_pgbench", "empty"]);
    pg = postgres.create_start('test_pgbench')
    print("postgres is running on 'test_pgbench' branch")
-    connstr = pg.connstr()
+    connstr = pg.connstr();
    pg_bin.run_capture(['pgbench', '-i', connstr])
    pg_bin.run_capture(['pgbench'] + '-c 10 -T 5 -P 1 -M prepared'.split() + [connstr])
@@ -1,86 +0,0 @@
 import pytest
 from contextlib import closing
 from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory
 pytest_plugins = ("fixtures.zenith_fixtures")
 #
 # Test restarting and recreating a postgres instance
 #
@pytest.mark.parametrize('with_wal_acceptors', [True, False])
 def test_restart_compute(
        zenith_cli,
        pageserver: ZenithPageserver,
        postgres: PostgresFactory,
        pg_bin,
        wa_factory,
        with_wal_acceptors: bool,
    ):
    wal_acceptor_connstrs = None
    zenith_cli.run(["branch", "test_restart_compute", "empty"])
    if with_wal_acceptors:
        wa_factory.start_n_new(3)
        wal_acceptor_connstrs = wa_factory.get_connstrs()
    pg = postgres.create_start('test_restart_compute',
                               wal_acceptors=wal_acceptor_connstrs)
    print("postgres is running on 'test_restart_compute' branch")
    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
            cur.execute('CREATE TABLE t(key int primary key, value text)')
            cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
            cur.execute('SELECT sum(key) FROM t')
            r = cur.fetchone()
            assert r == (5000050000, )
            print("res = ", r)
    # Remove data directory and restart
    pg.stop_and_destroy().create_start('test_restart_compute',
                                       wal_acceptors=wal_acceptor_connstrs)
    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
            # We can still see the row
            cur.execute('SELECT sum(key) FROM t')
            r = cur.fetchone()
            assert r == (5000050000, )
            print("res = ", r)
            # Insert another row
            cur.execute("INSERT INTO t VALUES (100001, 'payload2')")
            cur.execute('SELECT count(*) FROM t')
            r = cur.fetchone()
            assert r == (100001, )
            print("res = ", r)
    # Again remove data directory and restart
    pg.stop_and_destroy().create_start('test_restart_compute',
                                       wal_acceptors=wal_acceptor_connstrs)
    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
            # We can still see the rows
            cur.execute('SELECT count(*) FROM t')
            r = cur.fetchone()
            assert r == (100001, )
            print("res = ", r)
    # And again remove data directory and restart
    pg.stop_and_destroy().create_start('test_restart_compute',
                                       wal_acceptors=wal_acceptor_connstrs)
    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
            # We can still see the rows
            cur.execute('SELECT count(*) FROM t')
            r = cur.fetchone()
            assert r == (100001, )
            print("res = ", r)
@@ -1,124 +0,0 @@
 from contextlib import closing
 import psycopg2.extras
 import time;
 pytest_plugins = ("fixtures.zenith_fixtures")
 def print_gc_result(row):
    print("GC duration {elapsed} ms".format_map(row));
    print("  REL    total: {snapshot_relfiles_total}, needed_by_cutoff {snapshot_relfiles_needed_by_cutoff}, needed_by_branches: {snapshot_relfiles_needed_by_branches}, not_updated: {snapshot_relfiles_not_updated}, removed: {snapshot_relfiles_removed}, dropped: {snapshot_relfiles_dropped}".format_map(row))
    print("  NONREL total: {snapshot_nonrelfiles_total}, needed_by_cutoff {snapshot_nonrelfiles_needed_by_cutoff}, needed_by_branches: {snapshot_nonrelfiles_needed_by_branches}, not_updated: {snapshot_nonrelfiles_not_updated}, removed: {snapshot_nonrelfiles_removed}, dropped: {snapshot_nonrelfiles_dropped}".format_map(row))
 #
 # Test Garbage Collection of old snapshot files
 #
 # This test is pretty tightly coupled with the current implementation of layered
 # storage, in layered_repository.rs.
 #
 def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin):
    zenith_cli.run(["branch", "test_snapfiles_gc", "empty"])
    pg = postgres.create_start('test_snapfiles_gc')
    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
            with closing(pageserver.connect()) as psconn:
                with psconn.cursor(cursor_factory = psycopg2.extras.DictCursor) as pscur:
                    # Get the timeline ID of our branch. We need it for the 'do_gc' command
                    cur.execute("SHOW zenith.zenith_timeline")
                    timeline = cur.fetchone()[0]
                    # Create a test table
                    cur.execute("CREATE TABLE foo(x integer)")
                    cur.execute("INSERT INTO foo VALUES (1)")
                    cur.execute("select relfilenode from pg_class where oid = 'foo'::regclass");
                    row = cur.fetchone();
                    print("relfilenode is {}", row[0]);
                    # Run GC, to clear out any garbage left behind in the catalogs by
                    # the CREATE TABLE command. We want to have a clean slate with no garbage
                    # before running the actual tests below, otherwise the counts won't match
                    # what we expect.
                    #
                    # Also run vacuum first to make it less likely that autovacuum or pruning
                    # kicks in and confuses our numbers.
                    cur.execute("VACUUM")
                    # delete the row, to update the Visibility Map. We don't want the VM
                    # update to confuse our numbers either.
                    cur.execute("DELETE FROM foo")
                    print("Running GC before test")
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print_gc_result(row);
                    # remember the number of files
                    snapshot_relfiles_remain = row['snapshot_relfiles_total'] - row['snapshot_relfiles_removed']
                    assert snapshot_relfiles_remain > 0
                    # Insert a row.
                    print("Inserting one row and running GC")
                    cur.execute("INSERT INTO foo VALUES (1)")
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print_gc_result(row);
                    assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain + 1
                    assert row['snapshot_relfiles_removed'] == 1
                    assert row['snapshot_relfiles_dropped'] == 0
                    # Insert two more rows and run GC.
                    # This should create a new snapshot file with the new contents, and
                    # remove the old one.
                    print("Inserting two more rows and running GC")
                    cur.execute("INSERT INTO foo VALUES (2)")
                    cur.execute("INSERT INTO foo VALUES (3)")
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print_gc_result(row);
                    assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain + 1
                    assert row['snapshot_relfiles_removed'] == 1
                    assert row['snapshot_relfiles_dropped'] == 0
                    # Do it again. Should again create a new snapshot file and remove old one.
                    print("Inserting two more rows and running GC")
                    cur.execute("INSERT INTO foo VALUES (2)")
                    cur.execute("INSERT INTO foo VALUES (3)")
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print_gc_result(row);
                    assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain + 1
                    assert row['snapshot_relfiles_removed'] == 1
                    assert row['snapshot_relfiles_dropped'] == 0
                    # Run GC again, with no changes in the database. Should not remove anything.
                    print("Run GC again, with nothing to do")
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print_gc_result(row);
                    assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain
                    assert row['snapshot_relfiles_removed'] == 0
                    assert row['snapshot_relfiles_dropped'] == 0
                    #
                    # Test DROP TABLE checks that relation data and metadata was deleted by GC from object storage
                    #
                    print("Drop table and run GC again");
                    cur.execute("DROP TABLE foo")
                    pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0")
                    row = pscur.fetchone()
                    print_gc_result(row);
                    # Each relation fork is counted separately, hence 3.
                    assert row['snapshot_relfiles_dropped'] == 3
                    # The catalog updates also create new snapshot files of the catalogs, which
                    # are counted as 'removed'
                    assert row['snapshot_relfiles_removed'] > 0
                    # TODO: perhaps we should count catalog and user relations separately,
                    # to make this kind of testing more robust
@@ -1,48 +0,0 @@
 from contextlib import closing
 import pytest
 from fixtures.zenith_fixtures import (
    TenantFactory,
    ZenithCli,
    PostgresFactory,
 )
@pytest.mark.parametrize('with_wal_acceptors', [False, True])
 def test_tenants_normal_work(
    zenith_cli: ZenithCli,
    tenant_factory: TenantFactory,
    postgres: PostgresFactory,
    wa_factory,
    with_wal_acceptors: bool,
 ):
    """Tests tenants with and without wal acceptors"""
    tenant_1 = tenant_factory.create()
    tenant_2 = tenant_factory.create()
    zenith_cli.run(["branch", f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", "main", f"--tenantid={tenant_1}"])
    zenith_cli.run(["branch", f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", "main", f"--tenantid={tenant_2}"])
    if with_wal_acceptors:
        wa_factory.start_n_new(3)
    pg_tenant1 = postgres.create_start(
        f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}",
        tenant_1,
        wal_acceptors=wa_factory.get_connstrs() if with_wal_acceptors else None,
    )
    pg_tenant2 = postgres.create_start(
        f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}",
        tenant_2,
        wal_acceptors=wa_factory.get_connstrs() if with_wal_acceptors else None,
    )
    for pg in [pg_tenant1, pg_tenant2]:
        with closing(pg.connect()) as conn:
            with conn.cursor() as cur:
                # we rely upon autocommit after each statement
                # as waiting for acceptors happens there
                cur.execute("CREATE TABLE t(key int primary key, value text)")
                cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
                cur.execute("SELECT sum(key) FROM t")
                assert cur.fetchone() == (5000050000,)
@@ -1,88 +1,58 @@
 import os
 from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
 pytest_plugins = ("fixtures.zenith_fixtures")
 #
 # Test branching, when a transaction is in prepared state
 #
-def test_twophase(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+import pytest
-    zenith_cli.run(["branch", "test_twophase", "empty"])
+import getpass
 import psycopg2
-    pg = postgres.create_start('test_twophase', config_lines=['max_prepared_transactions=5'])
+pytest_plugins = ("fixtures.zenith_fixtures")
 def test_twophase(zenith_cli, pageserver, postgres, pg_bin):
    zenith_cli.run(["branch", "test_twophase", "empty"]);
    pg = postgres.create_start('test_twophase', ['max_prepared_transactions=5'])
    print("postgres is running on 'test_twophase' branch")
-    conn = pg.connect()
+    conn = psycopg2.connect(pg.connstr());
    conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
    cur = conn.cursor()
-    cur.execute('CREATE TABLE foo (t text)')
+    cur.execute('CREATE TABLE foo (t text)');
    # Prepare a transaction that will insert a row
-    cur.execute('BEGIN')
+    cur.execute('BEGIN');
-    cur.execute("INSERT INTO foo VALUES ('one')")
+    cur.execute("INSERT INTO foo VALUES ('one')");
-    cur.execute("PREPARE TRANSACTION 'insert_one'")
+    cur.execute("PREPARE TRANSACTION 'insert_one'");
    # Prepare another transaction that will insert a row
-    cur.execute('BEGIN')
+    cur.execute('BEGIN');
-    cur.execute("INSERT INTO foo VALUES ('two')")
+    cur.execute("INSERT INTO foo VALUES ('two')");
-    cur.execute("PREPARE TRANSACTION 'insert_two'")
+    cur.execute("PREPARE TRANSACTION 'insert_two'");
-    # Prepare a transaction that will insert a row
+    cur.execute('BEGIN');
-    cur.execute('BEGIN')
+    cur.execute("INSERT INTO foo VALUES ('three')");
-    cur.execute("INSERT INTO foo VALUES ('three')")
+    cur.execute("PREPARE TRANSACTION 'insert_three'");
-    cur.execute("PREPARE TRANSACTION 'insert_three'")
+    cur.execute("COMMIT PREPARED 'insert_three'");
-    # Prepare another transaction that will insert a row
+    cur.execute('SELECT pg_current_wal_insert_lsn()');
-    cur.execute('BEGIN')
+    lsn = cur.fetchone()[0]
    cur.execute("INSERT INTO foo VALUES ('four')")
    cur.execute("PREPARE TRANSACTION 'insert_four'")
    # On checkpoint state data copied to files in
    # pg_twophase directory and fsynced
    cur.execute('CHECKPOINT')
    twophase_files = os.listdir(pg.pg_twophase_dir_path())
    print(twophase_files)
    assert len(twophase_files) == 4
    cur.execute("COMMIT PREPARED 'insert_three'")
    cur.execute("ROLLBACK PREPARED 'insert_four'")
    cur.execute('CHECKPOINT')
    twophase_files = os.listdir(pg.pg_twophase_dir_path())
    print(twophase_files)
    assert len(twophase_files) == 2
    # Create a branch with the transaction in prepared state
-    zenith_cli.run(["branch", "test_twophase_prepared", "test_twophase"])
+    zenith_cli.run(["branch", "test_twophase_prepared", "test_twophase@"+lsn]);
-    # Create compute node, but don't start.
+    pg2 = postgres.create_start('test_twophase_prepared', ['max_prepared_transactions=5'])
-    # We want to observe pgdata before postgres starts
+    conn2 = psycopg2.connect(pg2.connstr());
-    pg2 = postgres.create(
+    conn2.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
        'test_twophase_prepared',
        config_lines=['max_prepared_transactions=5'],
    )
    # Check that we restored only needed twophase files
    twophase_files2 = os.listdir(pg2.pg_twophase_dir_path())
    print(twophase_files2)
    assert twophase_files2.sort() == twophase_files.sort()
    pg2 = pg2.start()
    conn2 = pg2.connect()
    cur2 = conn2.cursor()
-    # On the new branch, commit one of the prepared transactions,
+    # On the new branch, commit one of the prepared transactions, abort the other one.
-    # abort the other one.
+    cur2.execute("COMMIT PREPARED 'insert_one'");
-    cur2.execute("COMMIT PREPARED 'insert_one'")
+    cur2.execute("ROLLBACK PREPARED 'insert_two'");
    cur2.execute("ROLLBACK PREPARED 'insert_two'")
-    cur2.execute('SELECT * FROM foo')
+    cur2.execute('SELECT * FROM foo');
-    assert cur2.fetchall() == [('one',), ('three',)]
+    assert(cur2.fetchall() == [('one',),('three',)]);
-    # Only one committed insert is visible on the original branch
+    # Neither insert is visible on the original branch, the transactions are still
-    cur.execute('SELECT * FROM foo')
+    # in prepared state there.
-    assert cur.fetchall() == [('three',)]
+    cur.execute('SELECT * FROM foo');
    assert(cur.fetchall() == [('three',)]);
@@ -1,200 +0,0 @@
 import pytest
 import random
 import time
 from contextlib import closing
 from multiprocessing import Process, Value
 from fixtures.zenith_fixtures import WalAcceptorFactory, ZenithPageserver, PostgresFactory
 pytest_plugins = ("fixtures.zenith_fixtures")
 # basic test, write something in setup with wal acceptors, ensure that commits
 # succeed and data is written
 def test_normal_work(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory):
    zenith_cli.run(["branch", "test_wal_acceptors_normal_work", "empty"])
    wa_factory.start_n_new(3)
    pg = postgres.create_start('test_wal_acceptors_normal_work',
                               wal_acceptors=wa_factory.get_connstrs())
    with closing(pg.connect()) as conn:
        with conn.cursor() as cur:
            # we rely upon autocommit after each statement
            # as waiting for acceptors happens there
            cur.execute('CREATE TABLE t(key int primary key, value text)')
            cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
            cur.execute('SELECT sum(key) FROM t')
            assert cur.fetchone() == (5000050000, )
 # Run page server and multiple acceptors, and multiple compute nodes running
 # against different timelines.
 def test_many_timelines(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory):
    n_timelines = 2
    wa_factory.start_n_new(3)
    branches = ["test_wal_acceptors_many_timelines_{}".format(tlin) for tlin in range(n_timelines)]
    # start postgres on each timeline
    pgs = []
    for branch in branches:
        zenith_cli.run(["branch", branch, "empty"])
        pgs.append(postgres.create_start(branch, wal_acceptors=wa_factory.get_connstrs()))
    # Do everything in different loops to have actions on different timelines
    # interleaved.
    # create schema
    for pg in pgs:
        pg.safe_psql("CREATE TABLE t(key int primary key, value text)")
    # Populate data
    for pg in pgs:
        pg.safe_psql("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
    # Check data
    for pg in pgs:
        res = pg.safe_psql("SELECT sum(key) FROM t")
        assert res[0] == (5000050000, )
 # Check that dead minority doesn't prevent the commits: execute insert n_inserts
 # times, with fault_probability chance of getting a wal acceptor down or up
 # along the way. 2 of 3 are always alive, so the work keeps going.
 def test_restarts(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory: WalAcceptorFactory):
    fault_probability = 0.01
    n_inserts = 1000
    n_acceptors = 3
    wa_factory.start_n_new(n_acceptors)
    zenith_cli.run(["branch", "test_wal_acceptors_restarts", "empty"])
    pg = postgres.create_start('test_wal_acceptors_restarts',
                               wal_acceptors=wa_factory.get_connstrs())
    # we rely upon autocommit after each statement
    # as waiting for acceptors happens there
    pg_conn = pg.connect()
    cur = pg_conn.cursor()
    failed_node = None
    cur.execute('CREATE TABLE t(key int primary key, value text)')
    for i in range(n_inserts):
        cur.execute("INSERT INTO t values (%s, 'payload');", (i + 1, ))
        if random.random() <= fault_probability:
            if failed_node is None:
                failed_node = wa_factory.instances[random.randrange(0, n_acceptors)]
                failed_node.stop()
            else:
                failed_node.start()
                failed_node = None
    cur.execute('SELECT sum(key) FROM t')
    assert cur.fetchone() == (500500, )
 start_delay_sec = 2
 def delayed_wal_acceptor_start(wa):
    time.sleep(start_delay_sec)
    wa.start()
 # When majority of acceptors is offline, commits are expected to be frozen
 def test_unavailability(zenith_cli, postgres: PostgresFactory, wa_factory):
    wa_factory.start_n_new(2)
    zenith_cli.run(["branch", "test_wal_acceptors_unavailability", "empty"])
    pg = postgres.create_start('test_wal_acceptors_unavailability',
                               wal_acceptors=wa_factory.get_connstrs())
    # we rely upon autocommit after each statement
    # as waiting for acceptors happens there
    pg_conn = pg.connect()
    cur = pg_conn.cursor()
    # check basic work with table
    cur.execute('CREATE TABLE t(key int primary key, value text)')
    cur.execute("INSERT INTO t values (1, 'payload')")
    # shutdown one of two acceptors, that is, majority
    wa_factory.instances[0].stop()
    proc = Process(target=delayed_wal_acceptor_start, args=(wa_factory.instances[0], ))
    proc.start()
    start = time.time()
    cur.execute("INSERT INTO t values (2, 'payload')")
    # ensure that the query above was hanging while acceptor was down
    assert (time.time() - start) >= start_delay_sec
    proc.join()
    # for the world's balance, do the same with second acceptor
    wa_factory.instances[1].stop()
    proc = Process(target=delayed_wal_acceptor_start, args=(wa_factory.instances[1], ))
    proc.start()
    start = time.time()
    cur.execute("INSERT INTO t values (3, 'payload')")
    # ensure that the query above was hanging while acceptor was down
    assert (time.time() - start) >= start_delay_sec
    proc.join()
    cur.execute("INSERT INTO t values (4, 'payload')")
    cur.execute('SELECT sum(key) FROM t')
    assert cur.fetchone() == (10, )
 # shut down random subset of acceptors, sleep, wake them up, rinse, repeat
 def xmas_garland(acceptors, stop):
    while not bool(stop.value):
        victims = []
        for wa in acceptors:
            if random.random() >= 0.5:
                victims.append(wa)
        for v in victims:
            v.stop()
        time.sleep(1)
        for v in victims:
            v.start()
        time.sleep(1)
 # value which gets unset on exit
@pytest.fixture
 def stop_value():
    stop = Value('i', 0)
    yield stop
    stop.value = 1
 # do inserts while concurrently getting up/down subsets of acceptors
 def test_race_conditions(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory, stop_value):
    wa_factory.start_n_new(3)
    zenith_cli.run(["branch", "test_wal_acceptors_race_conditions", "empty"])
    pg = postgres.create_start('test_wal_acceptors_race_conditions',
                               wal_acceptors=wa_factory.get_connstrs())
    # we rely upon autocommit after each statement
    # as waiting for acceptors happens there
    pg_conn = pg.connect()
    cur = pg_conn.cursor()
    cur.execute('CREATE TABLE t(key int primary key, value text)')
    proc = Process(target=xmas_garland, args=(wa_factory.instances, stop_value))
    proc.start()
    for i in range(1000):
        cur.execute("INSERT INTO t values (%s, 'payload');", (i + 1, ))
    cur.execute('SELECT sum(key) FROM t')
    assert cur.fetchone() == (500500, )
    stop_value.value = 1
    proc.join()
--- a/Show More
+++ b/Show More