code cleanup for compute_node_rebase branch

2. Parse pg_control to retrieve systemid, lsn and so on. Store it in pagecache.
3. Setup compute node without files: only request a few essential files from pageserver to bootstrap. And after that route ALL I/O requests to pageserver.
Use initdb --compute-node flag to create such minimal node without files. And GUC 'computenode_mode=true'to request all pages from pageserver

.circleci/config.yml

@@ -1,295 +0,0 @@
-version: 2.1
-
-orbs:
-  python: circleci/python@1.4.0
-
-executors:
-  zenith-build-executor:
-    resource_class: xlarge
-    docker:
-      - image: cimg/rust:1.52.1
-
-jobs:
-
-  # A job to build postgres
-  build-postgres:
-    executor: zenith-build-executor
-    steps:
-        # Checkout the git repo (circleci doesn't have a flag to enable submodules here)
-      - checkout
-
-        # Grab the postgres git revision to build a cache key.
-        # Note this works even though the submodule hasn't been checkout out yet.
-      - run:
-          name: Get postgres cache key
-          command: |
-            git rev-parse HEAD:vendor/postgres > /tmp/cache-key-postgres
-
-      - restore_cache:
-          name: Restore postgres cache
-          keys:
-            # Restore ONLY if the rev key matches exactly
-            - v03-postgres-cache-{{ checksum "/tmp/cache-key-postgres" }}
-
-        # FIXME We could cache our own docker container, instead of installing packages every time.
-      - run:
-          name: apt install dependencies
-          command: |
-            if [ ! -e tmp_install/bin/postgres ]; then
-              sudo apt update
-              sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev
-            fi
-
-        # Build postgres if the restore_cache didn't find a build.
-        # `make` can't figure out whether the cache is valid, since
-        # it only compares file timestamps.
-      - run:
-          name: build postgres
-          command: |
-            if [ ! -e tmp_install/bin/postgres ]; then
-              # "depth 1" saves some time by not cloning the whole repo
-              git submodule update --init --depth 1
-              make postgres
-            fi
-
-      - save_cache:
-          name: Save postgres cache
-          key: v03-postgres-cache-{{ checksum "/tmp/cache-key-postgres" }}
-          paths:
-            - tmp_install
-
-  # A job to build zenith rust code
-  build-zenith:
-    executor: zenith-build-executor
-    parameters:
-      build_type:
-        type: enum
-        enum: ["debug", "release"]
-    steps:
-      - run:
-          name: apt install dependencies
-          command: |
-            sudo apt update
-            sudo apt install libssl-dev clang
-
-        # Checkout the git repo (without submodules)
-      - checkout
-
-        # Grab the postgres git revision to build a cache key.
-        # Note this works even though the submodule hasn't been checkout out yet.
-      - run:
-          name: Get postgres cache key
-          command: |
-            git rev-parse HEAD:vendor/postgres > /tmp/cache-key-postgres
-
-      - restore_cache:
-          name: Restore postgres cache
-          keys:
-            # Restore ONLY if the rev key matches exactly
-            - v03-postgres-cache-{{ checksum "/tmp/cache-key-postgres" }}
-
-      - restore_cache:
-          name: Restore rust cache
-          keys:
-            # Require an exact match. While an out of date cache might speed up the build,
-            # there's no way to clean out old packages, so the cache grows every time something
-            # changes.
-            - v03-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }}
-
-        # Build the rust code, including test binaries
-      - run:
-          name: Rust build << parameters.build_type >>
-          command: |
-            export CARGO_INCREMENTAL=0
-            BUILD_TYPE="<< parameters.build_type >>"
-            if [[ $BUILD_TYPE == "debug" ]]; then
-              echo "Build in debug mode"
-              cargo build --bins --tests
-            elif [[ $BUILD_TYPE == "release" ]]; then
-              echo "Build in release mode"
-              cargo build --release --bins --tests
-            fi
-
-      - save_cache:
-          name: Save rust cache
-          key: v03-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }}
-          paths:
-            - ~/.cargo/registry
-            - ~/.cargo/git
-            - target
-
-        # Run rust unit tests
-      - run: cargo test
-
-        # Install the rust binaries, for use by test jobs
-        # `--locked` is required; otherwise, `cargo install` will ignore Cargo.lock.
-        # FIXME: this is a really silly way to install; maybe we should just output
-        # a tarball as an artifact? Or a .deb package?
-      - run:
-          name: cargo install
-          command: |
-            export CARGO_INCREMENTAL=0
-            BUILD_TYPE="<< parameters.build_type >>"
-            if [[ $BUILD_TYPE == "debug" ]]; then
-              echo "Install debug mode"
-              CARGO_FLAGS="--debug"
-            elif [[ $BUILD_TYPE == "release" ]]; then
-              echo "Install release mode"
-              # The default is release mode; there is no --release flag.
-              CARGO_FLAGS=""
-            fi
-            cargo install $CARGO_FLAGS --locked --root /tmp/zenith --path pageserver
-            cargo install $CARGO_FLAGS --locked --root /tmp/zenith --path walkeeper
-            cargo install $CARGO_FLAGS --locked --root /tmp/zenith --path zenith
-
-        # Install the postgres binaries, for use by test jobs
-        # FIXME: this is a silly way to do "install"; maybe just output a standard
-        # postgres package, whatever the favored form is (tarball? .deb package?)
-        # Note that pg_regress needs some build artifacts that probably aren't
-        # in the usual package...?
-      - run:
-          name: postgres install
-          command: |
-            cp -a tmp_install /tmp/zenith/pg_install
-
-        # Save the rust output binaries for other jobs in this workflow.
-      - persist_to_workspace:
-          root: /tmp/zenith
-          paths:
-            - "*"
-
-  run-pytest:
-    #description: "Run pytest"
-    executor: python/default
-    parameters:
-      # pytest args to specify the tests to run.
-      #
-      # This can be a test file name, e.g. 'test_pgbench.py, or a subdirectory,
-      # or '-k foobar' to run tests containing string 'foobar'. See pytest man page
-      # section SPECIFYING TESTS / SELECTING TESTS for details.
-      #
-      # Select the type of Rust build. Must be "release" or "debug".
-      build_type:
-        type: string
-        default: "debug"
-      # This parameter is required, to prevent the mistake of running all tests in one job.
-      test_selection:
-        type: string
-        default: ""
-      # Arbitrary parameters to pytest. For example "-s" to prevent capturing stdout/stderr
-      extra_params:
-        type: string
-        default: ""
-      needs_postgres_source:
-        type: boolean
-        default: false
-    steps:
-      - attach_workspace:
-          at: /tmp/zenith
-      - checkout
-      - when:
-          condition: << parameters.needs_postgres_source >>
-          steps:
-            - run: git submodule update --init --depth 1
-      - run:
-          name: Install pipenv & deps
-          working_directory: test_runner
-          command: |
-            pip install pipenv
-            pipenv install
-      - run:
-          name: Run pytest
-          working_directory: test_runner
-          environment:
-            - ZENITH_BIN: /tmp/zenith/bin
-            - POSTGRES_DISTRIB_DIR: /tmp/zenith/pg_install
-            - TEST_OUTPUT: /tmp/test_output
-          command: |
-            TEST_SELECTION="<< parameters.test_selection >>"
-            EXTRA_PARAMS="<< parameters.extra_params >>"
-            if [ -z "$TEST_SELECTION" ]; then
-              echo "test_selection must be set"
-              exit 1
-            fi
-            # Run the tests.
-            #
-            # The junit.xml file allows CircleCI to display more fine-grained test information
-            # in its "Tests" tab in the results page.
-            # -s prevents pytest from capturing output, which helps to see
-            # what's going on if the test hangs
-            # --verbose prints name of each test (helpful when there are
-            # multiple tests in one file)
-            # -rA prints summary in the end
-            pipenv run pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short -s --verbose -rA $TEST_SELECTION $EXTRA_PARAMS
-      - run:
-          # CircleCI artifacts are preserved one file at a time, so skipping
-          # this step isn't a good idea. If you want to extract the
-          # pageserver state, perhaps a tarball would be a better idea.
-          name: Delete all data but logs
-          when: always
-          command: |
-            du -sh /tmp/test_output/*
-            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "wal_acceptor.log" ! -name "regression.diffs" -delete
-            du -sh /tmp/test_output/*
-      - store_artifacts:
-          path: /tmp/test_output
-      # The store_test_results step tells CircleCI where to find the junit.xml file.
-      - store_test_results:
-          path: /tmp/test_output
-
-  # Build zenithdb/zenith:latest image and push it to Docker hub
-  docker-image:
-    docker:
-      - image: cimg/base:2021.04
-    steps:
-      - checkout
-      - setup_remote_docker:
-          docker_layer_caching: true
-      - run:
-          name: Init postgres submodule
-          command: git submodule update --init --depth 1
-      - run:
-          name: Build and push Docker image
-          command: |
-            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
-            docker build -t zenithdb/zenith:latest . && docker push zenithdb/zenith:latest
-
-workflows:
-  build_and_test:
-    jobs:
-      - build-postgres
-      - build-zenith:
-          name: build-zenith-<< matrix.build_type >>
-          matrix:
-            parameters:
-              build_type: ["debug", "release"]
-          requires:
-            - build-postgres
-      - run-pytest:
-          name: pg_regress tests << matrix.build_type >>
-          matrix:
-            parameters:
-              build_type: ["debug", "release"]
-          test_selection: batch_pg_regress
-          needs_postgres_source: true
-          requires:
-            - build-zenith-<< matrix.build_type >>
-      - run-pytest:
-          name: other tests << matrix.build_type >>
-          matrix:
-            parameters:
-              build_type: ["debug", "release"]
-          test_selection: batch_others
-          requires:
-            - build-zenith-<< matrix.build_type >>
-      - docker-image:
-          # Context gives an ability to login
-          context: Docker Hub
-          # Build image only for commits to main
-          filters:
-            branches:
-              only:
-                - main
-          requires:
-            - pg_regress tests release
-            - other tests release

.dockerignore

@@ -1,13 +0,0 @@
-**/.git/
-**/__pycache__
-**/.pytest_cache
-
-/target
-/tmp_check
-/tmp_install
-/tmp_check_cli
-/test_output
-/.vscode
-/.zenith
-/integration_tests/.zenith
-/Dockerfile

.github/workflows/notifications.yml

@@ -1,45 +0,0 @@
-name: Send Notifications
-
-on:
-  push:
-    branches: [ main ]
-
-jobs:
-  send-notifications:
-    timeout-minutes: 30
-    name: send commit notifications
-    runs-on: ubuntu-latest
-
-    steps:
-
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          submodules: true
-          fetch-depth: 2
-
-      - name: Form variables for notification message
-        id: git_info_grab
-        run: |
-          git_stat=$(git show --stat=50)
-          git_stat="${git_stat//'%'/'%25'}"
-          git_stat="${git_stat//$'\n'/'%0A'}"
-          git_stat="${git_stat//$'\r'/'%0D'}"
-          git_stat="${git_stat// / }" # space -> 'Space En', as github tends to eat ordinary spaces
-          echo "::set-output name=git_stat::$git_stat"
-          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
-          echo "##[set-output name=git_branch;]$(echo ${GITHUB_REF#refs/heads/})"
-
-      - name: Send notification
-        uses: appleboy/telegram-action@master
-        with:
-          to: ${{ secrets.TELEGRAM_TO }}
-          token: ${{ secrets.TELEGRAM_TOKEN }}
-          format: markdown
-          args: |
-            *@${{ github.actor }} pushed to* [${{ github.repository }}:${{steps.git_info_grab.outputs.git_branch}}](github.com/${{ github.repository }}/commit/${{steps.git_info_grab.outputs.sha_short }})
-
-            ```
-            ${{ steps.git_info_grab.outputs.git_stat }}
-            ```
-

.github/workflows/testing.yml

@@ -1,41 +1,49 @@
-name: Build and Test
+name: regression check
 
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
+on: [push]
 
 jobs:
   regression-check:
-    strategy:
-      matrix:
-        # If we want to duplicate this job for different
-        # Rust toolchains (e.g. nightly or 1.37.0), add them here.
-        rust_toolchain: [stable]
-        os: [ubuntu-latest]
-    timeout-minutes: 30
     name: run regression test suite
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
 
     steps:
+
       - name: Checkout
         uses: actions/checkout@v2
         with:
           submodules: true
           fetch-depth: 2
 
-      - name: install rust toolchain ${{ matrix.rust_toolchain }}
-        uses: actions-rs/toolchain@v1
+      - name: Form variables for notification message
+        id: git_info_grab
+        run: |
+          git_stat=$(git show --stat=50)
+          git_stat="${git_stat//'%'/'%25'}"
+          git_stat="${git_stat//$'\n'/'%0A'}"
+          git_stat="${git_stat//$'\r'/'%0D'}"
+          git_stat="${git_stat// / }" # space -> 'Space En', as github tends to eat ordinary spaces
+          echo "::set-output name=git_stat::$git_stat"
+          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+          echo "##[set-output name=git_branch;]$(echo ${GITHUB_REF#refs/heads/})"
+
+      - name: Send notification
+        uses: appleboy/telegram-action@master
         with:
-          profile: minimal
-          toolchain: ${{ matrix.rust_toolchain }}
-          override: true
+          to: ${{ secrets.TELEGRAM_TO }}
+          token: ${{ secrets.TELEGRAM_TOKEN }}
+          format: markdown
+          args: |
+            *@${{ github.actor }} pushed to* [${{ github.repository }}:${{steps.git_info_grab.outputs.git_branch}}](github.com/${{ github.repository }}/commit/${{steps.git_info_grab.outputs.sha_short }})
+
+            ```
+            ${{ steps.git_info_grab.outputs.git_stat }}
+            ```
 
       - name: Install postgres dependencies
         run: |
           sudo apt update
-          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev
+          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libcurl4-openssl-dev
 
       - name: Set pg revision for caching
         id: pg_ver
@@ -52,7 +60,11 @@ jobs:
       - name: Build postgres
         if: steps.cache_pg.outputs.cache-hit != 'true'
         run: |
-          make postgres
+          ./pgbuild.sh
+
+      - name: Install rust
+        run: |
+          sudo apt install -y cargo
 
       - name: Cache cargo deps
         id: cache_cargo
@@ -64,10 +76,13 @@ jobs:
             target
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
 
-      - name: Run cargo build
+      # That build is only to build dependencies and can be skipped if Cargo.lock
+      # wasn't changed. Next steps need their own build
+      - name: Install cargo deps
+        if: steps.cache_cargo.outputs.cache-hit != 'true'
         run: |
-          cargo build --workspace --bins --examples --tests
+          cargo build
 
-      - name: Run cargo test
+      - name: Run test
         run: |
-          cargo test -- --nocapture --test-threads=1
+          cargo test --test test_pageserver -- --nocapture --test-threads=1

.gitignore

@@ -1,9 +1,3 @@
 /target
 /tmp_check
 /tmp_install
-/tmp_check_cli
-__pycache__/
-test_output/
-.vscode
-/.zenith
-/integration_tests/.zenith

CONTRIBUTING.md

@@ -1,31 +0,0 @@
-# How to contribute
-
-Howdy! Usual good software engineering practices apply. Write
-tests. Write comments. Follow standard Rust coding practices where
-possible. Use 'cargo fmt' and 'clippy' to tidy up formatting.
-
-There are soft spots in the code, which could use cleanup,
-refactoring, additional comments, and so forth. Let's try to raise the
-bar, and clean things up as we go. Try to leave code in a better shape
-than it was before.
-
-## Submitting changes
-
-1. Make a PR for every change.
-
-   Even seemingly trivial patches can break things in surprising ways.
-Use of common sense is OK. If you're only fixing a typo in a comment,
-it's probably fine to just push it. But if in doubt, open a PR.
-
-2. Get at least one +1 on your PR before you push.
-
-   For simple patches, it will only take a minute for someone to review
-it.
-
-3. Always keep the CI green.
-
-   Do not push, if the CI failed on your PR. Even if you think it's not
-your patch's fault. Help to fix the root cause if something else has
-broken the CI, before pushing.
-
-*Happy Hacking!*

COPYRIGHT

@@ -1,20 +0,0 @@
-This software is licensed under the Apache 2.0 License:
-
-----------------------------------------------------------------------------
-Copyright 2021 Zenith Labs, Inc
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-----------------------------------------------------------------------------
-
-The PostgreSQL submodule in vendor/postgres is licensed under the
-PostgreSQL license. See vendor/postgres/COPYRIGHT.

Cargo.toml

@@ -1,17 +1,6 @@
 [workspace]
 members = [
-    "control_plane",
+    "integration_tests",
     "pageserver",
-    "postgres_ffi",
-    "proxy",
     "walkeeper",
-    "workspace_hack",
-    "zenith",
-    "zenith_metrics",
-    "zenith_utils",
 ]
-
-[profile.release]
-# This is useful for profiling and, to some extent, debug.
-# Besides, debug info should not affect the performance.
-debug = true

Dockerfile

@@ -1,78 +0,0 @@
-#
-# Docker image for console integration testing.
-#
-
-#
-# Build Postgres separately --- this layer will be rebuilt only if one of
-# mentioned paths will get any changes.
-#
-FROM zenithdb/build:buster AS pg-build
-WORKDIR /zenith
-COPY ./vendor/postgres vendor/postgres
-COPY ./Makefile Makefile
-RUN make -j $(getconf _NPROCESSORS_ONLN) -s postgres
-
-#
-# Calculate cargo dependencies.
-# This will always run, but only generate recipe.json with list of dependencies without
-# installing them.
-#
-FROM zenithdb/build:buster AS cargo-deps-inspect
-WORKDIR /zenith
-COPY . .
-RUN cargo chef prepare --recipe-path /zenith/recipe.json
-
-#
-# Build cargo dependencies.
-# This temp cantainner should be rebuilt only if recipe.json was changed.
-#
-FROM zenithdb/build:buster AS deps-build
-WORKDIR /zenith
-COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
-COPY --from=cargo-deps-inspect /usr/local/cargo/bin/cargo-chef /usr/local/cargo/bin/
-COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
-RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json
-
-#
-# Build zenith binaries
-#
-FROM zenithdb/build:buster AS build
-WORKDIR /zenith
-COPY . .
-# Copy cached dependencies
-COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
-COPY --from=deps-build /zenith/target target
-COPY --from=deps-build /usr/local/cargo/ /usr/local/cargo/
-RUN cargo build --release
-
-#
-# Copy binaries to resulting image.
-#
-FROM debian:buster-slim
-WORKDIR /data
-
-RUN apt-get update && apt-get -yq install librocksdb-dev libseccomp-dev openssl && \
-    mkdir zenith_install
-
-COPY --from=build /zenith/target/release/pageserver /usr/local/bin
-COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin
-COPY --from=build /zenith/target/release/proxy /usr/local/bin
-COPY --from=pg-build /zenith/tmp_install postgres_install
-COPY docker-entrypoint.sh /docker-entrypoint.sh
-
-# Remove build artifacts (~ 500 MB)
-RUN rm -rf postgres_install/build && \
-    # 'Install' Postgres binaries locally
-    cp -r postgres_install/* /usr/local/ && \
-    # Prepare an archive of Postgres binaries (should be around 11 MB)
-    # and keep it inside container for an ease of deploy pipeline.
-    cd postgres_install && tar -czf /data/postgres_install.tar.gz . && cd .. && \
-    rm -rf postgres_install
-
-RUN useradd -m -d /data zenith
-
-VOLUME ["/data"]
-USER zenith
-EXPOSE 6400
-ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD ["pageserver"]

Dockerfile.alpine

@@ -1,95 +0,0 @@
-#
-# Docker image for console integration testing.
-#
-# We may also reuse it in CI to unify installation process and as a general binaries building
-# tool for production servers.
-#
-# Dynamic linking is used for librocksdb and libstdc++ bacause librocksdb-sys calls
-# bindgen with "dynamic" feature flag. This also prevents usage of dockerhub alpine-rust
-# images which are statically linked and have guards against any dlopen. I would rather
-# prefer all static binaries so we may change the way librocksdb-sys builds or wait until
-# we will have our own storage and drop rockdb dependency.
-#
-# Cargo-chef is used to separate dependencies building from main binaries building. This
-# way `docker build` will download and install dependencies only of there are changes to
-# out Cargo.toml files.
-#
-
-
-#
-# build postgres separately -- this layer will be rebuilt only if one of
-# mentioned paths will get any changes
-#
-FROM alpine:3.13 as pg-build
-RUN apk add --update clang llvm compiler-rt compiler-rt-static lld musl-dev binutils \
-                     make bison flex readline-dev zlib-dev perl linux-headers libseccomp-dev
-WORKDIR zenith
-COPY ./vendor/postgres vendor/postgres
-COPY ./Makefile Makefile
-# Build using clang and lld
-RUN CC='clang' LD='lld' CFLAGS='-fuse-ld=lld --rtlib=compiler-rt' make postgres -j4
-
-#
-# Calculate cargo dependencies.
-# This will always run, but only generate recipe.json with list of dependencies without
-# installing them.
-#
-FROM alpine:20210212 as cargo-deps-inspect
-RUN apk add --update rust cargo
-RUN cargo install cargo-chef
-WORKDIR zenith
-COPY . .
-RUN cargo chef prepare --recipe-path recipe.json
-
-#
-# Build cargo dependencies.
-# This temp cantainner would be build only if recipe.json was changed.
-#
-FROM alpine:20210212 as deps-build
-RUN apk add --update rust cargo openssl-dev clang build-base
-# rust-rocksdb can be built against system-wide rocksdb -- that saves about
-# 10 minutes during build. Rocksdb apk package is in testing now, but use it
-# anyway. In case of any troubles we can download and build rocksdb here manually
-# (to cache it as a docker layer).
-RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
-WORKDIR zenith
-COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
-COPY --from=cargo-deps-inspect /root/.cargo/bin/cargo-chef /root/.cargo/bin/
-COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
-RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json
-
-#
-# Build zenith binaries
-#
-FROM alpine:20210212 as build
-RUN apk add --update rust cargo openssl-dev clang build-base
-RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
-WORKDIR zenith
-COPY . .
-# Copy cached dependencies
-COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
-COPY --from=deps-build /zenith/target target
-COPY --from=deps-build /root/.cargo /root/.cargo
-RUN cargo build --release
-
-#
-# Copy binaries to resulting image.
-# build-base hare to provide libstdc++ (it will also bring gcc, but leave it this way until we figure
-# out how to statically link rocksdb or avoid it at all).
-#
-FROM alpine:3.13
-RUN apk add --update openssl build-base libseccomp-dev
-RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb
-COPY --from=build /zenith/target/release/pageserver /usr/local/bin
-COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin
-COPY --from=build /zenith/target/release/proxy /usr/local/bin
-COPY --from=pg-build /zenith/tmp_install /usr/local
-COPY docker-entrypoint.sh /docker-entrypoint.sh
-
-RUN addgroup zenith && adduser -h /data -D -G zenith zenith
-VOLUME ["/data"]
-WORKDIR /data
-USER zenith
-EXPOSE 6400
-ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD ["pageserver"]

Dockerfile.build

@@ -1,15 +0,0 @@
-#
-# Image with all the required dependencies to build https://github.com/zenithdb/zenith
-# and Postgres from https://github.com/zenithdb/postgres
-# Also includes some rust development and build tools.
-#
-FROM rust:slim-buster
-WORKDIR /zenith
-
-# Install postgres and zenith build dependencies
-# clang is for rocksdb
-RUN apt-get update && apt-get -yq install automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \
-                                          libseccomp-dev pkg-config libssl-dev librocksdb-dev clang
-
-# Install rust tools
-RUN rustup component add clippy && cargo install cargo-chef cargo-audit

LICENSE

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

Makefile

@@ -1,70 +0,0 @@
-# Seccomp BPF is only available for Linux
-UNAME_S := $(shell uname -s)
-ifeq ($(UNAME_S),Linux)
-	SECCOMP = --with-libseccomp
-else
-	SECCOMP =
-endif
-
-#
-# Top level Makefile to build Zenith and PostgreSQL
-#
-all: zenith postgres
-
-# We don't want to run 'cargo build' in parallel with the postgres build,
-# because interleaving cargo build output with postgres build output looks
-# confusing. Also, 'cargo build' is parallel on its own, so it would be too
-# much parallelism. (Recursive invocation of postgres target still gets any
-# '-j' flag from the command line, so 'make -j' is still useful.)
-.NOTPARALLEL:
-
-### Zenith Rust bits
-#
-# The 'postgres_ffi' depends on the Postgres headers.
-zenith: postgres-headers
-	cargo build
-
-### PostgreSQL parts
-tmp_install/build/config.status:
-	+@echo "Configuring postgres build"
-	mkdir -p tmp_install/build
-	(cd tmp_install/build && \
-	../../vendor/postgres/configure CFLAGS='-O0 -g3 $(CFLAGS)' \
-		--enable-cassert \
-		--enable-debug \
-		--enable-depend \
-		$(SECCOMP) \
-		--prefix=$(abspath tmp_install) > configure.log)
-
-# nicer alias for running 'configure'
-postgres-configure: tmp_install/build/config.status
-
-# Install the PostgreSQL header files into tmp_install/include
-postgres-headers: postgres-configure
-	+@echo "Installing PostgreSQL headers"
-	$(MAKE) -C tmp_install/build/src/include MAKELEVEL=0 install
-
-
-# Compile and install PostgreSQL and contrib/zenith
-postgres: postgres-configure
-	+@echo "Compiling PostgreSQL"
-	$(MAKE) -C tmp_install/build MAKELEVEL=0 install
-	+@echo "Compiling contrib/zenith"
-	$(MAKE) -C tmp_install/build/contrib/zenith install
-	+@echo "Compiling contrib/zenith_test_utils"
-	$(MAKE) -C tmp_install/build/contrib/zenith_test_utils install
-
-postgres-clean:
-	$(MAKE) -C tmp_install/build MAKELEVEL=0 clean
-
-# This doesn't remove the effects of 'configure'.
-clean:
-	cd tmp_install/build && ${MAKE} clean
-	cargo clean
-
-# This removes everything
-distclean:
-	rm -rf tmp_install
-	cargo clean
-
-.PHONY: postgres-configure postgres postgres-headers zenith

Pipfile

@@ -1 +0,0 @@
-./test_runner/Pipfile

Pipfile.lock

@@ -1 +0,0 @@
-./test_runner/Pipfile.lock

README.md

@@ -2,155 +2,43 @@
 
 Zenith substitutes PostgreSQL storage layer and redistributes data across a cluster of nodes
 
-## Running local installation
-
-1. Install build dependencies and other useful packages
-
-On Ubuntu or Debian this set of packages should be sufficient to build the code:
-```text
-apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libseccomp-dev \
-libssl-dev clang
-```
-
-[Rust] 1.52 or later is also required.
-
-To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively.
-
-To run the integration tests (not required to use the code), install
-Python (3.6 or higher), and install python3 packages with `pipenv` using `pipenv install` in the project directory.
-
-2. Build zenith and patched postgres
-```sh
-git clone --recursive https://github.com/zenithdb/zenith.git
-cd zenith
-make -j5
-```
-
-3. Start pageserver and postgres on top of it (should be called from repo root):
-```sh
-# Create repository in .zenith with proper paths to binaries and data
-# Later that would be responsibility of a package install script
-> ./target/debug/zenith init
-pageserver init succeeded
-
-# start pageserver
-> ./target/debug/zenith start
-Starting pageserver at '127.0.0.1:64000' in .zenith
-Pageserver started
-
-# start postgres on top on the pageserver
-> ./target/debug/zenith pg start main
-Starting postgres node at 'host=127.0.0.1 port=55432 user=stas'
-waiting for server to start.... done
-
-# check list of running postgres instances
-> ./target/debug/zenith pg list
-BRANCH	ADDRESS		LSN		STATUS
-main	127.0.0.1:55432	0/1609610	running
-```
-
-4. Now it is possible to connect to postgres and run some queries:
-```text
-> psql -p55432 -h 127.0.0.1 postgres
-postgres=# CREATE TABLE t(key int primary key, value text);
-CREATE TABLE
-postgres=# insert into t values(1,1);
-INSERT 0 1
-postgres=# select * from t;
- key | value
------+-------
-   1 | 1
-(1 row)
-```
-
-5. And create branches and run postgres on them:
-```sh
-# create branch named migration_check
-> ./target/debug/zenith branch migration_check main
-Created branch 'migration_check' at 0/1609610
-
-# check branches tree
-> ./target/debug/zenith branch
- main
- ┗━ @0/1609610: migration_check
-
-# start postgres on that branch
-> ./target/debug/zenith pg start migration_check
-Starting postgres node at 'host=127.0.0.1 port=55433 user=stas'
-waiting for server to start.... done
-
-# this new postgres instance will have all the data from 'main' postgres,
-# but all modifications would not affect data in original postgres
-> psql -p55433 -h 127.0.0.1 postgres
-postgres=# select * from t;
- key | value
------+-------
-   1 | 1
-(1 row)
-
-postgres=# insert into t values(2,2);
-INSERT 0 1
-```
-
 ## Running tests
 
 ```sh
 git clone --recursive https://github.com/libzenith/zenith.git
-make # builds also postgres and installs it to ./tmp_install
-cd test_runner
-pytest
+./pgbuild.sh # builds postgres and installs it to ./tmp_install
+cargo test -- --test-threads=1
 ```
 
-## Documentation
-
-Now we use README files to cover design ideas and overall architecture for each module and `rustdoc` style documentation comments. See also [/docs/](/docs/) a top-level overview of all available markdown documentation.
-
-To view your `rustdoc` documentation in a browser, try running `cargo doc --no-deps --open`
-
 ## Source tree layout
 
-`/control_plane`:
+/walkeeper:
 
-Local control plane.
-Functions to start, configure and stop pageserver and postgres instances running as a local processes.
-Intended to be used in integration tests and in CLI tools for local installations.
+WAL safekeeper. Written in Rust.
 
-`/zenith`
-
-Main entry point for the 'zenith' CLI utility.
-TODO: Doesn't it belong to control_plane?
-
-`/postgres_ffi`:
-
-Utility functions for interacting with PostgreSQL file formats.
-Misc constants, copied from PostgreSQL headers.
-
-`/zenith_utils`:
-
-Helpers that are shared between other crates in this repository.
-
-`/walkeeper`:
-
-WAL safekeeper (also known as WAL acceptor). Written in Rust.
-
-`/pageserver`:
+/pageserver:
 
 Page Server. Written in Rust.
 
 Depends on the modified 'postgres' binary for WAL redo.
 
-`/vendor/postgres`:
+/integration_tests:
+
+Tests with different combinations of a Postgres compute node, WAL safekeeper and Page Server.
+
+/mgmt-console:
+
+Web UI to launch (modified) Postgres servers, using S3 as the backing store. Written in Python.
+This is somewhat outdated, as it doesn't use the WAL safekeeper or Page Servers.
+
+/vendor/postgres:
 
 PostgreSQL source tree, with the modifications needed for Zenith.
 
-`/vendor/postgres/contrib/zenith`:
+/vendor/postgres/src/bin/safekeeper:
 
-PostgreSQL extension that implements storage manager API and network communications with remote page server.
+Extension (safekeeper_proxy) that runs in the compute node, and connects to the WAL safekeepers
+and streams the WAL
 
-`/test_runner`:
 
-Integration tests, written in Python using the `pytest` framework.
 
-`test_runner/zenith_regress`:
-
-Quick way to add new SQL regression test to integration tests set.

cli-v2-story.md

@@ -1,188 +0,0 @@
-Create a new Zenith repository in the current directory:
-
-    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli init
-    The files belonging to this database system will be owned by user "heikki".
-    This user must also own the server process.
-    
-    The database cluster will be initialized with locale "en_GB.UTF-8".
-    The default database encoding has accordingly been set to "UTF8".
-    The default text search configuration will be set to "english".
-    
-    Data page checksums are disabled.
-    
-    creating directory tmp ... ok
-    creating subdirectories ... ok
-    selecting dynamic shared memory implementation ... posix
-    selecting default max_connections ... 100
-    selecting default shared_buffers ... 128MB
-    selecting default time zone ... Europe/Helsinki
-    creating configuration files ... ok
-    running bootstrap script ... ok
-    performing post-bootstrap initialization ... ok
-    syncing data to disk ... ok
-    
-    initdb: warning: enabling "trust" authentication for local connections
-    You can change this by editing pg_hba.conf or using the option -A, or
-    --auth-local and --auth-host, the next time you run initdb.
-    new zenith repository was created in .zenith
-
-Initially, there is only one branch:
-
-    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli branch
-      main
-
-Start a local Postgres instance on the branch:
-
-    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli start main
-    Creating data directory from snapshot at 0/15FFB08...
-    waiting for server to start....2021-04-13 09:27:43.919 EEST [984664] LOG:  starting PostgreSQL 14devel on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit
-    2021-04-13 09:27:43.920 EEST [984664] LOG:  listening on IPv6 address "::1", port 5432
-    2021-04-13 09:27:43.920 EEST [984664] LOG:  listening on IPv4 address "127.0.0.1", port 5432
-    2021-04-13 09:27:43.927 EEST [984664] LOG:  listening on Unix socket "/tmp/.s.PGSQL.5432"
-    2021-04-13 09:27:43.939 EEST [984665] LOG:  database system was interrupted; last known up at 2021-04-13 09:27:33 EEST
-    2021-04-13 09:27:43.939 EEST [984665] LOG:  creating missing WAL directory "pg_wal/archive_status"
-    2021-04-13 09:27:44.189 EEST [984665] LOG:  database system was not properly shut down; automatic recovery in progress
-    2021-04-13 09:27:44.195 EEST [984665] LOG:  invalid record length at 0/15FFB80: wanted 24, got 0
-    2021-04-13 09:27:44.195 EEST [984665] LOG:  redo is not required
-    2021-04-13 09:27:44.225 EEST [984664] LOG:  database system is ready to accept connections
-     done
-    server started
-
-Run some commands against it:
-
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -c "create table foo (t text);" 
-    CREATE TABLE
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -c "insert into foo values ('inserted on the main branch');" 
-    INSERT 0 1
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -c "select * from foo" 
-                  t              
-    -----------------------------
-     inserted on the main branch
-    (1 row)
-
-Create a new branch called 'experimental'. We create it from the
-current end of the 'main' branch, but you could specify a different
-LSN as the start point instead.
-
-    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli branch experimental main
-    branching at end of WAL: 0/161F478
-    
-    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli branch 
-      experimental
-      main
-
-Start another Postgres instance off the 'experimental' branch:
-
-    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli start experimental -- -o -p5433
-    Creating data directory from snapshot at 0/15FFB08...
-    waiting for server to start....2021-04-13 09:28:41.874 EEST [984766] LOG:  starting PostgreSQL 14devel on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit
-    2021-04-13 09:28:41.875 EEST [984766] LOG:  listening on IPv6 address "::1", port 5433
-    2021-04-13 09:28:41.875 EEST [984766] LOG:  listening on IPv4 address "127.0.0.1", port 5433
-    2021-04-13 09:28:41.883 EEST [984766] LOG:  listening on Unix socket "/tmp/.s.PGSQL.5433"
-    2021-04-13 09:28:41.896 EEST [984767] LOG:  database system was interrupted; last known up at 2021-04-13 09:27:33 EEST
-    2021-04-13 09:28:42.265 EEST [984767] LOG:  database system was not properly shut down; automatic recovery in progress
-    2021-04-13 09:28:42.269 EEST [984767] LOG:  redo starts at 0/15FFB80
-    2021-04-13 09:28:42.272 EEST [984767] LOG:  invalid record length at 0/161F4B0: wanted 24, got 0
-    2021-04-13 09:28:42.272 EEST [984767] LOG:  redo done at 0/161F478 system usage: CPU: user: 0.00 s, system: 0.00 s, elapsed: 0.00 s
-    2021-04-13 09:28:42.321 EEST [984766] LOG:  database system is ready to accept connections
-     done
-    server started
-
-Insert some a row on the 'experimental' branch:
-
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -p5433 -c "select * from foo" 
-                  t              
-    -----------------------------
-     inserted on the main branch
-    (1 row)
-    
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -p5433 -c "insert into foo values ('inserted on experimental')" 
-    INSERT 0 1
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -p5433 -c "select * from foo" 
-                  t              
-    -----------------------------
-     inserted on the main branch
-     inserted on experimental
-    (2 rows)
-    
-See that the other Postgres instance is still running on 'main' branch on port 5432:
-
-
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -p5432 -c "select * from foo" 
-                  t              
-    -----------------------------
-     inserted on the main branch
-    (1 row)
-
-
-
-
-Everything is stored in the .zenith directory:
-
-    ~/git-sandbox/zenith (cli-v2)$ ls -l .zenith/
-    total 12
-    drwxr-xr-x 4 heikki heikki 4096 Apr 13 09:28 datadirs
-    drwxr-xr-x 4 heikki heikki 4096 Apr 13 09:27 refs
-    drwxr-xr-x 4 heikki heikki 4096 Apr 13 09:28 timelines
-
-The 'datadirs' directory contains the datadirs of the running instances:
-
-    ~/git-sandbox/zenith (cli-v2)$ ls -l .zenith/datadirs/
-    total 8
-    drwx------ 18 heikki heikki 4096 Apr 13 09:27 3c0c634c1674079b2c6d4edf7c91523e
-    drwx------ 18 heikki heikki 4096 Apr 13 09:28 697e3c103d4b1763cd6e82e4ff361d76
-    ~/git-sandbox/zenith (cli-v2)$ ls -l .zenith/datadirs/3c0c634c1674079b2c6d4edf7c91523e/
-    total 124
-    drwxr-xr-x 5 heikki heikki  4096 Apr 13 09:27 base
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 global
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_commit_ts
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_dynshmem
-    -rw------- 1 heikki heikki  4760 Apr 13 09:27 pg_hba.conf
-    -rw------- 1 heikki heikki  1636 Apr 13 09:27 pg_ident.conf
-    drwxr-xr-x 4 heikki heikki  4096 Apr 13 09:32 pg_logical
-    drwxr-xr-x 4 heikki heikki  4096 Apr 13 09:27 pg_multixact
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_notify
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_replslot
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_serial
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_snapshots
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_stat
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:34 pg_stat_tmp
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_subtrans
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_tblspc
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_twophase
-    -rw------- 1 heikki heikki     3 Apr 13 09:27 PG_VERSION
-    lrwxrwxrwx 1 heikki heikki    52 Apr 13 09:27 pg_wal -> ../../timelines/3c0c634c1674079b2c6d4edf7c91523e/wal
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_xact
-    -rw------- 1 heikki heikki    88 Apr 13 09:27 postgresql.auto.conf
-    -rw------- 1 heikki heikki 28688 Apr 13 09:27 postgresql.conf
-    -rw------- 1 heikki heikki    96 Apr 13 09:27 postmaster.opts
-    -rw------- 1 heikki heikki   149 Apr 13 09:27 postmaster.pid
-
-Note how 'pg_wal' is just a symlink to the 'timelines' directory. The
-datadir is ephemeral, you can delete it at any time, and it can be reconstructed
-from the snapshots and WAL stored in the 'timelines' directory. So if you push/pull
-the repository, the 'datadirs' are not included. (They are like git working trees)
-
-    ~/git-sandbox/zenith (cli-v2)$ killall -9 postgres
-    ~/git-sandbox/zenith (cli-v2)$ rm -rf .zenith/datadirs/*
-    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli start experimental -- -o -p5433
-    Creating data directory from snapshot at 0/15FFB08...
-    waiting for server to start....2021-04-13 09:37:05.476 EEST [985340] LOG:  starting PostgreSQL 14devel on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit
-    2021-04-13 09:37:05.477 EEST [985340] LOG:  listening on IPv6 address "::1", port 5433
-    2021-04-13 09:37:05.477 EEST [985340] LOG:  listening on IPv4 address "127.0.0.1", port 5433
-    2021-04-13 09:37:05.487 EEST [985340] LOG:  listening on Unix socket "/tmp/.s.PGSQL.5433"
-    2021-04-13 09:37:05.498 EEST [985341] LOG:  database system was interrupted; last known up at 2021-04-13 09:27:33 EEST
-    2021-04-13 09:37:05.808 EEST [985341] LOG:  database system was not properly shut down; automatic recovery in progress
-    2021-04-13 09:37:05.813 EEST [985341] LOG:  redo starts at 0/15FFB80
-    2021-04-13 09:37:05.815 EEST [985341] LOG:  invalid record length at 0/161F770: wanted 24, got 0
-    2021-04-13 09:37:05.815 EEST [985341] LOG:  redo done at 0/161F738 system usage: CPU: user: 0.00 s, system: 0.00 s, elapsed: 0.00 s
-    2021-04-13 09:37:05.866 EEST [985340] LOG:  database system is ready to accept connections
-     done
-    server started
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -p5433 -c "select * from foo" 
-                  t              
-    -----------------------------
-     inserted on the main branch
-     inserted on experimental
-    (2 rows)
-

control_plane/.gitignore

@@ -1 +0,0 @@
-tmp_check/

control_plane/Cargo.toml

@@ -1,28 +0,0 @@
-[package]
-name = "control_plane"
-version = "0.1.0"
-authors = ["Stas Kelvich <stas@zenith.tech>"]
-edition = "2018"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-rand = "0.8.3"
-tar = "0.4.33"
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1"
-toml = "0.5"
-lazy_static = "1.4"
-regex = "1"
-anyhow = "1.0"
-bytes = "1.0.1"
-nix = "0.20"
-url = "2.2.2"
-hex = { version = "0.4.3", features = ["serde"] }
-
-pageserver = { path = "../pageserver" }
-walkeeper = { path = "../walkeeper" }
-postgres_ffi = { path = "../postgres_ffi" }
-zenith_utils = { path = "../zenith_utils" }
-workspace_hack = { path = "../workspace_hack" }

control_plane/src/compute.rs

@@ -1,505 +0,0 @@
-use std::io::Write;
-use std::net::SocketAddr;
-use std::net::TcpStream;
-use std::os::unix::fs::PermissionsExt;
-use std::process::Command;
-use std::sync::Arc;
-use std::time::Duration;
-use std::{collections::BTreeMap, path::PathBuf};
-use std::{
-    fs::{self, File, OpenOptions},
-    io::Read,
-};
-
-use anyhow::{Context, Result};
-use lazy_static::lazy_static;
-use regex::Regex;
-use zenith_utils::connstring::connection_host_port;
-use zenith_utils::postgres_backend::AuthType;
-use zenith_utils::zid::ZTenantId;
-use zenith_utils::zid::ZTimelineId;
-
-use crate::local_env::LocalEnv;
-use crate::storage::PageServerNode;
-
-//
-// ComputeControlPlane
-//
-pub struct ComputeControlPlane {
-    base_port: u16,
-    pageserver: Arc<PageServerNode>,
-    pub nodes: BTreeMap<(ZTenantId, String), Arc<PostgresNode>>,
-    env: LocalEnv,
-}
-
-impl ComputeControlPlane {
-    // Load current nodes with ports from data directories on disk
-    // Directory structure has the following layout:
-    // pgdatadirs
-    // |- tenants
-    // |  |- <tenant_id>
-    // |  |   |- <branch name>
-    pub fn load(env: LocalEnv) -> Result<ComputeControlPlane> {
-        // TODO: since pageserver do not have config file yet we believe here that
-        // it is running on default port. Change that when pageserver will have config.
-        let pageserver = Arc::new(PageServerNode::from_env(&env));
-
-        let mut nodes = BTreeMap::default();
-        let pgdatadirspath = &env.pg_data_dirs_path();
-
-        for tenant_dir in fs::read_dir(&pgdatadirspath)
-            .with_context(|| format!("failed to list {}", pgdatadirspath.display()))?
-        {
-            let tenant_dir = tenant_dir?;
-            for timeline_dir in fs::read_dir(tenant_dir.path())
-                .with_context(|| format!("failed to list {}", tenant_dir.path().display()))?
-            {
-                let node = PostgresNode::from_dir_entry(timeline_dir?, &env, &pageserver)?;
-                nodes.insert((node.tenantid, node.name.clone()), Arc::new(node));
-            }
-        }
-
-        Ok(ComputeControlPlane {
-            base_port: 55431,
-            pageserver,
-            nodes,
-            env,
-        })
-    }
-
-    fn get_port(&mut self) -> u16 {
-        1 + self
-            .nodes
-            .iter()
-            .map(|(_name, node)| node.address.port())
-            .max()
-            .unwrap_or(self.base_port)
-    }
-
-    pub fn local(local_env: &LocalEnv, pageserver: &Arc<PageServerNode>) -> ComputeControlPlane {
-        ComputeControlPlane {
-            base_port: 65431,
-            pageserver: Arc::clone(pageserver),
-            nodes: BTreeMap::new(),
-            env: local_env.clone(),
-        }
-    }
-
-    pub fn new_node(
-        &mut self,
-        tenantid: ZTenantId,
-        branch_name: &str,
-        config_only: bool,
-    ) -> Result<Arc<PostgresNode>> {
-        let timeline_id = self
-            .pageserver
-            .branch_get_by_name(&tenantid, branch_name)?
-            .timeline_id;
-
-        let node = Arc::new(PostgresNode {
-            name: branch_name.to_owned(),
-            address: SocketAddr::new("127.0.0.1".parse().unwrap(), self.get_port()),
-            env: self.env.clone(),
-            pageserver: Arc::clone(&self.pageserver),
-            is_test: false,
-            timelineid: timeline_id,
-            tenantid,
-        });
-
-        node.init_from_page_server(self.env.auth_type, config_only)?;
-        self.nodes
-            .insert((tenantid, node.name.clone()), Arc::clone(&node));
-
-        // Configure the node to stream WAL directly to the pageserver
-        node.append_conf(
-            "postgresql.conf",
-            format!(
-                concat!(
-                    "synchronous_standby_names = 'pageserver'\n", // TODO: add a new function arg?
-                    "zenith.callmemaybe_connstring = '{}'\n",     // FIXME escaping
-                ),
-                node.connstr(),
-            )
-            .as_str(),
-        )?;
-
-        Ok(node)
-    }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-
-#[derive(Debug)]
-pub struct PostgresNode {
-    pub address: SocketAddr,
-    name: String,
-    pub env: LocalEnv,
-    pageserver: Arc<PageServerNode>,
-    is_test: bool,
-    pub timelineid: ZTimelineId,
-    pub tenantid: ZTenantId,
-}
-
-impl PostgresNode {
-    fn from_dir_entry(
-        entry: std::fs::DirEntry,
-        env: &LocalEnv,
-        pageserver: &Arc<PageServerNode>,
-    ) -> Result<PostgresNode> {
-        if !entry.file_type()?.is_dir() {
-            anyhow::bail!(
-                "PostgresNode::from_dir_entry failed: '{}' is not a directory",
-                entry.path().display()
-            );
-        }
-
-        lazy_static! {
-            static ref CONF_PORT_RE: Regex = Regex::new(r"(?m)^\s*port\s*=\s*(\d+)\s*$").unwrap();
-            static ref CONF_TIMELINE_RE: Regex =
-                Regex::new(r"(?m)^\s*zenith.zenith_timeline\s*=\s*'(\w+)'\s*$").unwrap();
-            static ref CONF_TENANT_RE: Regex =
-                Regex::new(r"(?m)^\s*zenith.zenith_tenant\s*=\s*'(\w+)'\s*$").unwrap();
-        }
-
-        // parse data directory name
-        let fname = entry.file_name();
-        let name = fname.to_str().unwrap().to_string();
-
-        // find out tcp port in config file
-        let cfg_path = entry.path().join("postgresql.conf");
-        let config = fs::read_to_string(cfg_path.clone()).with_context(|| {
-            format!(
-                "failed to read config file in {}",
-                cfg_path.to_str().unwrap()
-            )
-        })?;
-
-        // parse port
-        let err_msg = format!(
-            "failed to find port definition in config file {}",
-            cfg_path.to_str().unwrap()
-        );
-        let port: u16 = CONF_PORT_RE
-            .captures(config.as_str())
-            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 1"))?
-            .iter()
-            .last()
-            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 2"))?
-            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 3"))?
-            .as_str()
-            .parse()
-            .with_context(|| err_msg)?;
-
-        // parse timeline
-        let err_msg = format!(
-            "failed to find timeline definition in config file {}",
-            cfg_path.to_str().unwrap()
-        );
-        let timelineid: ZTimelineId = CONF_TIMELINE_RE
-            .captures(config.as_str())
-            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 1"))?
-            .iter()
-            .last()
-            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 2"))?
-            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 3"))?
-            .as_str()
-            .parse()
-            .with_context(|| err_msg)?;
-
-        // parse tenant
-        let err_msg = format!(
-            "failed to find tenant definition in config file {}",
-            cfg_path.to_str().unwrap()
-        );
-        let tenantid = CONF_TENANT_RE
-            .captures(config.as_str())
-            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 1"))?
-            .iter()
-            .last()
-            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 2"))?
-            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 3"))?
-            .as_str()
-            .parse()
-            .with_context(|| err_msg)?;
-
-        // ok now
-        Ok(PostgresNode {
-            address: SocketAddr::new("127.0.0.1".parse().unwrap(), port),
-            name,
-            env: env.clone(),
-            pageserver: Arc::clone(pageserver),
-            is_test: false,
-            timelineid,
-            tenantid,
-        })
-    }
-
-    pub fn do_basebackup(&self) -> Result<()> {
-        let pgdata = self.pgdata();
-
-        let sql = format!("basebackup {} {}", self.tenantid, self.timelineid);
-        let mut client = self
-            .pageserver
-            .page_server_psql_client()
-            .with_context(|| "connecting to page server failed")?;
-
-        let mut copyreader = client
-            .copy_out(sql.as_str())
-            .with_context(|| "page server 'basebackup' command failed")?;
-
-        // FIXME: Currently, we slurp the whole tarball into memory, and then extract it,
-        // but we really should do this:
-        //let mut ar = tar::Archive::new(copyreader);
-        let mut buf = vec![];
-        copyreader
-            .read_to_end(&mut buf)
-            .with_context(|| "reading base backup from page server failed")?;
-        let mut ar = tar::Archive::new(buf.as_slice());
-        ar.unpack(&pgdata)
-            .with_context(|| "extracting page backup failed")?;
-
-        Ok(())
-    }
-
-    // Connect to a page server, get base backup, and untar it to initialize a
-    // new data directory
-    pub fn init_from_page_server(&self, auth_type: AuthType, config_only: bool) -> Result<()> {
-        let pgdata = self.pgdata();
-
-        println!(
-            "Extracting base backup to create postgres instance: path={} port={}",
-            pgdata.display(),
-            self.address.port()
-        );
-
-        // initialize data directory
-        if self.is_test {
-            fs::remove_dir_all(&pgdata).ok();
-        }
-
-        fs::create_dir_all(&pgdata)
-            .with_context(|| format!("could not create data directory {}", pgdata.display()))?;
-        fs::set_permissions(pgdata.as_path(), fs::Permissions::from_mode(0o700)).with_context(
-            || {
-                format!(
-                    "could not set permissions in data directory {}",
-                    pgdata.display()
-                )
-            },
-        )?;
-
-        if config_only {
-            //Just create an empty config file
-            File::create(self.pgdata().join("postgresql.conf").to_str().unwrap())?;
-        } else {
-            self.do_basebackup()?;
-            fs::create_dir_all(self.pgdata().join("pg_wal"))?;
-            fs::create_dir_all(self.pgdata().join("pg_wal").join("archive_status"))?;
-        }
-
-        // wal_log_hints is mandatory when running against pageserver (see gh issue#192)
-        // TODO: is it possible to check wal_log_hints at pageserver side via XLOG_PARAMETER_CHANGE?
-        self.append_conf(
-            "postgresql.conf",
-            &format!(
-                "max_wal_senders = 10\n\
-                 wal_log_hints = on\n\
-                 max_replication_slots = 10\n\
-                 hot_standby = on\n\
-                 shared_buffers = 1MB\n\
-                 fsync = off\n\
-                 max_connections = 100\n\
-                 wal_sender_timeout = 0\n\
-                 wal_level = replica\n\
-                 listen_addresses = '{address}'\n\
-                 port = {port}\n",
-                address = self.address.ip(),
-                port = self.address.port()
-            ),
-        )?;
-
-        // Never clean up old WAL. TODO: We should use a replication
-        // slot or something proper, to prevent the compute node
-        // from removing WAL that hasn't been streamed to the safekeeper or
-        // page server yet. (gh issue #349)
-        self.append_conf("postgresql.conf", "wal_keep_size='10TB'\n")?;
-
-        // set up authentication
-        let password = if let AuthType::ZenithJWT = auth_type {
-            "$ZENITH_AUTH_TOKEN"
-        } else {
-            ""
-        };
-
-        // Configure that node to take pages from pageserver
-        let (host, port) = connection_host_port(&self.pageserver.connection_config);
-        self.append_conf(
-            "postgresql.conf",
-            format!(
-                concat!(
-                    "shared_preload_libraries = zenith\n",
-                    // $ZENITH_AUTH_TOKEN will be replaced with value from environment variable during compute pg startup
-                    // it is done this way because otherwise user will be able to retrieve the value using SHOW command or pg_settings
-                    "zenith.page_server_connstring = 'host={} port={} password={}'\n",
-                    "zenith.zenith_timeline='{}'\n",
-                    "zenith.zenith_tenant='{}'\n",
-                ),
-                host, port, password, self.timelineid, self.tenantid,
-            )
-            .as_str(),
-        )?;
-
-        Ok(())
-    }
-
-    pub fn pgdata(&self) -> PathBuf {
-        self.env.pg_data_dir(&self.tenantid, &self.name)
-    }
-
-    pub fn status(&self) -> &str {
-        let timeout = Duration::from_millis(300);
-        let has_pidfile = self.pgdata().join("postmaster.pid").exists();
-        let can_connect = TcpStream::connect_timeout(&self.address, timeout).is_ok();
-
-        match (has_pidfile, can_connect) {
-            (true, true) => "running",
-            (false, false) => "stopped",
-            (true, false) => "crashed",
-            (false, true) => "running, no pidfile",
-        }
-    }
-
-    pub fn append_conf(&self, config: &str, opts: &str) -> Result<()> {
-        OpenOptions::new()
-            .append(true)
-            .open(self.pgdata().join(config).to_str().unwrap())?
-            .write_all(opts.as_bytes())?;
-        Ok(())
-    }
-
-    fn pg_ctl(&self, args: &[&str], auth_token: &Option<String>) -> Result<()> {
-        let pg_ctl_path = self.env.pg_bin_dir().join("pg_ctl");
-        let mut cmd = Command::new(pg_ctl_path);
-        cmd.args(
-            [
-                &[
-                    "-D",
-                    self.pgdata().to_str().unwrap(),
-                    "-l",
-                    self.pgdata().join("pg.log").to_str().unwrap(),
-                    "-w", //wait till pg_ctl actually does what was asked
-                ],
-                args,
-            ]
-            .concat(),
-        )
-        .env_clear()
-        .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-        .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap());
-
-        if let Some(token) = auth_token {
-            cmd.env("ZENITH_AUTH_TOKEN", token);
-        }
-        let pg_ctl = cmd.status().with_context(|| "pg_ctl failed")?;
-
-        if !pg_ctl.success() {
-            anyhow::bail!("pg_ctl failed");
-        }
-        Ok(())
-    }
-
-    pub fn start(&self, auth_token: &Option<String>) -> Result<()> {
-        // Bail if the node already running.
-        if self.status() == "running" {
-            anyhow::bail!("The node is already running");
-        }
-
-        // 1. We always start compute node from scratch, so
-        // if old dir exists, preserve config files and drop the directory
-
-        // XXX Now we only use 'postgresql.conf'.
-        // If we will need 'pg_hba.conf', support it here too
-
-        let postgresql_conf_path = self.pgdata().join("postgresql.conf");
-        let postgresql_conf = fs::read(postgresql_conf_path.clone()).with_context(|| {
-            format!(
-                "failed to read config file in {}",
-                postgresql_conf_path.to_str().unwrap()
-            )
-        })?;
-
-        println!(
-            "Destroying postgres data directory '{}'",
-            self.pgdata().to_str().unwrap()
-        );
-        fs::remove_dir_all(&self.pgdata())?;
-
-        // 2. Create new node
-        self.init_from_page_server(self.env.auth_type, false)?;
-
-        // 3. Bring back config files
-
-        if let Ok(mut file) = OpenOptions::new()
-            .append(false)
-            .write(true)
-            .open(&postgresql_conf_path)
-        {
-            file.write_all(&postgresql_conf)?;
-            file.sync_all()?;
-        }
-
-        // 4. Finally start the compute node postgres
-        println!("Starting postgres node at '{}'", self.connstr());
-        self.pg_ctl(&["start"], auth_token)
-    }
-
-    pub fn restart(&self, auth_token: &Option<String>) -> Result<()> {
-        self.pg_ctl(&["restart"], auth_token)
-    }
-
-    pub fn stop(&self, destroy: bool) -> Result<()> {
-        self.pg_ctl(&["-m", "immediate", "stop"], &None)?;
-        if destroy {
-            println!(
-                "Destroying postgres data directory '{}'",
-                self.pgdata().to_str().unwrap()
-            );
-            fs::remove_dir_all(&self.pgdata())?;
-        }
-        Ok(())
-    }
-
-    pub fn connstr(&self) -> String {
-        format!(
-            "host={} port={} user={} dbname={}",
-            self.address.ip(),
-            self.address.port(),
-            "zenith_admin",
-            "postgres"
-        )
-    }
-
-    // XXX: cache that in control plane
-    pub fn whoami(&self) -> String {
-        let output = Command::new("whoami")
-            .output()
-            .expect("failed to execute whoami");
-
-        if !output.status.success() {
-            panic!("whoami failed");
-        }
-
-        String::from_utf8(output.stdout).unwrap().trim().to_string()
-    }
-}
-
-impl Drop for PostgresNode {
-    // destructor to clean up state after test is done
-    // XXX: we may detect failed test by setting some flag in catch_unwind()
-    // and checking it here. But let just clean datadirs on start.
-    fn drop(&mut self) {
-        if self.is_test {
-            let _ = self.stop(true);
-        }
-    }
-}

control_plane/src/lib.rs

@@ -1,31 +0,0 @@
-//
-// Local control plane.
-//
-// Can start, configure and stop postgres instances running as a local processes.
-//
-// Intended to be used in integration tests and in CLI tools for
-// local installations.
-//
-use anyhow::{anyhow, bail, Context, Result};
-use std::fs;
-use std::path::Path;
-
-pub mod compute;
-pub mod local_env;
-pub mod storage;
-
-/// Read a PID file
-///
-/// We expect a file that contains a single integer.
-/// We return an i32 for compatibility with libc and nix.
-pub fn read_pidfile(pidfile: &Path) -> Result<i32> {
-    let pid_str = fs::read_to_string(pidfile)
-        .with_context(|| format!("failed to read pidfile {:?}", pidfile))?;
-    let pid: i32 = pid_str
-        .parse()
-        .map_err(|_| anyhow!("failed to parse pidfile {:?}", pidfile))?;
-    if pid < 1 {
-        bail!("pidfile {:?} contained bad value '{}'", pidfile, pid);
-    }
-    Ok(pid)
-}

control_plane/src/local_env.rs

@@ -1,242 +0,0 @@
-//
-// This module is responsible for locating and loading paths in a local setup.
-//
-// Now it also provides init method which acts like a stub for proper installation
-// script which will use local paths.
-//
-use anyhow::{anyhow, Context, Result};
-use hex;
-use serde::{Deserialize, Serialize};
-use std::fs;
-use std::path::PathBuf;
-use std::process::{Command, Stdio};
-use std::{collections::BTreeMap, env};
-use url::Url;
-use zenith_utils::auth::{encode_from_key_path, Claims, Scope};
-use zenith_utils::postgres_backend::AuthType;
-use zenith_utils::zid::ZTenantId;
-
-pub type Remotes = BTreeMap<String, String>;
-
-//
-// This data structures represent deserialized zenith CLI config
-//
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct LocalEnv {
-    // Pageserver connection strings
-    pub pageserver_connstring: String,
-
-    // Base directory for both pageserver and compute nodes
-    pub base_data_dir: PathBuf,
-
-    // Path to postgres distribution. It's expected that "bin", "include",
-    // "lib", "share" from postgres distribution are there. If at some point
-    // in time we will be able to run against vanilla postgres we may split that
-    // to four separate paths and match OS-specific installation layout.
-    pub pg_distrib_dir: PathBuf,
-
-    // Path to pageserver binary. Empty for remote pageserver.
-    pub zenith_distrib_dir: Option<PathBuf>,
-
-    // keeping tenant id in config to reduce copy paste when running zenith locally with single tenant
-    #[serde(with = "hex")]
-    pub tenantid: ZTenantId,
-
-    // Repository format, 'rocksdb' or 'layered' or None for default
-    pub repository_format: Option<String>,
-
-    // jwt auth token used for communication with pageserver
-    pub auth_token: String,
-
-    // used to determine which auth type is used
-    pub auth_type: AuthType,
-
-    // used to issue tokens during e.g pg start
-    pub private_key_path: PathBuf,
-
-    pub remotes: Remotes,
-}
-
-impl LocalEnv {
-    // postgres installation paths
-    pub fn pg_bin_dir(&self) -> PathBuf {
-        self.pg_distrib_dir.join("bin")
-    }
-    pub fn pg_lib_dir(&self) -> PathBuf {
-        self.pg_distrib_dir.join("lib")
-    }
-
-    pub fn pageserver_bin(&self) -> Result<PathBuf> {
-        Ok(self
-            .zenith_distrib_dir
-            .as_ref()
-            .ok_or_else(|| anyhow!("Can not manage remote pageserver"))?
-            .join("pageserver"))
-    }
-
-    pub fn pg_data_dirs_path(&self) -> PathBuf {
-        self.base_data_dir.join("pgdatadirs").join("tenants")
-    }
-
-    pub fn pg_data_dir(&self, tenantid: &ZTenantId, branch_name: &str) -> PathBuf {
-        self.pg_data_dirs_path()
-            .join(tenantid.to_string())
-            .join(branch_name)
-    }
-
-    // TODO: move pageserver files into ./pageserver
-    pub fn pageserver_data_dir(&self) -> PathBuf {
-        self.base_data_dir.clone()
-    }
-}
-
-fn base_path() -> PathBuf {
-    match std::env::var_os("ZENITH_REPO_DIR") {
-        Some(val) => PathBuf::from(val.to_str().unwrap()),
-        None => ".zenith".into(),
-    }
-}
-
-//
-// Initialize a new Zenith repository
-//
-pub fn init(
-    remote_pageserver: Option<&str>,
-    tenantid: ZTenantId,
-    auth_type: AuthType,
-    repository_format: Option<&str>,
-) -> Result<()> {
-    // check if config already exists
-    let base_path = base_path();
-    if base_path.exists() {
-        anyhow::bail!(
-            "{} already exists. Perhaps already initialized?",
-            base_path.to_str().unwrap()
-        );
-    }
-    fs::create_dir(&base_path)?;
-
-    // ok, now check that expected binaries are present
-
-    // Find postgres binaries. Follow POSTGRES_DISTRIB_DIR if set, otherwise look in "tmp_install".
-    let pg_distrib_dir: PathBuf = {
-        if let Some(postgres_bin) = env::var_os("POSTGRES_DISTRIB_DIR") {
-            postgres_bin.into()
-        } else {
-            let cwd = env::current_dir()?;
-            cwd.join("tmp_install")
-        }
-    };
-    if !pg_distrib_dir.join("bin/postgres").exists() {
-        anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
-    }
-
-    // generate keys for jwt
-    // openssl genrsa -out private_key.pem 2048
-    let private_key_path = base_path.join("auth_private_key.pem");
-    let keygen_output = Command::new("openssl")
-        .arg("genrsa")
-        .args(&["-out", private_key_path.to_str().unwrap()])
-        .arg("2048")
-        .stdout(Stdio::null())
-        .output()
-        .with_context(|| "failed to generate auth private key")?;
-    if !keygen_output.status.success() {
-        anyhow::bail!(
-            "openssl failed: '{}'",
-            String::from_utf8_lossy(&keygen_output.stderr)
-        );
-    }
-
-    let public_key_path = base_path.join("auth_public_key.pem");
-    // openssl rsa -in private_key.pem -pubout -outform PEM -out public_key.pem
-    let keygen_output = Command::new("openssl")
-        .arg("rsa")
-        .args(&["-in", private_key_path.to_str().unwrap()])
-        .arg("-pubout")
-        .args(&["-outform", "PEM"])
-        .args(&["-out", public_key_path.to_str().unwrap()])
-        .stdout(Stdio::null())
-        .output()
-        .with_context(|| "failed to generate auth private key")?;
-    if !keygen_output.status.success() {
-        anyhow::bail!(
-            "openssl failed: '{}'",
-            String::from_utf8_lossy(&keygen_output.stderr)
-        );
-    }
-
-    let auth_token =
-        encode_from_key_path(&Claims::new(None, Scope::PageServerApi), &private_key_path)?;
-
-    let conf = if let Some(addr) = remote_pageserver {
-        // check that addr is parsable
-        let _uri = Url::parse(addr).map_err(|e| anyhow!("{}: {}", addr, e))?;
-
-        LocalEnv {
-            pageserver_connstring: format!("postgresql://{}/", addr),
-            pg_distrib_dir,
-            zenith_distrib_dir: None,
-            base_data_dir: base_path,
-            remotes: BTreeMap::default(),
-            tenantid,
-            repository_format: repository_format.map(|x| x.into()),
-            auth_token,
-            auth_type,
-            private_key_path,
-        }
-    } else {
-        // Find zenith binaries.
-        let zenith_distrib_dir = env::current_exe()?.parent().unwrap().to_owned();
-        if !zenith_distrib_dir.join("pageserver").exists() {
-            anyhow::bail!("Can't find pageserver binary.",);
-        }
-
-        LocalEnv {
-            pageserver_connstring: "postgresql://127.0.0.1:6400".to_string(),
-            pg_distrib_dir,
-            zenith_distrib_dir: Some(zenith_distrib_dir),
-            base_data_dir: base_path,
-            remotes: BTreeMap::default(),
-            tenantid,
-            repository_format: repository_format.map(|x| x.into()),
-            auth_token,
-            auth_type,
-            private_key_path,
-        }
-    };
-
-    fs::create_dir_all(conf.pg_data_dirs_path())?;
-
-    let toml = toml::to_string_pretty(&conf)?;
-    fs::write(conf.base_data_dir.join("config"), toml)?;
-
-    Ok(())
-}
-
-// Locate and load config
-pub fn load_config() -> Result<LocalEnv> {
-    let repopath = base_path();
-
-    if !repopath.exists() {
-        anyhow::bail!(
-            "Zenith config is not found in {}. You need to run 'zenith init' first",
-            repopath.to_str().unwrap()
-        );
-    }
-
-    // TODO: check that it looks like a zenith repository
-
-    // load and parse file
-    let config = fs::read_to_string(repopath.join("config"))?;
-    toml::from_str(config.as_str()).map_err(|e| e.into())
-}
-
-// Save config. We use that to change set of remotes from CLI itself.
-pub fn save_config(conf: &LocalEnv) -> Result<()> {
-    let config_path = base_path().join("config");
-    let conf_str = toml::to_string_pretty(conf)?;
-
-    fs::write(config_path, conf_str)?;
-    Ok(())
-}

control_plane/src/storage.rs

@@ -1,267 +0,0 @@
-use std::collections::HashMap;
-use std::net::TcpStream;
-use std::path::PathBuf;
-use std::process::Command;
-use std::thread;
-use std::time::Duration;
-
-use anyhow::{anyhow, bail, Result};
-use nix::sys::signal::{kill, Signal};
-use nix::unistd::Pid;
-use postgres::{Config, NoTls};
-use zenith_utils::postgres_backend::AuthType;
-use zenith_utils::zid::ZTenantId;
-
-use crate::local_env::LocalEnv;
-use crate::read_pidfile;
-use pageserver::branches::BranchInfo;
-use zenith_utils::connstring::connection_address;
-
-//
-// Control routines for pageserver.
-//
-// Used in CLI and tests.
-//
-#[derive(Debug)]
-pub struct PageServerNode {
-    pub kill_on_exit: bool,
-    pub connection_config: Config,
-    pub env: LocalEnv,
-}
-
-impl PageServerNode {
-    pub fn from_env(env: &LocalEnv) -> PageServerNode {
-        let password = if matches!(env.auth_type, AuthType::ZenithJWT) {
-            &env.auth_token
-        } else {
-            ""
-        };
-
-        PageServerNode {
-            kill_on_exit: false,
-            connection_config: Self::default_config(password), // default
-            env: env.clone(),
-        }
-    }
-
-    fn default_config(password: &str) -> Config {
-        format!("postgresql://no_user:{}@localhost:64000/no_db", password)
-            .parse()
-            .unwrap()
-    }
-
-    pub fn init(
-        &self,
-        create_tenant: Option<&str>,
-        enable_auth: bool,
-        repository_format: Option<&str>,
-    ) -> Result<()> {
-        let mut cmd = Command::new(self.env.pageserver_bin()?);
-        let mut args = vec![
-            "--init",
-            "-D",
-            self.env.base_data_dir.to_str().unwrap(),
-            "--postgres-distrib",
-            self.env.pg_distrib_dir.to_str().unwrap(),
-        ];
-
-        if enable_auth {
-            args.extend(&["--auth-validation-public-key-path", "auth_public_key.pem"]);
-            args.extend(&["--auth-type", "ZenithJWT"]);
-        }
-
-        if let Some(repo_format) = repository_format {
-            args.extend(&["--repository-format", repo_format]);
-        }
-
-        create_tenant.map(|tenantid| args.extend(&["--create-tenant", tenantid]));
-        let status = cmd
-            .args(args)
-            .env_clear()
-            .env("RUST_BACKTRACE", "1")
-            .status()
-            .expect("pageserver init failed");
-
-        if status.success() {
-            Ok(())
-        } else {
-            Err(anyhow!("pageserver init failed"))
-        }
-    }
-
-    pub fn repo_path(&self) -> PathBuf {
-        self.env.pageserver_data_dir()
-    }
-
-    pub fn pid_file(&self) -> PathBuf {
-        self.repo_path().join("pageserver.pid")
-    }
-
-    pub fn start(&self) -> Result<()> {
-        println!(
-            "Starting pageserver at '{}' in {}",
-            connection_address(&self.connection_config),
-            self.repo_path().display()
-        );
-
-        let mut cmd = Command::new(self.env.pageserver_bin()?);
-        cmd.args(&["-D", self.repo_path().to_str().unwrap()])
-            .arg("-d")
-            .env_clear()
-            .env("RUST_BACKTRACE", "1");
-
-        if !cmd.status()?.success() {
-            bail!(
-                "Pageserver failed to start. See '{}' for details.",
-                self.repo_path().join("pageserver.log").display()
-            );
-        }
-
-        // It takes a while for the page server to start up. Wait until it is
-        // open for business.
-        for retries in 1..15 {
-            match self.page_server_psql_client() {
-                Ok(_) => {
-                    println!("Pageserver started");
-                    return Ok(());
-                }
-                Err(err) => {
-                    println!(
-                        "Pageserver not responding yet, err {} retrying ({})...",
-                        err, retries
-                    );
-                    thread::sleep(Duration::from_secs(1));
-                }
-            }
-        }
-        bail!("pageserver failed to start");
-    }
-
-    pub fn stop(&self) -> Result<()> {
-        let pid = read_pidfile(&self.pid_file())?;
-        let pid = Pid::from_raw(pid);
-        if kill(pid, Signal::SIGTERM).is_err() {
-            bail!("Failed to kill pageserver with pid {}", pid);
-        }
-
-        // wait for pageserver stop
-        let address = connection_address(&self.connection_config);
-        for _ in 0..5 {
-            let stream = TcpStream::connect(&address);
-            thread::sleep(Duration::from_secs(1));
-            if let Err(_e) = stream {
-                println!("Pageserver stopped");
-                return Ok(());
-            }
-            println!("Stopping pageserver on {}", address);
-        }
-
-        bail!("Failed to stop pageserver with pid {}", pid);
-    }
-
-    pub fn page_server_psql(&self, sql: &str) -> Vec<postgres::SimpleQueryMessage> {
-        let mut client = self.connection_config.connect(NoTls).unwrap();
-
-        println!("Pageserver query: '{}'", sql);
-        client.simple_query(sql).unwrap()
-    }
-
-    pub fn page_server_psql_client(&self) -> Result<postgres::Client, postgres::Error> {
-        self.connection_config.connect(NoTls)
-    }
-
-    pub fn tenants_list(&self) -> Result<Vec<String>> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query("tenant_list")?;
-        let tenants_json = query_result
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
-            })
-            .flatten()
-            .ok_or_else(|| anyhow!("missing tenants"))?;
-
-        Ok(serde_json::from_str(tenants_json)?)
-    }
-
-    pub fn tenant_create(&self, tenantid: &ZTenantId) -> Result<()> {
-        let mut client = self.page_server_psql_client()?;
-        client.simple_query(format!("tenant_create {}", tenantid).as_str())?;
-        Ok(())
-    }
-
-    pub fn branches_list(&self, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query(&format!("branch_list {}", tenantid))?;
-        let branches_json = query_result
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
-            })
-            .flatten()
-            .ok_or_else(|| anyhow!("missing branches"))?;
-
-        Ok(serde_json::from_str(branches_json)?)
-    }
-
-    pub fn branch_create(
-        &self,
-        branch_name: &str,
-        startpoint: &str,
-        tenantid: &ZTenantId,
-    ) -> Result<BranchInfo> {
-        let mut client = self.page_server_psql_client()?;
-        let query_result = client.simple_query(
-            format!("branch_create {} {} {}", tenantid, branch_name, startpoint).as_str(),
-        )?;
-
-        let branch_json = query_result
-            .first()
-            .map(|msg| match msg {
-                postgres::SimpleQueryMessage::Row(row) => row.get(0),
-                _ => None,
-            })
-            .flatten()
-            .ok_or_else(|| anyhow!("missing branch"))?;
-
-        let res: BranchInfo = serde_json::from_str(branch_json).map_err(|e| {
-            anyhow!(
-                "failed to parse branch_create response: {}: {}",
-                branch_json,
-                e
-            )
-        })?;
-
-        Ok(res)
-    }
-
-    // TODO: make this a separate request type and avoid loading all the branches
-    pub fn branch_get_by_name(
-        &self,
-        tenantid: &ZTenantId,
-        branch_name: &str,
-    ) -> Result<BranchInfo> {
-        let branch_infos = self.branches_list(tenantid)?;
-        let branche_by_name: Result<HashMap<String, BranchInfo>> = branch_infos
-            .into_iter()
-            .map(|branch_info| Ok((branch_info.name.clone(), branch_info)))
-            .collect();
-        let branche_by_name = branche_by_name?;
-
-        let branch = branche_by_name
-            .get(branch_name)
-            .ok_or_else(|| anyhow!("Branch {} not found", branch_name))?;
-
-        Ok(branch.clone())
-    }
-}
-
-impl Drop for PageServerNode {
-    fn drop(&mut self) {
-        if self.kill_on_exit {
-            let _ = self.stop();
-        }
-    }
-}

docker-entrypoint.sh

@@ -1,11 +0,0 @@
-#!/bin/sh
-if [ "$1" = 'pageserver' ]; then
-    if [ ! -d "/data/tenants" ]; then
-        echo "Initializing pageserver data directory"
-        pageserver --init -D /data --postgres-distrib /usr/local
-    fi
-    echo "Staring pageserver at 0.0.0.0:6400"
-    pageserver -l 0.0.0.0:6400 -D /data
-else
-    "$@"
-fi

docs/README.md

@@ -1,11 +0,0 @@
-# Zenith documentation
-
-## Table of contents
-
-- [authentication.md](authentication.md) — pageserver JWT authentication.
-- [docker.md](docker.md) — Docker images and building pipeline.
-- [multitenancy.md](multitenancy.md) — how multitenancy is organized in the pageserver and Zenith CLI.
-- [pageserver/README](/pageserver/README) — pageserver overview.
-- [postgres_ffi/README](/postgres_ffi/README) — Postgres FFI overview.
-- [test_runner/README.md](/test_runner/README.md) — tests infrastructure overview.
-- [walkeeper/README](/walkeeper/README.md) — WAL service overview.

docs/authentication.md

@@ -1,30 +0,0 @@
-## Authentication
-
-### Overview
-
-Current state of authentication includes usage of JWT tokens in communication between compute and pageserver and between CLI and pageserver. JWT token is signed using RSA keys. CLI generates a key pair during call to `zenith init`. Using following openssl commands:
-
-```bash
-openssl genrsa -out private_key.pem 2048
-openssl rsa -in private_key.pem -pubout -outform PEM -out public_key.pem
-```
-
-CLI also generates signed token and saves it in the config for later access to pageserver. Now authentication is optional. Pageserver has two variables in config: `auth_validation_public_key_path` and `auth_type`, so when auth type present and set to `ZenithJWT` pageserver will require authentication for connections. Actual JWT is passed in password field of connection string. There is a caveat for psql, it silently truncates passwords to 100 symbols, so to correctly pass JWT via psql you have to either use PGPASSWORD environment variable, or store password in psql config file.
-
-Currently there is no authentication between compute and safekeepers, because this communication layer is under heavy refactoring. After this refactoring support for authentication will be added there too. Now safekeeper supports "hardcoded" token passed via environment variable to be able to use callmemaybe command in pageserver.
-
-Compute uses token passed via environment variable to communicate to pageserver and in the future to the safekeeper too.
-
-JWT authentication now supports two scopes: tenant and pageserverapi. Tenant scope is intended for use in tenant related api calls, e.g. create_branch. Compute launched for particular tenant also uses this scope. Scope pageserver api is intended to be used by console to manage pageserver. For now we have only one management operation - create tenant.
-
-Examples for token generation in python:
-
-```python
-# generate pageserverapi token
-management_token = jwt.encode({"scope": "pageserverapi"}, auth_keys.priv, algorithm="RS256")
-
-# generate tenant token
-tenant_token = jwt.encode({"scope": "tenant", "tenant_id": ps.initial_tenant}, auth_keys.priv, algorithm="RS256")
-```
-
-Utility functions to work with jwts in rust are located in zenith_utils/src/auth.rs

docs/docker.md

@@ -1,38 +0,0 @@
-# Docker images of Zenith
-
-## Images
-
-Currently we build two main images:
-
-- [zenithdb/zenith](https://hub.docker.com/repository/docker/zenithdb/zenith) — image with pre-built `pageserver`, `wal_acceptor` and `proxy` binaries and all the required runtime dependencies. Built from [/Dockerfile](/Dockerfile).
-- [zenithdb/compute-node](https://hub.docker.com/repository/docker/zenithdb/compute-node) — compute node image with pre-built Postgres binaries from [zenithdb/postgres](https://github.com/zenithdb/postgres).
-
-And two intermediate images used either to reduce build time or to deliver some additional binary tools from other repos:
-
-- [zenithdb/build](https://hub.docker.com/repository/docker/zenithdb/build) — image with all the dependencies required to build Zenith and compute node images. This image is based on `rust:slim-buster`, so it also has a proper `rust` environment. Built from [/Dockerfile.build](/Dockerfile.build).
-- [zenithdb/compute-tools](https://hub.docker.com/repository/docker/zenithdb/compute-tools) — compute node configuration management tools.
-
-## Building pipeline
-
-1. Image `zenithdb/compute-tools` is re-built automatically.
-
-2. Image `zenithdb/build` is built manually. If you want to introduce any new compile time dependencies to Zenith or compute node you have to update this image as well, build it and push to Docker Hub.
-
-Build:
-```sh
-docker build -t zenithdb/build:buster -f Dockerfile.build .
-```
-
-Login:
-```sh
-docker login
-```
-
-Push to Docker Hub:
-```sh
-docker push zenithdb/build:buster
-```
-
-3. Image `zenithdb/compute-node` is built independently in the [zenithdb/postgres](https://github.com/zenithdb/postgres) repo.
-
-4. Image `zenithdb/zenith` is built in this repo after a successful `release` tests run and pushed to Docker Hub automatically.

docs/multitenancy.md

@@ -1,59 +0,0 @@
-## Multitenancy
-
-### Overview
-
-Zenith supports multitenancy. One pageserver can serve multiple tenants at once. Tenants can be managed via zenith CLI. During page server setup tenant can be created using ```zenith init --create-tenant``` Also tenants can be added into the system on the fly without pageserver restart. This can be done using the following cli command: ```zenith tenant create``` Tenants use random identifiers which can be represented as a 32 symbols hexadecimal string. So zenith tenant create accepts desired tenant id as an optional argument. The concept of timelines/branches is working independently per tenant.
-
-### Tenants in other commands
-
-By default during `zenith init` new tenant is created on the pageserver. Newly created tenant's id is saved to cli config, so other commands can use it automatically if no direct arugment `--tenantid=<tenantid>` is provided. So generally tenantid more frequently appears in internal pageserver interface. Its commands take tenantid argument to distinguish to which tenant operation should be applied. CLI support creation of new tenants.
-
-Examples for cli:
-
-```sh
-zenith tenant list
-
-zenith tenant create // generates new id
-
-zenith tenant create ee6016ec31116c1b7c33dfdfca38892f
-
-zenith pg create main // default tenant from zenith init
-
-zenith pg create main --tenantid=ee6016ec31116c1b7c33dfdfca38892f
-
-zenith branch --tenantid=ee6016ec31116c1b7c33dfdfca38892f
-```
-
-### Data layout
-
-On the page server tenants introduce one level of indirection, so data directory structured the following way:
-```
-<pageserver working directory>
-├── pageserver.log
-├── pageserver.pid
-├── pageserver.toml
-└── tenants
-   ├── 537cffa58a4fa557e49e19951b5a9d6b
-   ├── de182bc61fb11a5a6b390a8aed3a804a
-   └── ee6016ec31116c1b7c33dfdfca38891f
-```
-Wal redo activity, timelines, snapshots are managed for each tenant independently.
-
-For local environment used for example in tests there also new level of indirection for tenants. It touches `pgdatadirs` directory. Now it contains `tenants` subdirectory so the structure looks the following way:
-
-```
-pgdatadirs
-└── tenants
-   ├── de182bc61fb11a5a6b390a8aed3a804a
-   │  └── main
-   └── ee6016ec31116c1b7c33dfdfca38892f
-      └── main
-```
-
-### Changes to postgres
-
-Tenant id is passed to postgres via GUC the same way as the timeline. Tenant id is added to commands issued to pageserver, namely: pagestream, callmemaybe. Tenant id is also exists in ServerInfo structure, this is needed to pass the value to wal receiver to be able to forward it to the pageserver.
-
-### Safety
-
-For now particular tenant can only appear on a particular pageserver. Set of WAL acceptors are also pinned to particular (tenantid, timeline) pair so there can only be one writer for particular (tenantid, timeline).

integration_tests/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "integration_tests"
+version = "0.1.0"
+authors = ["Stas Kelvich <stas@zenith.tech>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+lazy_static = "1.4.0"
+rand = "0.8.3"
+postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" }
+tokio-postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" }
+
+pageserver = { path = "../pageserver" }
+walkeeper = { path = "../walkeeper" }

integration_tests/tests/control_plane/mod.rs

@@ -0,0 +1,844 @@
+//
+// Local control plane.
+//
+// Can start, cofigure and stop postgres instances running as a local processes.
+//
+// Intended to be used in integration tests and in CLI tools for
+// local installations.
+//
+
+use std::fs::File;
+use std::fs::{self, OpenOptions};
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::str;
+use std::sync::Arc;
+use std::{
+    io::Write,
+    net::{IpAddr, Ipv4Addr, SocketAddr},
+};
+
+use lazy_static::lazy_static;
+use postgres::{Client, NoTls};
+
+use postgres;
+
+lazy_static! {
+    // postgres would be there if it was build by 'make postgres' here in the repo
+    pub static ref PG_BIN_DIR : PathBuf = Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("../tmp_install/bin");
+    pub static ref PG_LIB_DIR : PathBuf = Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("../tmp_install/lib");
+
+    pub static ref BIN_DIR : PathBuf = cargo_bin_dir();
+
+    pub static ref TEST_WORKDIR : PathBuf = Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("tmp_check");
+}
+
+// Find the directory where the binaries were put (i.e. target/debug/)
+pub fn cargo_bin_dir() -> PathBuf {
+    let mut pathbuf = std::env::current_exe().ok().unwrap();
+
+    pathbuf.pop();
+    if pathbuf.ends_with("deps") {
+        pathbuf.pop();
+    }
+
+    return pathbuf;
+}
+
+//
+// I'm intendedly modelling storage and compute control planes as a separate entities
+// as it is closer to the actual setup.
+//
+pub struct StorageControlPlane {
+    pub wal_acceptors: Vec<WalAcceptorNode>,
+    pub page_servers: Vec<PageServerNode>,
+}
+
+impl StorageControlPlane {
+    // postgres <-> page_server
+    pub fn one_page_server(froms3: bool) -> StorageControlPlane {
+        let mut cplane = StorageControlPlane {
+            wal_acceptors: Vec::new(),
+            page_servers: Vec::new(),
+        };
+
+        let pserver = PageServerNode {
+            page_service_addr: "127.0.0.1:65200".parse().unwrap(),
+            data_dir: TEST_WORKDIR.join("pageserver"),
+        };
+        pserver.init();
+        if froms3 {
+            pserver.start_froms3();
+        } else {
+            pserver.start();
+        }
+
+        cplane.page_servers.push(pserver);
+        cplane
+    }
+
+    pub fn fault_tolerant(redundancy: usize) -> StorageControlPlane {
+        let mut cplane = StorageControlPlane {
+            wal_acceptors: Vec::new(),
+            page_servers: Vec::new(),
+        };
+        const WAL_ACCEPTOR_PORT: usize = 54321;
+
+        for i in 0..redundancy {
+            let wal_acceptor = WalAcceptorNode {
+                listen: format!("127.0.0.1:{}", WAL_ACCEPTOR_PORT + i)
+                    .parse()
+                    .unwrap(),
+                data_dir: TEST_WORKDIR.join(format!("wal_acceptor_{}", i)),
+            };
+            wal_acceptor.init();
+            wal_acceptor.start();
+            cplane.wal_acceptors.push(wal_acceptor);
+        }
+        cplane
+    }
+
+    pub fn stop(&self) {
+        for wa in self.wal_acceptors.iter() {
+            wa.stop();
+        }
+    }
+
+    // // postgres <-> wal_acceptor x3 <-> page_server
+    // fn local(&mut self) -> StorageControlPlane {
+    // }
+
+    pub fn page_server_addr(&self) -> &SocketAddr {
+        &self.page_servers[0].page_service_addr
+    }
+
+    pub fn get_wal_acceptor_conn_info(&self) -> String {
+        self.wal_acceptors
+            .iter()
+            .map(|wa| wa.listen.to_string().to_string())
+            .collect::<Vec<String>>()
+            .join(",")
+    }
+
+    pub fn page_server_psql(&self, sql: &str) -> Vec<postgres::SimpleQueryMessage> {
+        let addr = &self.page_servers[0].page_service_addr;
+
+        let connstring = format!(
+            "host={} port={} dbname={} user={}",
+            addr.ip(),
+            addr.port(),
+            "no_db",
+            "no_user",
+        );
+        let mut client = Client::connect(connstring.as_str(), NoTls).unwrap();
+
+        println!("Pageserver query: '{}'", sql);
+        client.simple_query(sql).unwrap()
+    }
+}
+
+impl Drop for StorageControlPlane {
+    fn drop(&mut self) {
+        self.stop();
+    }
+}
+
+pub struct PageServerNode {
+    page_service_addr: SocketAddr,
+    data_dir: PathBuf,
+}
+
+impl PageServerNode {
+    // TODO: method to force redo on a specific relation
+
+    // TODO: make wal-redo-postgres workable without data directory?
+    pub fn init(&self) {
+        fs::create_dir_all(self.data_dir.clone()).unwrap();
+
+        let datadir_path = self.data_dir.join("wal_redo_pgdata");
+        fs::remove_dir_all(datadir_path.to_str().unwrap()).ok();
+
+        let initdb = Command::new(PG_BIN_DIR.join("initdb"))
+            .args(&["-D", datadir_path.to_str().unwrap()])
+            .arg("-N")
+            .arg("--no-instructions")
+            .env_clear()
+            .env("LD_LIBRARY_PATH", PG_LIB_DIR.to_str().unwrap())
+            .status()
+            .expect("failed to execute initdb");
+        if !initdb.success() {
+            panic!("initdb failed");
+        }
+    }
+
+    pub fn start(&self) {
+        println!("Starting pageserver at '{}'", self.page_service_addr);
+
+        let status = Command::new(BIN_DIR.join("pageserver"))
+            .args(&["-D", self.data_dir.to_str().unwrap()])
+            .args(&["-l", self.page_service_addr.to_string().as_str()])
+            .arg("-d")
+            .arg("--skip-recovery")
+            .env_clear()
+            .env("PATH", PG_BIN_DIR.to_str().unwrap()) // path to postres-wal-redo binary
+            .status()
+            .expect("failed to start pageserver");
+
+        if !status.success() {
+            panic!("pageserver start failed");
+        }
+    }
+
+    pub fn start_froms3(&self) {
+        println!("Starting pageserver at '{}'", self.page_service_addr);
+
+        let status = Command::new(BIN_DIR.join("pageserver"))
+            .args(&["-D", self.data_dir.to_str().unwrap()])
+            .args(&["-l", self.page_service_addr.to_string().as_str()])
+            .arg("-d")
+            .env_clear()
+            .env("PATH", PG_BIN_DIR.to_str().unwrap()) // path to postres-wal-redo binary
+            .env("S3_ENDPOINT", "https://127.0.0.1:9000")
+            .env("S3_REGION", "us-east-1")
+            .env("S3_ACCESSKEY", "minioadmin")
+            .env("S3_SECRET", "minioadmin")
+            .status()
+            .expect("failed to start pageserver");
+
+        if !status.success() {
+            panic!("pageserver start failed");
+        }
+    }
+
+    pub fn stop(&self) {
+        let pidfile = self.data_dir.join("pageserver.pid");
+        let pid = fs::read_to_string(pidfile).unwrap();
+        let status = Command::new("kill")
+            .arg(pid)
+            .env_clear()
+            .status()
+            .expect("failed to execute kill");
+
+        if !status.success() {
+            panic!("kill start failed");
+        }
+    }
+}
+
+impl Drop for PageServerNode {
+    fn drop(&mut self) {
+        self.stop();
+        // fs::remove_dir_all(self.data_dir.clone()).unwrap();
+    }
+}
+
+pub struct WalAcceptorNode {
+    listen: SocketAddr,
+    data_dir: PathBuf,
+}
+
+impl WalAcceptorNode {
+    pub fn init(&self) {
+        if self.data_dir.exists() {
+            fs::remove_dir_all(self.data_dir.clone()).unwrap();
+        }
+        fs::create_dir_all(self.data_dir.clone()).unwrap();
+    }
+
+    pub fn start(&self) {
+        println!(
+            "Starting wal_acceptor in {} listening '{}'",
+            self.data_dir.to_str().unwrap(),
+            self.listen
+        );
+
+        let status = Command::new(BIN_DIR.join("wal_acceptor"))
+            .args(&["-D", self.data_dir.to_str().unwrap()])
+            .args(&["-l", self.listen.to_string().as_str()])
+            .arg("-d")
+            .arg("-n")
+            .status()
+            .expect("failed to start wal_acceptor");
+
+        if !status.success() {
+            panic!("wal_acceptor start failed");
+        }
+    }
+
+    pub fn stop(&self) {
+        let pidfile = self.data_dir.join("wal_acceptor.pid");
+        if let Ok(pid) = fs::read_to_string(pidfile) {
+            let _status = Command::new("kill")
+                .arg(pid)
+                .env_clear()
+                .status()
+                .expect("failed to execute kill");
+        }
+    }
+}
+
+impl Drop for WalAcceptorNode {
+    fn drop(&mut self) {
+        self.stop();
+        // fs::remove_dir_all(self.data_dir.clone()).unwrap();
+    }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+//
+// ComputeControlPlane
+//
+pub struct ComputeControlPlane<'a> {
+    pg_bin_dir: PathBuf,
+    work_dir: PathBuf,
+    last_assigned_port: u16,
+    storage_cplane: &'a StorageControlPlane,
+    nodes: Vec<Arc<PostgresNode>>,
+}
+
+impl ComputeControlPlane<'_> {
+    pub fn local(storage_cplane: &StorageControlPlane) -> ComputeControlPlane {
+        ComputeControlPlane {
+            pg_bin_dir: PG_BIN_DIR.to_path_buf(),
+            work_dir: TEST_WORKDIR.to_path_buf(),
+            last_assigned_port: 65431,
+            storage_cplane: storage_cplane,
+            nodes: Vec::new(),
+        }
+    }
+
+    // TODO: check port availability and
+    fn get_port(&mut self) -> u16 {
+        let port = self.last_assigned_port + 1;
+        self.last_assigned_port += 1;
+        port
+    }
+
+    pub fn new_vanilla_node<'a>(&mut self) -> &Arc<PostgresNode> {
+        // allocate new node entry with generated port
+        let node_id = self.nodes.len() + 1;
+        let node = PostgresNode {
+            _node_id: node_id,
+            port: self.get_port(),
+            ip: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
+            pgdata: self.work_dir.join(format!("compute/pg{}", node_id)),
+            pg_bin_dir: self.pg_bin_dir.clone(),
+        };
+        self.nodes.push(Arc::new(node));
+        let node = self.nodes.last().unwrap();
+
+        // initialize data directory
+        fs::remove_dir_all(node.pgdata.to_str().unwrap()).ok();
+        let initdb_path = self.pg_bin_dir.join("initdb");
+        println!("initdb_path: {}", initdb_path.to_str().unwrap());
+        let initdb = Command::new(initdb_path)
+            .args(&["-D", node.pgdata.to_str().unwrap()])
+            .arg("-N")
+            .arg("--no-instructions")
+            .env_clear()
+            .env("LD_LIBRARY_PATH", PG_LIB_DIR.to_str().unwrap())
+            .status()
+            .expect("failed to execute initdb");
+
+        if !initdb.success() {
+            panic!("initdb failed");
+        }
+
+        // // allow local replication connections
+        // node.append_conf("pg_hba.conf", format!("\
+        //     host replication all {}/32 sspi include_realm=1 map=regress\n\
+        // ", node.ip).as_str());
+
+        // listen for selected port
+        node.append_conf(
+            "postgresql.conf",
+            format!(
+                "\
+            max_wal_senders = 10\n\
+            max_replication_slots = 10\n\
+            hot_standby = on\n\
+            shared_buffers = 1MB\n\
+            max_connections = 100\n\
+            wal_level = replica\n\
+            listen_addresses = '{address}'\n\
+            port = {port}\n\
+        ",
+                address = node.ip,
+                port = node.port
+            )
+            .as_str(),
+        );
+
+        node
+    }
+
+    // Init compute node without files, only datadir structure
+    // use initdb --compute-node flag and GUC 'computenode_mode'
+    // to distinguish the node
+    pub fn new_minimal_node<'a>(&mut self) -> &Arc<PostgresNode> {
+        // allocate new node entry with generated port
+        let node_id = self.nodes.len() + 1;
+        let node = PostgresNode {
+            _node_id: node_id,
+            port: self.get_port(),
+            ip: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
+            pgdata: self.work_dir.join(format!("compute/pg{}", node_id)),
+            pg_bin_dir: self.pg_bin_dir.clone(),
+        };
+        self.nodes.push(Arc::new(node));
+        let node = self.nodes.last().unwrap();
+
+        // initialize data directory
+        fs::remove_dir_all(node.pgdata.to_str().unwrap()).ok();
+        let initdb_path = self.pg_bin_dir.join("initdb");
+        println!("initdb_path: {}", initdb_path.to_str().unwrap());
+        let initdb = Command::new(initdb_path)
+            .args(&["-D", node.pgdata.to_str().unwrap()])
+            .arg("-N")
+            .arg("--no-instructions")
+            .arg("--compute-node")
+            .env_clear()
+            .env("LD_LIBRARY_PATH", PG_LIB_DIR.to_str().unwrap())
+            .status()
+            .expect("failed to execute initdb");
+
+        if !initdb.success() {
+            panic!("initdb failed");
+        }
+
+        // // allow local replication connections
+        // node.append_conf("pg_hba.conf", format!("\
+        //     host replication all {}/32 sspi include_realm=1 map=regress\n\
+        // ", node.ip).as_str());
+
+        // listen for selected port
+        node.append_conf(
+            "postgresql.conf",
+            format!(
+                "\
+            max_wal_senders = 10\n\
+            max_replication_slots = 10\n\
+            hot_standby = on\n\
+            shared_buffers = 1MB\n\
+            max_connections = 100\n\
+            wal_level = replica\n\
+            listen_addresses = '{address}'\n\
+            port = {port}\n\
+            computenode_mode = true\n\
+            ",
+                address = node.ip,
+                port = node.port
+            )
+            .as_str(),
+        );
+        node
+    }
+
+    pub fn new_node_wo_data(&mut self) -> Arc<PostgresNode> {
+        let storage_cplane = self.storage_cplane;
+        let node = self.new_minimal_node();
+
+        let pserver = storage_cplane.page_server_addr();
+
+        // Configure that node to take pages from pageserver
+        node.append_conf(
+            "postgresql.conf",
+            format!(
+                "\
+            page_server_connstring = 'host={} port={}'\n\
+        ",
+                pserver.ip(),
+                pserver.port()
+            )
+            .as_str(),
+        );
+
+        node.clone()
+    }
+
+    pub fn new_node(&mut self) -> Arc<PostgresNode> {
+        let storage_cplane = self.storage_cplane;
+        let node = self.new_vanilla_node();
+
+        let pserver = storage_cplane.page_server_addr();
+
+        // Configure that node to take pages from pageserver
+        node.append_conf(
+            "postgresql.conf",
+            format!(
+                "\
+            page_server_connstring = 'host={} port={}'\n\
+        ",
+                pserver.ip(),
+                pserver.port()
+            )
+            .as_str(),
+        );
+
+        node.clone()
+    }
+
+    pub fn new_master_node(&mut self) -> Arc<PostgresNode> {
+        let node = self.new_vanilla_node();
+
+        node.append_conf(
+            "postgresql.conf",
+            "synchronous_standby_names = 'safekeeper_proxy'\n\
+						 ",
+        );
+        node.clone()
+    }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+pub struct WalProposerNode {
+    pid: u32,
+}
+
+impl WalProposerNode {
+    pub fn stop(&self) {
+        let status = Command::new("kill")
+            .arg(self.pid.to_string())
+            .env_clear()
+            .status()
+            .expect("failed to execute kill");
+
+        if !status.success() {
+            panic!("kill start failed");
+        }
+    }
+}
+
+impl Drop for WalProposerNode {
+    fn drop(&mut self) {
+        self.stop();
+    }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+pub struct PostgresNode {
+    _node_id: usize,
+    pub port: u16,
+    pub ip: IpAddr,
+    pgdata: PathBuf,
+    pg_bin_dir: PathBuf,
+}
+
+impl PostgresNode {
+    pub fn append_conf(&self, config: &str, opts: &str) {
+        OpenOptions::new()
+            .append(true)
+            .open(self.pgdata.join(config).to_str().unwrap())
+            .unwrap()
+            .write_all(opts.as_bytes())
+            .unwrap();
+    }
+
+    fn pg_ctl(&self, args: &[&str], check_ok: bool) {
+        let pg_ctl_path = self.pg_bin_dir.join("pg_ctl");
+        let pg_ctl = Command::new(pg_ctl_path)
+            .args(
+                [
+                    &[
+                        "-D",
+                        self.pgdata.to_str().unwrap(),
+                        "-l",
+                        self.pgdata.join("log").to_str().unwrap(),
+                    ],
+                    args,
+                ]
+                .concat(),
+            )
+            .env_clear()
+            .env("LD_LIBRARY_PATH", PG_LIB_DIR.to_str().unwrap())
+            .status()
+            .expect("failed to execute pg_ctl");
+
+        if check_ok && !pg_ctl.success() {
+            panic!("pg_ctl failed");
+        }
+    }
+
+    pub fn start(&self, storage_cplane: &StorageControlPlane) {
+        if storage_cplane.page_servers.len() != 0 {
+            let _res =
+                storage_cplane.page_server_psql(format!("callmemaybe {}", self.connstr()).as_str());
+        }
+        println!("Starting postgres node at '{}'", self.connstr());
+        self.pg_ctl(&["start"], true);
+    }
+
+    pub fn restart(&self) {
+        self.pg_ctl(&["restart"], true);
+    }
+
+    pub fn stop(&self) {
+        self.pg_ctl(&["-m", "immediate", "stop"], true);
+    }
+
+    pub fn connstr(&self) -> String {
+        format!("host={} port={} user={}", self.ip, self.port, self.whoami())
+    }
+
+    // XXX: cache that in control plane
+    pub fn whoami(&self) -> String {
+        let output = Command::new("whoami")
+            .output()
+            .expect("failed to execute whoami");
+
+        if !output.status.success() {
+            panic!("whoami failed");
+        }
+
+        String::from_utf8(output.stdout).unwrap().trim().to_string()
+    }
+
+    pub fn safe_psql(&self, db: &str, sql: &str) -> Vec<tokio_postgres::Row> {
+        let connstring = format!(
+            "host={} port={} dbname={} user={}",
+            self.ip,
+            self.port,
+            db,
+            self.whoami()
+        );
+        let mut client = Client::connect(connstring.as_str(), NoTls).unwrap();
+
+        println!("Running {}", sql);
+        client.query(sql, &[]).unwrap()
+    }
+
+    pub fn open_psql(&self, db: &str) -> Client {
+        let connstring = format!(
+            "host={} port={} dbname={} user={}",
+            self.ip,
+            self.port,
+            db,
+            self.whoami()
+        );
+        Client::connect(connstring.as_str(), NoTls).unwrap()
+    }
+
+    pub fn get_pgdata(&self) -> Option<&str> {
+        self.pgdata.to_str()
+    }
+
+    // Request from pageserver stub controlfile, respective xlog
+    // and a bunch of files needed to start computenode
+    //
+    // NOTE this "file" request is a crutch.
+    // It asks pageserver to write requested page to the provided filepath
+    // and thus only works locally.
+    // TODO receive pages via some libpq protocol.
+    // The problem I've met is that nonrelfiles are not valid utf8 and cannot be
+    // handled by simple_query(). that expects test.
+    // And reqular query() uses prepared queries.
+
+    // TODO pass sysid as parameter
+    pub fn setup_compute_node(&self, sysid: u64, storage_cplane: &StorageControlPlane) {
+        let mut query;
+        //Request pg_control from pageserver
+        query = format!(
+            "file {}/global/pg_control,{},{},{},{},{},{},{}",
+            self.pgdata.to_str().unwrap(),
+            sysid as u64, //sysid
+            1664,         //tablespace
+            0,            //dboid
+            0,            //reloid
+            42,           //forknum pg_control
+            0,            //blkno
+            0             //lsn
+        );
+        storage_cplane.page_server_psql(query.as_str());
+
+        //Request pg_xact and pg_multixact from pageserver
+        //We need them for initial pageserver startup and authentication
+        //TODO figure out which block number we really need
+        query = format!(
+            "file {}/pg_xact/0000,{},{},{},{},{},{},{}",
+            self.pgdata.to_str().unwrap(),
+            sysid as u64, //sysid
+            0,            //tablespace
+            0,            //dboid
+            0,            //reloid
+            44,           //forknum
+            0,            //blkno
+            0             //lsn
+        );
+        storage_cplane.page_server_psql(query.as_str());
+
+        query = format!(
+            "file {}/pg_multixact/offsets/0000,{},{},{},{},{},{},{}",
+            self.pgdata.to_str().unwrap(),
+            sysid as u64, //sysid
+            0,            //tablespace
+            0,            //dboid
+            0,            //reloid
+            45,           //forknum
+            0,            //blkno
+            0             //lsn
+        );
+        storage_cplane.page_server_psql(query.as_str());
+
+        query = format!(
+            "file {}/pg_multixact/members/0000,{},{},{},{},{},{},{}",
+            self.pgdata.to_str().unwrap(),
+            sysid as u64, //sysid
+            0,            //tablespace
+            0,            //dboid
+            0,            //reloid
+            46,           //forknum
+            0,            //blkno
+            0             //lsn
+        );
+        storage_cplane.page_server_psql(query.as_str());
+
+        //Request a few shared catalogs needed for authentication
+        //Without them we cannot setup connection with pageserver to request further pages
+        let reloids = [1260, 1261, 1262, 2396];
+        for reloid in reloids.iter() {
+            //FIXME request all blocks from file, not just 10
+            for blkno in 0..10 {
+                query = format!(
+                    "file {}/global/{},{},{},{},{},{},{},{}",
+                    self.pgdata.to_str().unwrap(),
+                    reloid,       //suse it as filename
+                    sysid as u64, //sysid
+                    1664,         //tablespace
+                    0,            //dboid
+                    reloid,       //reloid
+                    0,            //forknum
+                    blkno,        //blkno
+                    0             //lsn
+                );
+                storage_cplane.page_server_psql(query.as_str());
+            }
+        }
+
+        fs::create_dir(format!("{}/base/13006", self.pgdata.to_str().unwrap())).unwrap();
+        fs::create_dir(format!("{}/base/13007", self.pgdata.to_str().unwrap())).unwrap();
+
+        //FIXME figure out what wal file we need to successfully start
+        let walfilepath = format!(
+            "{}/pg_wal/000000010000000000000001",
+            self.pgdata.to_str().unwrap()
+        );
+        fs::copy(
+            "/home/anastasia/zenith/zenith/tmp_check/pgdata/pg_wal/000000010000000000000001",
+            walfilepath,
+        )
+        .unwrap();
+
+        println!("before resetwal ");
+
+        let pg_resetwal_path = self.pg_bin_dir.join("pg_resetwal");
+
+        // Now it does nothing, just prints existing content of pg_control.
+        // TODO update values with most recent lsn, xid, oid requested from pageserver
+        let pg_resetwal = Command::new(pg_resetwal_path)
+            .args(&["-D", self.pgdata.to_str().unwrap()])
+            .arg("-n") //dry run
+            //.arg("-f")
+            //.args(&["--next-transaction-id", "100500"])
+            //.args(&["--next-oid", "17000"])
+            //.args(&["--next-transaction-id", "100500"])
+            .status()
+            .expect("failed to execute pg_resetwal");
+
+        if !pg_resetwal.success() {
+            panic!("pg_resetwal failed");
+        }
+
+        println!("setup done");
+    }
+
+    pub fn start_proxy(&self, wal_acceptors: String) -> WalProposerNode {
+        let proxy_path = PG_BIN_DIR.join("safekeeper_proxy");
+        match Command::new(proxy_path.as_path())
+            .args(&["-s", &wal_acceptors])
+            .args(&["-h", &self.ip.to_string()])
+            .args(&["-p", &self.port.to_string()])
+            .arg("-v")
+            .stderr(File::create(TEST_WORKDIR.join("safepkeeper_proxy.log")).unwrap())
+            .spawn()
+        {
+            Ok(child) => WalProposerNode { pid: child.id() },
+            Err(e) => panic!("Failed to launch {:?}: {}", proxy_path, e),
+        }
+    }
+
+    pub fn push_to_s3(&self) {
+        println!("Push to s3 node  at '{}'", self.pgdata.to_str().unwrap());
+
+        let zenith_push_path = self.pg_bin_dir.join("zenith_push");
+        println!("zenith_push_path: {}", zenith_push_path.to_str().unwrap());
+
+        let status = Command::new(zenith_push_path)
+            .args(&["-D", self.pgdata.to_str().unwrap()])
+            .env_clear()
+            .env("S3_ENDPOINT", "https://127.0.0.1:9000")
+            .env("S3_REGION", "us-east-1")
+            .env("S3_ACCESSKEY", "minioadmin")
+            .env("S3_SECRET", "minioadmin")
+            // .env("S3_BUCKET", "zenith-testbucket")
+            .status()
+            .expect("failed to push node to s3");
+
+        if !status.success() {
+            panic!("zenith_push failed");
+        }
+    }
+
+    // TODO
+    pub fn pg_bench() {}
+    pub fn pg_regress() {}
+}
+
+impl Drop for PostgresNode {
+    // destructor to clean up state after test is done
+    // XXX: we may detect failed test by setting some flag in catch_unwind()
+    // and checking it here. But let just clean datadirs on start.
+    fn drop(&mut self) {
+        self.stop();
+        // fs::remove_dir_all(self.pgdata.clone()).unwrap();
+    }
+}
+
+pub fn regress_check(pg: &PostgresNode) {
+    pg.safe_psql("postgres", "CREATE DATABASE regression");
+
+    let regress_run_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("tmp_check/regress");
+    fs::create_dir_all(regress_run_path.clone()).unwrap();
+    std::env::set_current_dir(regress_run_path).unwrap();
+
+    let regress_build_path =
+        Path::new(env!("CARGO_MANIFEST_DIR")).join("../tmp_install/build/src/test/regress");
+    let regress_src_path =
+        Path::new(env!("CARGO_MANIFEST_DIR")).join("../vendor/postgres/src/test/regress");
+
+    let _regress_check = Command::new(regress_build_path.join("pg_regress"))
+        .args(&[
+            "--bindir=''",
+            "--use-existing",
+            format!("--bindir={}", PG_BIN_DIR.to_str().unwrap()).as_str(),
+            format!("--dlpath={}", regress_build_path.to_str().unwrap()).as_str(),
+            format!(
+                "--schedule={}",
+                regress_src_path.join("parallel_schedule").to_str().unwrap()
+            )
+            .as_str(),
+            format!("--inputdir={}", regress_src_path.to_str().unwrap()).as_str(),
+        ])
+        .env_clear()
+        .env("LD_LIBRARY_PATH", PG_LIB_DIR.to_str().unwrap())
+        .env("PGPORT", pg.port.to_string())
+        .env("PGUSER", pg.whoami())
+        .env("PGHOST", pg.ip.to_string())
+        .status()
+        .expect("pg_regress failed");
+}

integration_tests/tests/test_compute.rs

@@ -0,0 +1,7 @@
+// test node resettlement to an empty datadir
+#[test]
+fn test_resettlement() {}
+
+// test seq scan of everythin after restart
+#[test]
+fn test_cold_seqscan() {}

integration_tests/tests/test_control_plane.rs

@@ -0,0 +1,5 @@
+#[test]
+fn test_actions() {}
+
+#[test]
+fn test_regress() {}

integration_tests/tests/test_pageserver.rs

@@ -0,0 +1,210 @@
+#[allow(dead_code)]
+mod control_plane;
+use std::thread::sleep;
+use std::time::Duration;
+
+use control_plane::ComputeControlPlane;
+use control_plane::StorageControlPlane;
+
+// XXX: force all redo at the end
+// -- restart + seqscan won't read deleted stuff
+// -- pageserver api endpoint to check all rels
+
+//Handcrafted cases with wal records that are (were) problematic for redo.
+#[test]
+#[ignore]
+fn test_redo_cases() {
+    // Start pageserver that reads WAL directly from that postgres
+    let storage_cplane = StorageControlPlane::one_page_server(false);
+    let mut compute_cplane = ComputeControlPlane::local(&storage_cplane);
+
+    // start postgres
+    let node = compute_cplane.new_node();
+    node.start(&storage_cplane);
+
+    println!("await pageserver connection...");
+    sleep(Duration::from_secs(3));
+
+    // check basic work with table
+    node.safe_psql(
+        "postgres",
+        "CREATE TABLE t(key int primary key, value text)",
+    );
+    node.safe_psql(
+        "postgres",
+        "INSERT INTO t SELECT generate_series(1,100), 'payload'",
+    );
+    let count: i64 = node
+        .safe_psql("postgres", "SELECT sum(key) FROM t")
+        .first()
+        .unwrap()
+        .get(0);
+    println!("sum = {}", count);
+    assert_eq!(count, 5050);
+
+    //check 'create table as'
+    node.safe_psql("postgres", "CREATE TABLE t2 AS SELECT * FROM t");
+    let count: i64 = node
+        .safe_psql("postgres", "SELECT sum(key) FROM t")
+        .first()
+        .unwrap()
+        .get(0);
+    println!("sum = {}", count);
+    assert_eq!(count, 5050);
+}
+
+// Runs pg_regress on a compute node
+#[test]
+#[ignore]
+fn test_regress() {
+    // Start pageserver that reads WAL directly from that postgres
+    let storage_cplane = StorageControlPlane::one_page_server(false);
+    let mut compute_cplane = ComputeControlPlane::local(&storage_cplane);
+
+    // start postgres
+    let node = compute_cplane.new_node();
+    node.start(&storage_cplane);
+
+    println!("await pageserver connection...");
+    sleep(Duration::from_secs(3));
+
+    control_plane::regress_check(&node);
+}
+
+// Run two postgres instances on one pageserver
+#[test]
+#[ignore]
+fn test_pageserver_multitenancy() {
+    // Start pageserver that reads WAL directly from that postgres
+    let storage_cplane = StorageControlPlane::one_page_server(false);
+    let mut compute_cplane = ComputeControlPlane::local(&storage_cplane);
+
+    // Allocate postgres instance, but don't start
+    let node1 = compute_cplane.new_node();
+    let node2 = compute_cplane.new_node();
+    node1.start(&storage_cplane);
+    node2.start(&storage_cplane);
+
+    // XXX: add some extension func to postgres to check walsender conn
+    // XXX: or better just drop that
+    println!("await pageserver connection...");
+    sleep(Duration::from_secs(3));
+
+    // check node1
+    node1.safe_psql(
+        "postgres",
+        "CREATE TABLE t(key int primary key, value text)",
+    );
+    node1.safe_psql(
+        "postgres",
+        "INSERT INTO t SELECT generate_series(1,100), 'payload'",
+    );
+    let count: i64 = node1
+        .safe_psql("postgres", "SELECT sum(key) FROM t")
+        .first()
+        .unwrap()
+        .get(0);
+    println!("sum = {}", count);
+    assert_eq!(count, 5050);
+
+    // check node2
+    node2.safe_psql(
+        "postgres",
+        "CREATE TABLE t(key int primary key, value text)",
+    );
+    node2.safe_psql(
+        "postgres",
+        "INSERT INTO t SELECT generate_series(100,200), 'payload'",
+    );
+    let count: i64 = node2
+        .safe_psql("postgres", "SELECT sum(key) FROM t")
+        .first()
+        .unwrap()
+        .get(0);
+    println!("sum = {}", count);
+    assert_eq!(count, 15150);
+}
+
+#[test]
+#[ignore]
+// Start pageserver using s3 base image
+//
+// Requires working minio with hardcoded setup:
+// .env("S3_ENDPOINT", "https://127.0.0.1:9000")
+// .env("S3_REGION", "us-east-1")
+// .env("S3_ACCESSKEY", "minioadmin")
+// .env("S3_SECRET", "minioadmin")
+// .env("S3_BUCKET", "zenith-testbucket")
+// TODO use env variables in test
+fn test_pageserver_recovery() {
+    //This test expects that image is already uploaded to s3
+    //To upload it use zenith_push before test (see node.push_to_s3() for details)
+    let storage_cplane = StorageControlPlane::one_page_server(true);
+    let mut compute_cplane = ComputeControlPlane::local(&storage_cplane);
+
+    //Wait while daemon uploads pages from s3
+    sleep(Duration::from_secs(15));
+
+    let node_restored = compute_cplane.new_node_wo_data();
+
+    //TODO 6947041219207877724 is a hardcoded sysid for my cluster. Get it somewhere
+    node_restored.setup_compute_node(6947041219207877724, &storage_cplane);
+
+    node_restored.start(&storage_cplane);
+
+    let rows = node_restored.safe_psql("postgres", "SELECT relname from pg_class;");
+
+    assert_eq!(rows.len(), 395);
+}
+
+#[test]
+#[ignore]
+//Scenario for future test. Not implemented yet
+fn test_pageserver_node_switch() {
+    //Create pageserver
+    let storage_cplane = StorageControlPlane::one_page_server(false);
+    let mut compute_cplane = ComputeControlPlane::local(&storage_cplane);
+
+    //Create reqular node
+    let node = compute_cplane.new_node();
+    node.start(&storage_cplane);
+
+    node.safe_psql(
+        "postgres",
+        "CREATE TABLE t(key int primary key, value text)",
+    );
+    node.safe_psql(
+        "postgres",
+        "INSERT INTO t SELECT generate_series(1,100), 'payload'",
+    );
+    let count: i64 = node
+        .safe_psql("postgres", "SELECT sum(key) FROM t")
+        .first()
+        .unwrap()
+        .get(0);
+    println!("sum = {}", count);
+    assert_eq!(count, 5050);
+
+    //Push all node files to s3
+    //TODO upload them directly to pageserver
+    node.push_to_s3();
+    //Upload data from s3 to pageserver
+    //storage_cplane.upload_from_s3() //Not implemented yet
+
+    //Shut down the node
+    node.stop();
+
+    //Create new node without files
+    let node_restored = compute_cplane.new_node_wo_data();
+
+    // Setup minimal set of files needed to start node and setup pageserver connection
+    // TODO 6947041219207877724 is a hardcoded sysid. Get it from node
+    node_restored.setup_compute_node(6947041219207877724, &storage_cplane);
+
+    //Start compute node without files
+    node_restored.start(&storage_cplane);
+
+    //Ensure that is has table created on initial node
+    let rows = node_restored.safe_psql("postgres", "SELECT key from t;");
+    assert_eq!(rows.len(), 5050);
+}

integration_tests/tests/test_wal_acceptor.rs

@@ -0,0 +1,222 @@
+// Restart acceptors one by one while compute is under the load.
+#[allow(dead_code)]
+mod control_plane;
+use control_plane::ComputeControlPlane;
+use control_plane::StorageControlPlane;
+
+use rand::Rng;
+use std::sync::Arc;
+use std::time::SystemTime;
+use std::{thread, time};
+
+#[test]
+fn test_acceptors_normal_work() {
+    // Start pageserver that reads WAL directly from that postgres
+    const REDUNDANCY: usize = 3;
+    let storage_cplane = StorageControlPlane::fault_tolerant(REDUNDANCY);
+    let mut compute_cplane = ComputeControlPlane::local(&storage_cplane);
+    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
+
+    // start postgre
+    let node = compute_cplane.new_master_node();
+    node.start(&storage_cplane);
+
+    // start proxy
+    let _proxy = node.start_proxy(wal_acceptors);
+
+    // check basic work with table
+    node.safe_psql(
+        "postgres",
+        "CREATE TABLE t(key int primary key, value text)",
+    );
+    node.safe_psql(
+        "postgres",
+        "INSERT INTO t SELECT generate_series(1,100000), 'payload'",
+    );
+    let count: i64 = node
+        .safe_psql("postgres", "SELECT sum(key) FROM t")
+        .first()
+        .unwrap()
+        .get(0);
+    println!("sum = {}", count);
+    assert_eq!(count, 5000050000);
+    // check wal files equality
+}
+
+// Majority is always alive
+#[test]
+fn test_acceptors_restarts() {
+    // Start pageserver that reads WAL directly from that postgres
+    const REDUNDANCY: usize = 3;
+    const FAULT_PROBABILITY: f32 = 0.01;
+
+    let storage_cplane = StorageControlPlane::fault_tolerant(REDUNDANCY);
+    let mut compute_cplane = ComputeControlPlane::local(&storage_cplane);
+    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
+    let mut rng = rand::thread_rng();
+
+    // start postgre
+    let node = compute_cplane.new_master_node();
+    node.start(&storage_cplane);
+
+    // start proxy
+    let _proxy = node.start_proxy(wal_acceptors);
+    let mut failed_node: Option<usize> = None;
+
+    // check basic work with table
+    node.safe_psql(
+        "postgres",
+        "CREATE TABLE t(key int primary key, value text)",
+    );
+    let mut psql = node.open_psql("postgres");
+    for i in 1..=1000 {
+        psql.execute("INSERT INTO t values ($1, 'payload')", &[&i])
+            .unwrap();
+        let prob: f32 = rng.gen();
+        if prob <= FAULT_PROBABILITY {
+            if let Some(node) = failed_node {
+                storage_cplane.wal_acceptors[node].start();
+                failed_node = None;
+            } else {
+                let node: usize = rng.gen_range(0..REDUNDANCY);
+                failed_node = Some(node);
+                storage_cplane.wal_acceptors[node].stop();
+            }
+        }
+    }
+    let count: i64 = node
+        .safe_psql("postgres", "SELECT sum(key) FROM t")
+        .first()
+        .unwrap()
+        .get(0);
+    println!("sum = {}", count);
+    assert_eq!(count, 500500);
+}
+
+fn start_acceptor(cplane: &Arc<StorageControlPlane>, no: usize) {
+    let cp = cplane.clone();
+    thread::spawn(move || {
+        thread::sleep(time::Duration::from_secs(1));
+        cp.wal_acceptors[no].start();
+    });
+}
+
+// Stop majority of acceptors while compute is under the load. Boot
+// them again and check that nothing was losed. Repeat.
+// N_CRASHES env var
+#[test]
+fn test_acceptors_unavalability() {
+    // Start pageserver that reads WAL directly from that postgres
+    const REDUNDANCY: usize = 2;
+
+    let storage_cplane = StorageControlPlane::fault_tolerant(REDUNDANCY);
+    let mut compute_cplane = ComputeControlPlane::local(&storage_cplane);
+    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
+
+    // start postgre
+    let node = compute_cplane.new_master_node();
+    node.start(&storage_cplane);
+
+    // start proxy
+    let _proxy = node.start_proxy(wal_acceptors);
+
+    // check basic work with table
+    node.safe_psql(
+        "postgres",
+        "CREATE TABLE t(key int primary key, value text)",
+    );
+    let mut psql = node.open_psql("postgres");
+    psql.execute("INSERT INTO t values (1, 'payload')", &[])
+        .unwrap();
+
+    storage_cplane.wal_acceptors[0].stop();
+    let cp = Arc::new(storage_cplane);
+    start_acceptor(&cp, 0);
+    let now = SystemTime::now();
+    psql.execute("INSERT INTO t values (2, 'payload')", &[])
+        .unwrap();
+    assert!(now.elapsed().unwrap().as_secs() > 1);
+    psql.execute("INSERT INTO t values (3, 'payload')", &[])
+        .unwrap();
+
+    cp.wal_acceptors[1].stop();
+    start_acceptor(&cp, 1);
+    psql.execute("INSERT INTO t values (4, 'payload')", &[])
+        .unwrap();
+    assert!(now.elapsed().unwrap().as_secs() > 2);
+
+    psql.execute("INSERT INTO t values (5, 'payload')", &[])
+        .unwrap();
+
+    let count: i64 = node
+        .safe_psql("postgres", "SELECT sum(key) FROM t")
+        .first()
+        .unwrap()
+        .get(0);
+    println!("sum = {}", count);
+    assert_eq!(count, 15);
+}
+
+fn simulate_failures(cplane: &Arc<StorageControlPlane>) {
+    let mut rng = rand::thread_rng();
+    let n_acceptors = cplane.wal_acceptors.len();
+    let failure_period = time::Duration::from_secs(1);
+    loop {
+        thread::sleep(failure_period);
+        let mask: u32 = rng.gen_range(0..(1 << n_acceptors));
+        for i in 0..n_acceptors {
+            if (mask & (1 << i)) != 0 {
+                cplane.wal_acceptors[i].stop();
+            }
+        }
+        thread::sleep(failure_period);
+        for i in 0..n_acceptors {
+            if (mask & (1 << i)) != 0 {
+                cplane.wal_acceptors[i].start();
+            }
+        }
+    }
+}
+
+// Race condition test
+#[test]
+fn test_race_conditions() {
+    // Start pageserver that reads WAL directly from that postgres
+    const REDUNDANCY: usize = 3;
+
+    let storage_cplane = StorageControlPlane::fault_tolerant(REDUNDANCY);
+    let mut compute_cplane = ComputeControlPlane::local(&storage_cplane);
+    let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
+
+    // start postgre
+    let node = compute_cplane.new_master_node();
+    node.start(&storage_cplane);
+
+    // start proxy
+    let _proxy = node.start_proxy(wal_acceptors);
+
+    // check basic work with table
+    node.safe_psql(
+        "postgres",
+        "CREATE TABLE t(key int primary key, value text)",
+    );
+    let cplane = Arc::new(storage_cplane);
+    let cp = cplane.clone();
+    thread::spawn(move || {
+        simulate_failures(&cp);
+    });
+
+    let mut psql = node.open_psql("postgres");
+    for i in 1..=1000 {
+        psql.execute("INSERT INTO t values ($1, 'payload')", &[&i])
+            .unwrap();
+    }
+    let count: i64 = node
+        .safe_psql("postgres", "SELECT sum(key) FROM t")
+        .first()
+        .unwrap()
+        .get(0);
+    println!("sum = {}", count);
+    assert_eq!(count, 500500);
+    cplane.stop();
+}

mgmt-console/.gitignore

@@ -0,0 +1,23 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# production
+/build
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*

mgmt-console/README

@@ -0,0 +1,55 @@
+Mock implementation of a management console.
+
+See demo-howto.txt for usage.
+
+Building and Installation
+-------------------------
+
+To compile Postgres:
+  sudo apt build-dep postgresql
+  sudo apt install bison flex libz-dev libssl-dev
+  sudo apt install ccache
+  sudo apt install libcurl4-openssl-dev libxml2-dev
+
+For the webapp:
+  # NOTE: This requires at least version 1.1.0 of python3-flask. That's not
+  # available in Debian Buster, need at least Bullseye.
+
+  sudo apt install python3 python3-flask python3-pip npm webpack
+  pip3 install Flask-BasicAuth
+  pip3 install boto3
+
+git clone and compile and install patched version of Postgres:
+
+  git clone https://github.com/libzenith/postgres.git
+  cd postgres
+  git checkout zenith-experiments
+  ./configure --enable-debug --enable-cassert --with-openssl --prefix=/home/heikki/pgsql-install --with-libxml CC="ccache gcc" CFLAGS="-O0"
+  make -j4 -s install
+
+Get the webapp:
+  cd ~
+  git clone https://github.com/libzenith/zenith-mgmt-console.git
+  cd zenith-mgmt-console
+  mkdir pgdatadirs
+
+
+  openssl req -new -x509 -days 365 -nodes -text -out server.crt \
+    -keyout server.key -subj "/CN=zenith-demo"
+
+For Mock S3 server (unless you want to test against a real cloud service):
+  sudo apt install python3-tornado
+
+  cd ~/zenith-mgmt-console
+  git clone https://github.com/hlinnaka/ms3.git
+
+Compile & run it:
+  npm install
+  webpack # compile React app
+
+  BASIC_AUTH_PASSWORD=<password> ./launch-local.sh
+
+
+You can view the contents of the S3 bucket with browser:
+
+http://<server>/list_bucket

mgmt-console/app.py

@@ -0,0 +1,340 @@
+from flask import request
+from flask_basicauth import BasicAuth
+from flask import render_template
+from subprocess import PIPE, STDOUT, run, Popen
+import html
+import os
+import re
+import shutil
+import logging
+import time
+
+import boto3
+from boto3.session import Session
+from botocore.client import Config
+from botocore.handlers import set_list_objects_encoding_type_url
+
+from flask import Flask
+
+import waldump
+
+
+app = Flask(__name__)
+
+app.config['BASIC_AUTH_USERNAME'] = 'zenith'
+app.config['BASIC_AUTH_PASSWORD'] = os.getenv('BASIC_AUTH_PASSWORD')
+app.config['BASIC_AUTH_FORCE'] = True
+
+basic_auth = BasicAuth(app)
+
+# S3 configuration:
+
+ENDPOINT = os.getenv('S3_ENDPOINT', 'https://localhost:9000')
+ACCESS_KEY = os.getenv('S3_ACCESSKEY', 'minioadmin')
+SECRET = os.getenv('S3_SECRET', '')
+BUCKET = os.getenv('S3_BUCKET', 'foobucket')
+
+print("Using bucket at " + ENDPOINT);
+
+#boto3.set_stream_logger('botocore', logging.DEBUG)
+
+session = Session(aws_access_key_id=ACCESS_KEY,
+                  aws_secret_access_key=SECRET,
+                  region_name=os.getenv('S3_REGION', 'auto'))
+
+# needed for google cloud?
+session.events.unregister('before-parameter-build.s3.ListObjects',
+                          set_list_objects_encoding_type_url)
+
+s3resource = session.resource('s3',
+                              endpoint_url=ENDPOINT,
+                              verify=False,
+                              config=Config(signature_version='s3v4'))
+s3bucket = s3resource.Bucket(BUCKET)
+
+s3_client = boto3.client('s3',
+                         endpoint_url=ENDPOINT,
+                         verify=False,
+                         config=Config(signature_version='s3v4'),
+                         aws_access_key_id=ACCESS_KEY,
+                         aws_secret_access_key=SECRET)
+
+
+@app.route("/")
+def index():
+    return render_template("index.html")
+
+
+@app.route("/api/waldump")
+def render_waldump():
+    return render_template("waldump.html")
+
+@app.route('/api/fetch_wal')
+def fetch_wal():
+    return waldump.fetch_wal(request, s3bucket);
+
+@app.route("/api/server_status")
+def server_status():
+    dirs = os.listdir("pgdatadirs")
+    dirs.sort()
+
+    primary = None
+    standbys = []
+
+    for dirname in dirs:
+        
+        result = run("pg_ctl status -D pgdatadirs/" + dirname, stdout=PIPE, stderr=STDOUT, universal_newlines=True, shell=True)
+
+        srv = {
+            'datadir': dirname,
+            'status': result.stdout,
+            'port': None
+        }
+
+        if dirname == 'primary':
+            primary = srv;
+            primary['port'] = 5432;
+        else:
+            standby_match = re.search('standby_([0-9]+)', dirname)
+            if standby_match:
+                srv['port'] = int(standby_match.group(1))
+
+            standbys.append(srv);
+
+    return {'primary': primary, 'standbys': standbys}
+
+@app.route('/api/list_bucket')
+def list_bucket():
+
+    response = 'cloud bucket contents:<br>\n'
+
+    for file in s3bucket.objects.all():
+        response = response + html.escape(file.key) + '<br>\n'
+
+    return response
+
+def walpos_str(walpos):
+    return '{:X}/{:X}'.format(walpos >> 32, walpos & 0xFFFFFFFF)
+
+@app.route('/api/bucket_summary')
+def bucket_summary():
+
+    nonrelimages = []
+    minwal = int(0)
+    maxwal = int(0)
+    minseqwal = int(0)
+    maxseqwal = int(0)
+
+    for file in s3bucket.objects.all():
+        path = file.key
+        match = re.search('nonreldata/nonrel_([0-9A-F]+).tar', path)
+        if match:
+            walpos = int(match.group(1), 16)
+            nonrelimages.append(walpos_str(walpos))
+
+        match = re.search('nonreldata/nonrel_([0-9A-F]+)-([0-9A-F]+)', path)
+        if match:
+            endwal = int(match.group(2), 16)
+            if endwal > maxwal:
+                maxwal = endwal
+
+        match = re.search('walarchive/([0-9A-F]{8})([0-9A-F]{8})([0-9A-F]{8})', path)
+        if match:
+            tli = int(match.group(1), 16)
+            logno = int(match.group(2), 16)
+            segno = int(match.group(3), 16)
+            # FIXME: this assumes default 16 MB wal segment size
+            logsegno = logno * (0x100000000 / (16*1024*1024)) + segno
+
+            seqwal = int((logsegno + 1) * (16*1024*1024))
+
+            if seqwal > maxseqwal:
+                maxseqwal = seqwal;
+            if minseqwal == 0 or seqwal < minseqwal:
+                minseqwal = seqwal;
+
+    return {
+        'nonrelimages': nonrelimages,
+        'minwal': walpos_str(minwal),
+        'maxwal': walpos_str(maxwal),
+        'minseqwal': walpos_str(minseqwal),
+        'maxseqwal': walpos_str(maxseqwal)
+        }
+
+def print_cmd_result(cmd_result):
+    return print_cmd_result_ex(cmd_result.args, cmd_result.returncode, cmd_result.stdout)
+
+def print_cmd_result_ex(cmd, returncode, stdout):
+    res = ''
+    res += 'ran command:\n' + str(cmd) + '\n'
+    res += 'It returned code ' + str(returncode) + '\n'
+    res += '\n'
+    res += 'stdout/stderr:\n'
+    res += stdout
+
+    return res
+
+@app.route('/api/init_primary', methods=['GET', 'POST'])
+def init_primary():
+    
+    initdb_result = run("initdb -D pgdatadirs/primary --username=zenith --pwfile=pg-password.txt", stdout=PIPE, stderr=STDOUT, universal_newlines=True, shell=True)
+    if initdb_result.returncode != 0:
+        return print_cmd_result(initdb_result)
+    
+    # Append archive_mode and archive_command and port to postgresql.conf
+    f=open("pgdatadirs/primary/postgresql.conf", "a+")
+    f.write("listen_addresses='*'\n")
+    f.write("archive_mode=on\n")
+    f.write("archive_command='zenith_push --archive-wal-path=%p --archive-wal-fname=%f'\n")
+    f.write("ssl=on\n")
+    f.close()
+
+    f=open("pgdatadirs/primary/pg_hba.conf", "a+")
+    f.write("# allow SSL connections with password from anywhere\n")
+    f.write("hostssl    all             all             0.0.0.0/0           md5\n")
+    f.write("hostssl    all             all             ::0/0               md5\n")
+    f.close()
+
+    shutil.copyfile("server.crt", "pgdatadirs/primary/server.crt")
+    shutil.copyfile("server.key", "pgdatadirs/primary/server.key")
+    os.chmod("pgdatadirs/primary/server.key", 0o0600)
+    
+    start_proc = Popen(args=["pg_ctl", "start", "-D", "pgdatadirs/primary", "-l", "pgdatadirs/primary/log"], stdout=PIPE, stderr=STDOUT, universal_newlines=True, shell=False, start_new_session=True, close_fds=True)
+    start_rc = start_proc.wait()
+    start_stdout, start_stderr = start_proc.communicate()
+
+    responsestr = print_cmd_result(initdb_result) + '\n'
+    responsestr += print_cmd_result_ex(start_proc.args, start_rc, start_stdout)
+
+    return responsestr
+
+@app.route('/api/zenith_push', methods=['GET', 'POST'])
+def zenith_push():
+    # Stop the primary if it's running
+    stop_result = run(args=["pg_ctl", "stop", "-D", "pgdatadirs/primary"], stdout=PIPE, stderr=STDOUT, universal_newlines=True, shell=False, start_new_session=True, close_fds=True)
+    
+    # Call zenith_push
+    push_result = run("zenith_push -D pgdatadirs/primary", stdout=PIPE, stderr=STDOUT, universal_newlines=True, shell=True)
+
+    # Restart the primary
+    start_proc = Popen(args=["pg_ctl", "start", "-D", "pgdatadirs/primary", "-l", "pgdatadirs/primary/log"], stdout=PIPE, stderr=STDOUT, universal_newlines=True, shell=False, start_new_session=True, close_fds=True)
+    start_rc = start_proc.wait()
+    start_stdout, start_stderr = start_proc.communicate()
+    
+    responsestr = print_cmd_result(stop_result) + '\n'
+    responsestr += print_cmd_result(push_result) + '\n'
+    responsestr += print_cmd_result_ex(start_proc.args, start_rc, start_stdout) + '\n'
+
+    return responsestr
+
+@app.route('/api/create_standby', methods=['GET', 'POST'])
+def create_standby():
+
+    walpos = request.form.get('walpos')
+    if not walpos:
+        return 'no walpos'
+    
+    dirs = os.listdir("pgdatadirs")
+
+    last_port = 5432
+
+    for dirname in dirs:
+
+        standby_match = re.search('standby_([0-9]+)', dirname)
+        if standby_match:
+            port = int(standby_match.group(1))
+            if port > last_port:
+                last_port = port
+
+    standby_port = last_port + 1
+
+    standby_dir = "pgdatadirs/standby_" + str(standby_port)
+
+    # Call zenith_restore
+    restore_result = run(["zenith_restore", "--end=" + walpos, "-D", standby_dir], stdout=PIPE, stderr=STDOUT, encoding='latin1')
+    responsestr = print_cmd_result(restore_result)
+
+    if restore_result.returncode == 0:
+        # Append hot_standby and port to postgresql.conf
+        f=open(standby_dir + "/postgresql.conf", "a+")
+        f.write("hot_standby=on\n")
+        f.write("port=" + str(standby_port) + "\n")
+        f.close()
+
+        start_proc = Popen(args=["pg_ctl", "start", "-D", standby_dir, "-l", standby_dir + "/log"], stdout=PIPE, stderr=STDOUT, universal_newlines=True, shell=False, start_new_session=True, close_fds=True)
+        start_rc = start_proc.wait()
+        start_stdout, start_stderr = start_proc.communicate()
+        responsestr += '\n\n' + print_cmd_result_ex(start_proc.args, start_rc, start_stdout)
+
+    return responsestr
+
+@app.route('/api/destroy_server', methods=['GET', 'POST'])
+def destroy_primary():
+
+    datadir = request.form.get('datadir')
+
+    # Check that the datadir parameter doesn't contain anything funny.
+    if not re.match("^[A-Za-z0-9_-]+$", datadir):
+        raise Exception('invalid datadir: ' + datadir)
+    
+    # Stop the server if it's running
+    stop_result = run(args=["pg_ctl", "stop", "-m", "immediate", "-D", "pgdatadirs/" + datadir], stdout=PIPE, stderr=STDOUT, universal_newlines=True, shell=False, start_new_session=True, close_fds=True)
+
+    shutil.rmtree('pgdatadirs/' + datadir, ignore_errors=True)
+
+    responsestr = print_cmd_result(stop_result) + '\n'
+    responsestr += 'Deleted datadir ' + datadir + '.\n'
+
+    return responsestr
+
+@app.route('/api/restore_primary', methods=['GET', 'POST'])
+def restore_primary():
+
+    # Call zenith_restore
+    restore_result = run(["zenith_restore", "-D", "pgdatadirs/primary"], stdout=PIPE, stderr=STDOUT, encoding='latin1')
+    responsestr = print_cmd_result(restore_result)
+
+    # Append restore_command to postgresql.conf, so that it can find the last raw WAL segments
+    f=open("pgdatadirs/primary/postgresql.conf", "a+")
+    f.write("listen_addresses='*'\n")
+    f.write("restore_command='zenith_restore --archive-wal-path=%p --archive-wal-fname=%f'\n")
+    f.write("ssl=on\n")
+    f.close()
+    
+    if restore_result.returncode == 0:
+        start_proc = Popen(args=["pg_ctl", "start", "-D", "pgdatadirs/primary", "-l", "pgdatadirs/primary/log"], stdout=PIPE, stderr=STDOUT, universal_newlines=True, shell=False, start_new_session=True, close_fds=True)
+        start_rc = start_proc.wait()
+        start_stdout, start_stderr = start_proc.communicate()
+        responsestr += print_cmd_result_ex(start_proc.args, start_rc, start_stdout)
+
+    return responsestr
+
+@app.route('/api/slicedice', methods=['GET', 'POST'])
+def run_slicedice():
+    result = run("zenith_slicedice", stdout=PIPE, stderr=STDOUT, universal_newlines=True, shell=True)
+    
+    responsestr = print_cmd_result(result)
+
+    return responsestr
+
+@app.route('/api/reset_demo', methods=['POST'])
+def reset_all():
+    result = run("pkill -9 postgres", stdout=PIPE, stderr=STDOUT, universal_newlines=True, shell=True)
+
+    dirs = os.listdir("pgdatadirs")
+    for dirname in dirs:
+        shutil.rmtree('pgdatadirs/' + dirname)
+        
+    for file in s3bucket.objects.all():
+        s3_client.delete_object(Bucket = BUCKET, Key = file.key)
+
+    responsestr = print_cmd_result(result) + '\n'
+    responsestr += '''
+Deleted all Postgres datadirs.
+Deleted all files in object storage bucket.
+'''
+
+    return responsestr
+
+if __name__ == '__main__':
+    app.run()

mgmt-console/babel.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+    presets: ["@babel/preset-env", "@babel/preset-react"],
+};

mgmt-console/demo-howto.txt

@@ -0,0 +1,67 @@
+Mock implementation of a management console.
+
+This isn't very different from a "normal" PostgreSQL installation with
+a base backup and WAL archive. The main user-visible difference is
+that when you create a standby server, we don't restore the whole data
+directory, but only the "non-relation" files. Relation files are
+restored on demand, when they're accessed the first time. That makes
+the "create standby" operation is very fast, but with some delay when
+you connect and start running queries instead.  Most visible if you
+have a large database. (However, see note below about large databases)
+
+Note: lots of things are broken/unsafe. Things will fail if a table is
+larger than 1 GB. Or if there are more than 1000 files in the cloud
+bucket.
+
+How to use this demo:
+
+1. If there are any leftovers from previous runs, reset by clicking
+   the RESET DEMO button.  This kills and deletes all Postgres servers,
+   and empties the cloud storage bucket
+
+2. Create primary server by clicking on the "Init primary" button
+
+3. Push a base image of the primary to cloud storage, by clicking the
+   "push base image" button.  (This takes about 30 seconds, be
+   patient)
+
+4. Connect to primary with psql, and create a test table with a little data.
+
+      psql postgres  -p5432 -U zenith -h<host>
+
+      create table mytable (i int4);
+
+      insert into mytable values (1);
+      select pg_switch_wal();
+
+   The Postgres password is the same as for the management console.
+
+3. Now that there's a new WAL segment in the arhive, we can "slice &
+   dice" it. Click on the "Slice & dice button".
+
+4. Perform more updates on the primary, to generate more WAL.
+
+      insert into mytable values (2); select pg_switch_wal();
+      insert into mytable values (3); select pg_switch_wal();
+      insert into mytable values (4); select pg_switch_wal();
+      insert into mytable values (5); select pg_switch_wal();
+
+5. Slice & Dice the WAL again
+
+6. Now you can create read-only standby servers at any point in the
+   WAL. Type a WAL position in the text box (or use the slider), and
+   click "Create new standby". The first standby is created at port 5433,
+   the second at port 5434, and so forth.
+
+7. Connect to the standby with "psql -p 5433". Note that it takes a
+   few seconds until the connection is established. That's because the
+   standby has to restore the basic system catalogs, like pg_database and
+   pg_authid from the backup. After connecting, you can do "\d" to list
+   tables, this will also take a few seconds, as more catalog tables are
+   restored from backup.  Subsequent commands will be faster.
+
+   Run queries in the standby:
+
+      select * from mytable;
+
+   the result depends on the LSN that you picked when you created the server.

mgmt-console/js/app.js

@@ -0,0 +1,463 @@
+import React, { useState, useEffect } from 'react';
+import ReactDOM from 'react-dom';
+import Loader from "react-loader-spinner";
+import { Router, Route, Link, IndexRoute, hashHistory, browserHistory } from 'react-router';
+
+function ServerStatus(props) {
+    const datadir = props.server.datadir;
+    const status = props.server.status;
+    const port = props.server.port;
+
+    return (
+	<div>
+	    <h2>{ datadir == 'primary' ? 'Primary' : datadir }</h2>
+	    status: <div className='status'>{status}</div><br/>
+	    to connect: <span className='shellcommand'>psql -h { window.location.hostname } -p { port } -U zenith postgres</span><br/>
+	</div>
+    );
+}
+
+function StandbyList(props) {
+    const bucketSummary = props.bucketSummary;
+    const standbys = props.standbys;
+    const maxwalpos = bucketSummary.maxwal ? walpos_to_int(bucketSummary.maxwal) : 0;
+
+    const [walposInput, setWalposInput] = useState({ src: 'text', value: '0/0'});
+
+    // find earliest base image
+    const minwalpos = bucketSummary.nonrelimages ? bucketSummary.nonrelimages.reduce((minpos, imgpos_str, index, array) => {
+	const imgpos = walpos_to_int(imgpos_str);
+	return (minpos == 0 || imgpos < minpos) ? imgpos : minpos;
+    }, 0) : 0;
+
+    const can_create_standby = minwalpos > 0 && maxwalpos > 0 && maxwalpos >= minwalpos;
+    var walpos_valid = true;
+
+    function create_standby() {
+	const formdata = new FormData();
+	formdata.append("walpos", walposStr);
+
+	props.startOperation('Creating new standby at ' + walposStr + '...',
+			     fetch("/api/create_standby", { method: 'POST', body: formdata }));
+    }
+
+    function destroy_standby(datadir) {
+	const formdata = new FormData();
+	formdata.append("datadir", datadir);
+	props.startOperation('Destroying ' + datadir + '...',
+			     fetch("/api/destroy_server", { method: 'POST', body: formdata }));
+    }
+
+    const handleSliderChange = (event) => {
+	setWalposInput({ src: 'slider', value: event.target.value });
+    }    
+
+    const handleWalposChange = (event) => {
+	setWalposInput({ src: 'text', value: event.target.value });
+    }
+
+    var sliderValue;
+    var walposStr;
+    if (walposInput.src == 'text')
+    {
+	const walpos = walpos_to_int(walposInput.value);
+
+	if (walpos >= minwalpos && walpos <= maxwalpos)
+	    walpos_valid = true;
+	else
+	    walpos_valid = false;
+	
+	sliderValue = Math.round((walpos - minwalpos) / (maxwalpos - minwalpos) * 100);
+	walposStr = walposInput.value;
+    }
+    else
+    {
+	const slider = walposInput.value;
+	const new_walpos = minwalpos + slider / 100 * (maxwalpos - minwalpos);
+
+	console.log('minwalpos: '+ minwalpos);
+	console.log('maxwalpos: '+ maxwalpos);
+
+	walposStr = int_to_walpos(Math.round(new_walpos));
+	walpos_valid = true;
+	console.log(walposStr);
+    }
+
+    var standbystatus = ''
+    if (standbys)
+    {
+	standbystatus = 
+	    <div>
+		{
+		    standbys.length > 0 ? 
+ 			standbys.map((server) =>
+			    <>
+				<ServerStatus key={ 'status_' + server.datadir} server={server}/>
+				<button key={ 'destroy_' + server.datadir} onClick={e => destroy_standby(server.datadir)}>Destroy standby</button>
+			    </>
+			) : "no standby servers"
+		}
+	    </div>
+    }
+
+    return (
+	<div>
+	    <h2>Standbys</h2>
+	    <button onClick={create_standby} disabled={!can_create_standby || !walpos_valid}>Create new Standby</button> at LSN 
+            <input type="text" id="walpos_input" value={ walposStr } onChange={handleWalposChange} disabled={!can_create_standby}/>
+	    <input type="range" id="walpos_slider" min="0" max="100" steps="1" value={sliderValue}  onChange={handleSliderChange} disabled={!can_create_standby}/>
+	    <br/>
+	    { standbystatus }
+	</div>
+    );
+}
+
+function ServerList(props) {
+    const primary = props.serverStatus ? props.serverStatus.primary : null;
+    const standbys = props.serverStatus ? props.serverStatus.standbys : [];
+    const bucketSummary = props.bucketSummary;
+
+    var primarystatus = '';
+
+    function destroy_primary() {
+	const formdata = new FormData();
+	formdata.append("datadir", 'primary');
+	props.startOperation('Destroying primary...',
+			     fetch("/api/destroy_server", { method: 'POST', body: formdata }));
+    }    
+
+    function restore_primary() {
+	props.startOperation('Restoring primary...',
+			     fetch("/api/restore_primary", { method: 'POST' }));
+    }    
+    
+    if (primary)
+    {
+	primarystatus =
+	    <div>
+		<ServerStatus server={primary}/>
+		<button onClick={destroy_primary}>Destroy primary</button>
+	    </div>
+    }
+    else
+    {
+	primarystatus =
+	    <div>
+		no primary server<br/>
+		<button onClick={restore_primary}>Restore primary</button>
+	    </div>
+    }
+
+    return (
+	<>
+	    { primarystatus }
+	    <StandbyList standbys={standbys} startOperation={props.startOperation} bucketSummary={props.bucketSummary}/>
+	    <p className="todo">
+		Should we list the WAL safekeeper nodes here? Or are they part of the Storage? Or not visible to users at all?
+	    </p>
+	</>
+    );
+}
+
+function BucketSummary(props) {
+    const bucketSummary = props.bucketSummary;
+    const startOperation = props.startOperation;
+
+    function slicedice() {
+	startOperation('Slicing sequential WAL to per-relation WAL...',
+		       fetch("/api/slicedice", { method: 'POST' }));
+    }
+    
+    if (!bucketSummary.nonrelimages)
+    {
+	return <>loading...</>
+    }
+
+    return (
+	<div>
+	    <div>Base images at following WAL positions:
+		<ul>
+		    {bucketSummary.nonrelimages.map((img) => (
+			<li key={img}>{img}</li>
+		    ))}
+		</ul>
+	    </div>
+            Sliced WAL is available up to { bucketSummary.maxwal }<br/>
+	    Raw WAL is available up to { bucketSummary.maxseqwal }<br/>
+
+	    <br/>
+	    <button onClick={slicedice}>Slice & Dice WAL</button>
+	    <p className="todo">
+		Currently, the slicing or "sharding" of the WAL needs to be triggered manually, by clicking the above button.
+		<br/>
+		TODO: make it a continuous process that runs in the WAL safekeepers, or in the Page Servers, or as a standalone service.
+	    </p>
+	</div>
+    );
+}
+
+function ProgressIndicator()
+{
+    return (
+	<div>
+	    <Loader
+		type="Puff"
+		color="#00BFFF"
+		height={100}
+		width={100}
+	    />
+	</div>
+    )
+}
+
+function walpos_to_int(walpos)
+{
+    const [hi, lo] = walpos.split('/');
+
+    return parseInt(hi, 16) + parseInt(lo, 16);
+}
+
+function int_to_walpos(x)
+{
+    console.log('converting ' + x);
+    return (Math.floor((x / 0x100000000)).toString(16) + '/' + (x % 0x100000000).toString(16)).toUpperCase();
+}
+
+function OperationStatus(props) {
+    const lastOperation = props.lastOperation;
+    const inProgress = props.inProgress;
+    const operationResult = props.operationResult;
+
+    if (lastOperation)
+    {
+	return (
+	    <div><h2>Last operation:</h2>
+		<div>{lastOperation} { (!inProgress && lastOperation) ? 'done!' : '' }</div>
+		<div className='result'>
+		    {inProgress ? <ProgressIndicator/> : <pre>{operationResult}</pre>}
+		</div>
+	    </div>
+	);
+    }
+    else
+	return '';
+}
+
+function ActionButtons(props) {
+
+    const startOperation = props.startOperation;
+    const bucketSummary = props.bucketSummary;
+    
+    function reset_demo() {
+	startOperation('resetting everything...',
+		       fetch("/api/reset_demo", { method: 'POST' }));
+    }
+
+    function init_primary() {
+	startOperation('Initializing new primary...',
+		       fetch("/api/init_primary", { method: 'POST' }));
+    }
+
+    function zenith_push() {
+	startOperation('Pushing new base image...',
+		       fetch("/api/zenith_push", { method: 'POST' }));
+    }
+	
+    return (
+	<div>
+	    <p className="todo">
+		RESET DEMO deletes everything in the storage bucket, and stops and destroys all servers. This resets the whole demo environment to the initial state.
+	    </p>
+	    <button onClick={reset_demo}>RESET DEMO</button>
+	    <p className="todo">
+		Init Primary runs initdb to create a new primary server. Click this after Resetting the demo.
+	    </p>
+
+	    <button onClick={init_primary}>Init primary</button>
+
+	    <p className="todo">
+		Push Base Image stops the primary, copies the current state of the primary to the storage bucket as a new base backup, and restarts the primary.
+		<br/>
+		TODO: This should be handled by a continuous background process, probably running in the storage nodes. And without having to shut down the cluster, of course.
+	    </p>
+
+	    <button onClick={zenith_push}>Push base image</button>
+
+	</div>
+    );
+}
+
+function Sidenav(props)
+{
+    const toPage = (page) => (event) => {
+	//event.preventDefault()
+	props.switchPage(page);
+    };
+    return (
+	<div>
+	    <h3 className="sidenav-item">Menu</h3>
+	    <a href="#servers" onClick={toPage('servers')} className="sidenav-item">Servers</a>
+	    <a href="#storage" onClick={toPage('storage')} className="sidenav-item">Storage</a>
+	    <a href="#snapshots" onClick={toPage('snapshots')} className="sidenav-item">Snapshots</a>
+	    <a href="#demo" onClick={toPage('demo')} className="sidenav-item">Demo</a>
+	    <a href="#import" onClick={toPage('import')}  className="sidenav-item">Import / Export</a>
+	    <a href="#jobs" onClick={toPage('jobs')} className="sidenav-item">Jobs</a>
+	</div>
+    );
+}
+
+function App()
+{
+    const [page, setPage] = useState('servers');
+    const [serverStatus, setServerStatus] = useState({});
+    const [bucketSummary, setBucketSummary] = useState({});
+    const [lastOperation, setLastOperation] = useState('');
+    const [inProgress, setInProgress] = useState('');
+    const [operationResult, setOperationResult] = useState('');
+
+    useEffect(() => {
+	reloadStatus();
+    }, []);
+
+    function startOperation(operation, promise)
+    {
+	promise.then(result => result.text()).then(resultText => {
+	    operationFinished(resultText);
+	});
+	
+	setLastOperation(operation);
+	setInProgress(true);
+	setOperationResult('');
+    }
+
+    function operationFinished(result)
+    {
+	setInProgress(false);
+	setOperationResult(result);
+	reloadStatus();
+    }
+
+    function clearOperation()
+    {
+	setLastOperation('')
+	setInProgress('');
+	setOperationResult('');
+	console.log("cleared");
+    }
+    
+    function reloadStatus()
+    {
+	fetch('/api/server_status').then(res => res.json()).then(data => {
+	    setServerStatus(data);
+	});
+
+	fetch('/api/bucket_summary').then(res => res.json()).then(data => {
+	    setBucketSummary(data);
+	});
+    }
+
+    const content = () => {
+	console.log(page);
+	if (page === 'servers') {
+	    return (
+		<>
+		    <h1>Server status</h1>
+		    <ServerList startOperation={ startOperation }
+				serverStatus={ serverStatus }
+				bucketSummary={ bucketSummary }/>
+		</>
+	    );
+	} else if (page === 'storage') {
+	    return (
+		<>
+		    <h1>Storage bucket status</h1>
+		    <BucketSummary startOperation={ startOperation }
+				   bucketSummary={ bucketSummary }/>
+		</>
+	    );
+	} else if (page === 'snapshots') {
+	    return (
+		<>
+		    <h1>Snapshots</h1>
+		    <p className="todo">
+			In Zenith, snapshots are just specific points (LSNs) in the WAL history, with a label. A snapshot prevents garbage collecting old data that's still needed to reconstruct the database at that LSN.
+		    </p>
+		    <p className="todo">
+			TODO:
+			<ul>
+			    <li>List existing snapshots</li>
+			    <li>Create new snapshot manually, from current state or from a given LSN</li>
+			    <li>Drill into the WAL stream to see what have happened. Provide tools for e.g. finding point where a table was dropped</li>
+			    <li>Create snapshots automatically based on events in the WAL, like if you call pg_create_restore_point(() in the primary</li>
+			    <li>Launch new reader instance at a snapshot</li>
+			    <li>Export snapshot</li>
+			    <li>Rollback cluster to a snapshot</li>
+			</ul>
+		    </p>
+		</>
+	    );
+	} else if (page === 'demo') {
+	    return (
+		<>
+		    <h1>Misc actions</h1>
+		    <ActionButtons startOperation={ startOperation }
+				   bucketSummary={ bucketSummary }/>
+		</>
+	    );
+	} else if (page === 'import') {
+	    return (
+		<>
+		    <h1>Import & Export tools</h1>
+		    <p className="TODO">TODO:
+			<ul>
+			    <li>Initialize database from existing backup (pg_basebackup, WAL-G, pgbackrest)</li>
+			    <li>Initialize from a pg_dump or other SQL script</li>
+			    <li>Launch batch job to import data files from S3</li>
+			    <li>Launch batch job to export database with pg_dump to S3</li>
+			</ul>
+			These jobs can be run in against reader processing nodes. We can even
+			spawn a new reader node dedicated to a job, and destry it when the job is done.
+		    </p>
+		</>
+	    );
+	} else if (page === 'jobs') {
+	    return (
+		<>
+		    <h1>Batch jobs</h1>
+		    <p className="TODO">TODO:
+			<ul>
+			    <li>List running jobs launched from Import & Export tools</li>
+			    <li>List other batch jobs launched by the user</li>
+			    <li>Launch new batch jobs</li>
+			</ul>
+		    </p>
+		</>
+	    );
+	}
+    }
+
+    function switchPage(page)
+    {
+	console.log("topage " + page);
+	setPage(page)
+	clearOperation();
+    };
+
+    return (
+	<div className="row">
+	    <div className="sidenav">
+		<Sidenav switchPage={switchPage} className="column"/>
+	    </div>
+	    <div className="column">
+		<div>
+		    { content() }
+		</div>
+		<OperationStatus lastOperation={ lastOperation }
+				 inProgress = { inProgress }
+				 operationResult = { operationResult }/>
+	    </div>
+	</div>
+    );
+}
+
+ReactDOM.render(<App/>, document.getElementById('reactApp'));

mgmt-console/js/waldump.js

@@ -0,0 +1,105 @@
+import React, { useState, useEffect } from 'react';
+import ReactDOM from 'react-dom';
+import Loader from "react-loader-spinner";
+
+function walpos_to_int(walpos)
+{
+    const [hi, lo] = walpos.split('/');
+
+    return parseInt(hi, 16) + parseInt(lo, 16);
+}
+
+const palette = [
+    "#003f5c",
+    "#2f4b7c",
+    "#665191",
+    "#a05195",
+    "#d45087",
+    "#f95d6a",
+    "#ff7c43",
+    "#ffa600"];
+
+function WalRecord(props)
+{
+    const firstwalpos = props.firstwalpos;
+    const endwalpos = props.endwalpos;
+    const record = props.record;
+    const index = props.index;
+    const xidmap = props.xidmap;
+
+    const startpos = walpos_to_int(record.start)
+    const endpos = walpos_to_int(record.end)
+
+    const scale = 1000 / (16*1024*1024)
+    const startx = (startpos - firstwalpos) * scale;
+    const endx = (endpos - firstwalpos) * scale;
+
+    const xidindex = xidmap[record.xid];
+    const color = palette[index % palette.length];
+
+    const y = 5 + (xidindex) * 20 + (index % 2) * 2;
+    
+    return (
+	<line x1={ startx } y1={y} x2={endx} y2={y} stroke={ color } strokeWidth="5">
+	    <title>
+		start: { record.start } end: { record.end }
+	    </title>
+	</line>
+    )
+}
+
+function WalFile(props)
+{
+    const walContent = props.walContent;
+    const firstwalpos = props.firstwalpos;
+    const xidmap = props.xidmap;
+   
+    return <svg width="1000" height="200">
+	       {
+		   walContent.records ? 
+ 		       walContent.records.map((record, index) =>
+			   <WalRecord key={record.start} firstwalpos={firstwalpos} record={record} index={index} xidmap={xidmap}/>
+		       ) : "no records"
+	       }
+	   </svg>
+}
+
+function WalDumpApp()
+{
+    const [walContent, setWalContent] = useState({});
+
+    const filename = '00000001000000000000000C';
+
+    useEffect(() => {
+	fetch('/fetch_wal?filename='+filename).then(res => res.json()).then(data => {
+	    setWalContent(data);
+	});
+    }, []);
+
+    var firstwalpos = 0;
+    var endwalpos = 0;
+    var numxids = 0;
+    var xidmap = {};
+    if (walContent.records && walContent.records.length > 0)
+    {
+	firstwalpos = walpos_to_int(walContent.records[0].start);
+	endwalpos = firstwalpos + 16*1024*1024;
+
+	walContent.records.forEach(rec => {
+	    if (!xidmap[rec.xid])
+	    {
+		xidmap[rec.xid] = ++numxids;
+	    }
+	});
+    }
+
+    return (
+	<>
+	    <h2>{filename}</h2>
+	    <WalFile walContent={walContent} firstwalpos={firstwalpos} endwalpos={endwalpos} xidmap={xidmap}/>
+	</>
+    );
+}
+
+console.log('hey there');
+ReactDOM.render(<WalDumpApp/>, document.getElementById('waldump'));

mgmt-console/launch-google-cloud.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+# NOTE: You must set the following environment variables before running this:
+#  BASIC_AUTH_PASSWORD - basic http auth password
+#  S3_ACCESSKEY
+#  S3_SECRET
+
+
+S3_ENDPOINT=https://storage.googleapis.com S3_BUCKET=zenith-testbucket PATH=/home/heikki/pgsql-install/bin:$PATH flask run --host=0.0.0.0

mgmt-console/launch-local.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+# NOTE: You should set the BASIC_AUTH_PASSWORD environment variable before calling
+
+# Launch S3 server
+(cd ms3 && python3 -m ms3.app --listen-address=localhost) &
+
+FLASK_ENV=development S3_REGION=auto S3_ENDPOINT=http://localhost:9009 S3_BUCKET=zenith-testbucket PATH=/home/heikki/pgsql.fsmfork/bin:$PATH flask run --host=0.0.0.0

mgmt-console/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "starter-kit",
+  "version": "1.1.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "build": "webpack",
+    "start": "python app.py"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "react": "^17.0.1",
+    "react-dom": "^17.0.1",
+    "react-loader-spinner": "^4.0.0",
+    "react-router": "^5.2.0"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.13.1",
+    "@babel/preset-env": "^7.13.5",
+    "@babel/preset-react": "^7.12.13",
+    "babel-loader": "^8.2.2",
+    "webpack": "^5.24.2",
+    "webpack-cli": "^4.5.0"
+  }
+}

mgmt-console/templates/index.html

@@ -0,0 +1,58 @@
+<head>
+
+<style>
+  .status {
+      font-family: monospace;
+      background-color: lightgrey;
+  }
+  .shellcommand {
+      font-family: monospace;
+      background-color: lightgrey;
+  }
+  .result {
+      font-family: monospace;
+      background-color: lightgrey;
+      padding: 10px;
+  }
+
+
+  .todo   {font-style: italic;}
+
+
+  h1   {color: blue;}
+
+  .column {
+      float: left;
+      width: 50%;
+      padding: 10px;
+  }
+  /* Clear floats after the columns */
+  .row:after {
+      content: "";
+      display: table;
+      clear: both;
+  }
+
+  .sidenav {
+      float: left;
+      width: 150px;
+      padding: 10px;
+      background-color: pink;
+  }
+
+  .sidenav-item {
+      padding:10px 0px;
+      border:none;
+      display:block;
+  }
+
+</style>
+
+</head>
+
+<body>
+  <div id="reactApp"></div>
+
+  <!-- Attach React components -->
+  <script type="text/javascript" src="{{ url_for('static', filename='app_bundle.js') }}"></script>
+</body>

mgmt-console/templates/waldump.html

@@ -0,0 +1,46 @@
+<head>
+
+<style>
+  .status {
+      font-family: monospace;
+      background-color: lightgrey;
+  }
+  .shellcommand {
+      font-family: monospace;
+      background-color: lightgrey;
+  }
+  .result {
+      font-family: monospace;
+      background-color: lightgrey;
+      padding: 10px;
+  }
+h1   {color: blue;}
+p    {color: red;}
+
+* {
+  box-sizing: border-box;
+}
+
+.row {
+  display: flex;
+}
+
+/* Create two equal columns that sits next to each other */
+.column1 {
+  flex: 30%;
+  padding: 10px;
+}
+.column2 {
+  flex: 70%;
+  padding: 10px;
+}
+</style>
+
+</head>
+
+<body>
+  <div id="waldump"></div>
+
+  <!-- Attach React components -->
+  <script type="text/javascript" src="{{ url_for('static', filename='waldump_bundle.js') }}"></script>
+</body>

mgmt-console/waldump.py

@@ -0,0 +1,25 @@
+#
+# This file contains work-in-progress code to visualize WAL contents.
+#
+# This is the API endpoint that calls a 'zenith_wal_to_json' executable,
+# which is a hacked version of pg_waldump that prints information about the
+# records in JSON format. The code in js/waldump.js displays it.
+#
+
+import os
+import re
+from subprocess import PIPE, STDOUT, run, Popen
+
+def fetch_wal(request, s3bucket):
+    filename = request.args.get('filename')
+    if not re.match("^[A-Za-z0-9_]+$", filename):
+        raise Exception('invalid WAL filename: ' + filename)
+
+    # FIXME: this downloads the WAL file to current dir. Use a temp dir? Pipe?
+    s3bucket.download_file('walarchive/' + filename, filename)
+
+    result = run("zenith_wal_to_json " + filename, stdout=PIPE, universal_newlines=True, shell=True)
+
+    os.unlink(filename);
+
+    return result.stdout

mgmt-console/webpack.config.js

@@ -0,0 +1,27 @@
+var webpack = require('webpack');  
+module.exports = {  
+    entry: {
+	app: './js/app.js',
+	waldump: './js/waldump.js'
+    },
+    output: {
+	filename: "[name]_bundle.js",
+	path: __dirname + '/static'
+    },
+    module: {
+	rules: [
+	    {
+		test: /\.js?$/,
+		exclude: /node_modules/,
+		use: {
+		    loader: 'babel-loader',
+		    options: {
+			presets: ['@babel/preset-env']
+		    }
+		}
+	    }
+	]
+    },
+    plugins: [
+    ]
+};

mgmt-console/zenith.py

@@ -0,0 +1,179 @@
+#zenith.py
+import click
+import testgres
+import os
+
+from testgres import PostgresNode
+from tabulate import tabulate
+
+zenith_base_dir = '/home/anastasia/zenith/basedir'
+
+@click.group()
+def main():
+    """Run the Zenith CLI."""
+
+@click.group()
+def pg():
+    """Db operations
+
+        NOTE: 'database' here means one postgresql node
+    """
+
+@click.command(name='create')
+@click.option('--name', required=True)
+@click.option('-s', '--storage-name', help='Name of the storage',
+                                 default='zenith-local',
+                                 show_default=True)
+@click.option('--snapshot', help='init from the snapshot. Snap is a name or URL')
+@click.option('--no-start', is_flag=True, help='Do not start created node',
+                            default=False, show_default=True)
+def pg_create(name, storage_name, snapshot, no_start):
+    """Initialize the database"""
+    node = PostgresNode()
+    base_dir = os.path.join(zenith_base_dir, 'pg', name)
+    node = testgres.get_new_node(name, base_dir=base_dir)
+    # TODO skip init, instead of that link node with storage or upload it from snapshot
+    node.init()
+    if(no_start==False):
+        node.start()
+
+@click.command(name='start')
+@click.option('--name', required=True)
+@click.option('--snapshot')
+@click.option('--read-only', is_flag=True, help='Start read-only node', show_default=True)
+def pg_start(name, snapshot, read_only):
+    """Start the database"""
+    node = PostgresNode()
+    base_dir = os.path.join(zenith_base_dir, 'pg', name)
+    node = testgres.get_new_node(name, base_dir=base_dir)
+    # TODO pass snapshot as a parameter
+    node.start()
+
+@click.command(name='stop')
+@click.option('--name', required=True)
+def pg_stop(name):
+    """Stop the database"""
+    node = PostgresNode()
+    base_dir = os.path.join(zenith_base_dir, 'pg', name)
+    node = testgres.get_new_node(name, base_dir=base_dir)
+    node.stop()
+
+@click.command(name='destroy')
+@click.option('--name', required=True)
+def pg_destroy(name):
+    """Drop the database"""
+    node = PostgresNode()
+    base_dir = os.path.join(zenith_base_dir, 'pg', name)
+    node = testgres.get_new_node(name, base_dir=base_dir)
+    node.cleanup()
+
+@click.command(name='list')
+def pg_list():
+    """List existing databases"""
+    dirs = os.listdir(os.path.join(zenith_base_dir, 'pg'))
+    path={}
+    status={}
+    data=[]
+
+    for dirname in dirs:
+        path[dirname] = os.path.join(zenith_base_dir, 'pg', dirname)
+        fname = os.path.join( path[dirname], 'data/postmaster.pid')
+        try:
+            f = open(fname,'r')
+            status[dirname] = f.readlines()[-1]
+        except OSError as err:
+            status[dirname]='inactive'
+        data.append([dirname , status[dirname], path[dirname]])
+
+    print(tabulate(data, headers=['Name', 'Status', 'Path']))
+
+pg.add_command(pg_create)
+pg.add_command(pg_destroy)
+pg.add_command(pg_start)   
+pg.add_command(pg_stop)   
+pg.add_command(pg_list)
+
+
+
+@click.group()
+def storage():
+    """Storage operations"""
+
+@click.command(name='attach')
+@click.option('--name')
+def storage_attach(name):
+    """Attach the storage"""
+
+@click.command(name='detach')
+@click.option('--name')
+@click.option('--force', is_flag=True, show_default=True)
+def storage_detach(name):
+    """Detach the storage"""
+
+@click.command(name='list')
+def storage_list():
+    """List existing storages"""
+
+storage.add_command(storage_attach)
+storage.add_command(storage_detach)
+storage.add_command(storage_list)
+
+@click.group()
+def snapshot():
+    """Snapshot operations"""
+
+@click.command(name='create')
+def snapshot_create():
+    """Create new snapshot"""
+
+@click.command(name='destroy')
+def snapshot_destroy():
+    """Destroy the snapshot"""
+
+@click.command(name='pull')
+def snapshot_pull():
+    """Pull remote snapshot"""
+
+@click.command(name='push')
+def snapshot_push():
+    """Push snapshot to remote"""
+
+@click.command(name='import')
+def snapshot_import():
+    """Convert given format to zenith snapshot"""
+
+@click.command(name='export')
+def snapshot_export():
+    """Convert zenith snapshot to PostgreSQL compatible format"""
+
+snapshot.add_command(snapshot_create)
+snapshot.add_command(snapshot_destroy)
+snapshot.add_command(snapshot_pull)
+snapshot.add_command(snapshot_push)
+snapshot.add_command(snapshot_import)
+snapshot.add_command(snapshot_export)
+
+@click.group()
+def wal():
+    """WAL operations"""
+
+@click.command()
+def wallist(name="list"):
+    """List WAL files"""
+
+wal.add_command(wallist)
+
+
+@click.command()
+def console():
+    """Open web console"""
+
+main.add_command(pg)
+main.add_command(storage)
+main.add_command(snapshot)
+main.add_command(wal)
+main.add_command(console)
+
+
+if __name__ == '__main__':
+    main()

monitoring/docker-compose.yml

@@ -1,25 +0,0 @@
-version: "3"
-services:
-
-  prometheus:
-    container_name: prometheus
-    image: prom/prometheus:latest
-    volumes:
-      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
-    # ports:
-    #   - "9090:9090"
-    # TODO: find a proper portable solution
-    network_mode: "host"
-
-  grafana:
-    image: grafana/grafana:latest
-    volumes:
-      - ./grafana.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
-    environment:
-      - GF_AUTH_ANONYMOUS_ENABLED=true
-      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
-      - GF_AUTH_DISABLE_LOGIN_FORM=true
-    # ports:
-    #   - "3000:3000"
-    # TODO: find a proper portable solution
-    network_mode: "host"

monitoring/grafana.yaml

@@ -1,12 +0,0 @@
-apiVersion: 1
-
-datasources:
-- name: Prometheus
-  type: prometheus
-  access: proxy
-  orgId: 1
-  url: http://localhost:9090
-  basicAuth: false
-  isDefault: false
-  version: 1
-  editable: false

monitoring/prometheus.yaml

@@ -1,5 +0,0 @@
-scrape_configs:
-  - job_name: 'default'
-    scrape_interval: 10s
-    static_configs:
-      - targets: ['localhost:9898']

pageserver/Cargo.toml

@@ -7,12 +7,13 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bookfile = "^0.3"
 chrono = "0.4.19"
+crossbeam-channel = "0.5.0"
 rand = "0.8.3"
 regex = "1.4.5"
-bytes = { version = "1.0.1", features = ['serde'] }
+bytes = "1.0.1"
 byteorder = "1.4.3"
+fs2 = "0.4.3"
 futures = "0.3.13"
 lazy_static = "1.4.0"
 slog-stdlog = "4.1.0"
@@ -22,29 +23,14 @@ slog-term = "2.8.0"
 slog = "2.7.0"
 log = "0.4.14"
 clap = "2.33.0"
+termion = "1.5.6"
+tui = "0.14.0"
 daemonize = "0.4.1"
-rust-s3 = { version = "0.27.0-rc4", features = ["no-verify-ssl"] }
-tokio = { version = "1.5.0", features = ["full"] }
-tokio-stream = { version = "0.1.5" }
-postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-# by default rust-rocksdb tries to build a lot of compression algos. Use lz4 only for now as it is simplest dependency.
-rocksdb = { version = "0.16.0", features = ["lz4"], default-features = false }
+rust-s3 = { git = "https://github.com/hlinnaka/rust-s3", features = ["no-verify-ssl"] }
+tokio = { version = "1.3.0", features = ["full"] }
+tokio-stream = { version = "0.1.4" }
+tokio-postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" }
+postgres-protocol = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" }
+postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" }
 anyhow = "1.0"
 crc32c = "0.6.0"
-walkdir = "2"
-thiserror = "1.0"
-hex = "0.4.3"
-tar = "0.4.33"
-humantime = "2.1.0"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1"
-fs_extra = "1.2.0"
-toml = "0.5"
-scopeguard = "1.1.0"
-
-postgres_ffi = { path = "../postgres_ffi" }
-zenith_metrics = { path = "../zenith_metrics" }
-zenith_utils = { path = "../zenith_utils" }
-workspace_hack = { path = "../workspace_hack" }

pageserver/README

@@ -1,4 +1,82 @@
-## Page server architecture
+Page Server
+===========
+
+
+How to test
+-----------
+
+
+1. Compile and install Postgres from this repository (there are
+   modifications, so vanilla Postgres won't do)
+
+    ./configure --prefix=/home/heikki/zenith-install
+
+2. Compile the page server
+
+    cd pageserver
+    cargo build
+
+3. Create another "dummy" cluster that will be used by the page server when it applies
+   the WAL records. (shouldn't really need this, getting rid of it is a TODO):
+
+    /home/heikki/zenith-install/bin/initdb -D /data/zenith-dummy
+
+
+4. Initialize and start a new postgres cluster
+
+    /home/heikki/zenith-install/bin/initdb -D /data/zenith-test-db --username=postgres
+    /home/heikki/zenith-install/bin/postgres -D /data/zenith-test-db
+
+5. In another terminal, start the page server.
+
+    PGDATA=/data/zenith-dummy PATH=/home/heikki/zenith-install/bin:$PATH ./target/debug/pageserver
+
+   It should connect to the postgres instance using streaming replication, and print something
+   like this:
+
+    $ PGDATA=/data/zenith-dummy PATH=/home/heikki/zenith-install/bin:$PATH ./target/debug/pageserver
+    Starting WAL receiver
+    connecting...
+    Starting page server on 127.0.0.1:5430
+    connected!
+    page cache is empty
+
+6. You can now open another terminal and issue DDL commands. Generated WAL records will
+   be streamed to the page servers, and attached to blocks that they apply to in its
+   page cache
+
+    $ psql postgres -U postgres
+    psql (14devel)
+    Type "help" for help.
+    
+    postgres=# create table mydata (i int4);
+    CREATE TABLE
+    postgres=# insert into mydata select g from generate_series(1,100) g;
+    INSERT 0 100
+    postgres=# 
+
+7. The GetPage@LSN interface to the compute nodes isn't working yet, but to simulate
+   that, the page server generates a test GetPage@LSN call every 5 seconds on a random
+   block that's in the page cache. In a few seconds, you should see output from that:
+
+    testing GetPage@LSN for block 0
+    WAL record at LSN 23584576 initializes the page
+    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167DF40
+    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167DF80
+    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167DFC0
+    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E018
+    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E058
+    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E098
+    2021-03-19 11:03:13.791 EET [11439] LOG:  applied WAL record at 0/167E0D8
+    2021-03-19 11:03:13.792 EET [11439] LOG:  applied WAL record at 0/167E118
+    2021-03-19 11:03:13.792 EET [11439] LOG:  applied WAL record at 0/167E158
+    2021-03-19 11:03:13.792 EET [11439] LOG:  applied WAL record at 0/167E198
+    applied 10 WAL records to produce page image at LSN 18446744073709547246
+
+
+
+Architecture
+============
 
 The Page Server is responsible for all operations on a number of
 "chunks" of relation data. A chunk corresponds to a PostgreSQL
@@ -6,10 +84,8 @@ relation segment (i.e. one max. 1 GB file in the data directory), but
 it holds all the different versions of every page in the segment that
 are still needed by the system.
 
-Currently we do not specifically organize data in chunks.
-All page images and corresponding WAL records are stored as entries in a key-value storage,
-where StorageKey is a zenith_timeline_id + BufferTag + LSN.
-
+Determining which chunk each Page Server holds is handled elsewhere. (TODO:
+currently, there is only one Page Server which holds all chunks)
 
 The Page Server has a few different duties:
 
@@ -78,33 +154,11 @@ and stores them to the page cache.
 Page Cache
 ----------
 
-The Page Cache is a switchboard to access different Repositories.
+The Page Cache is a data structure, to hold all the different page versions.
+It is accessed by all the other threads, to perform their duties.
 
-#### Repository
-Repository corresponds to one .zenith directory.
-Repository is needed to manage Timelines.
-
-#### Timeline
-Timeline is a page cache workhorse that accepts page changes
-and serves get_page_at_lsn() and get_rel_size() requests.
-Note: this has nothing to do with PostgreSQL WAL timeline.
-
-#### Branch
-We can create branch at certain LSN.
-Each Branch lives in a corresponding timeline and has an ancestor.
-
-To get full snapshot of data at certain moment we need to traverse timeline and its ancestors.
-
-#### ObjectRepository
-ObjectRepository implements Repository and has associated ObjectStore and WAL redo service.
-
-#### ObjectStore
-ObjectStore is an interface for key-value store for page images and wal records.
-Currently it has one implementation - RocksDB.
-
-#### WAL redo service
-WAL redo service - service that runs PostgreSQL in a special wal_redo mode
-to apply given WAL records over an old page image and return new page image.
+Currently, the page cache is implemented fully in-memory. TODO: Store it
+on disk. Define a file format.
 
 
 TODO: Garbage Collection / Compaction
@@ -123,7 +177,3 @@ The backup service is responsible for periodically pushing the chunks to S3.
 TODO: How/when do restore from S3? Whenever we get a GetPage@LSN request for
 a chunk we don't currently have? Or when an external Control Plane tells us?
 
-TODO: Sharding
---------------------
-
-We should be able to run multiple Page Servers that handle sharded data.

pageserver/build.rs

@@ -0,0 +1,41 @@
+//
+//   Triggers postgres build if there is no postgres binary present at
+// 'REPO_ROOT/tmp_install/bin/postgres'.
+//
+//   I can see a lot of disadvantages with such automatization and main
+// advantage here is ability to build everything and run integration tests
+// in a bare repo by running 'cargo test'.
+//
+//   We can interceipt whether it is debug or release build and run
+// corresponding pg build. But it seems like an overkill for now.
+//
+// Problem #1 -- language server in my editor likes calling 'cargo build'
+// by himself. So if I delete tmp_install directory it would magically reappear
+// after some time. During this compilation 'cargo build' may whine about
+// "waiting for file lock on build directory".
+//
+// Problem #2 -- cargo build would run this only if something is changed in
+// the crate.
+//
+//   And generally speaking postgres is not a build dependency for the pageserver,
+// just for integration tests. So let's not mix that. I'll leave this file in
+// place for some time just in case if anybody would start doing the same.
+//
+
+// use std::path::Path;
+// use std::process::{Command};
+
+fn main() {
+    // // build some postgres if it is not done none yet
+    // if !Path::new("../tmp_install/bin/postgres").exists() {
+    //     let make_res = Command::new("make")
+    //         .arg("postgres")
+    //         .env_clear()
+    //         .status()
+    //         .expect("failed to execute 'make postgres'");
+
+    //     if !make_res.success() {
+    //         panic!("postgres build failed");
+    //     }
+    // }
+}

pageserver/launch.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+#
+# Set up a simple Compute Node + Page Server combination locally.
+#
+# NOTE: This doesn't clean up between invocations. You'll need to manually:
+#
+# - Kill any previous 'postgres' and 'pageserver' processes
+# - Clear the S3 bucket
+# - Remove the 'zenith-pgdata' directory
+
+
+set -e
+
+# Set up some config.
+#
+# CHANGE THESE ACCORDING TO YOUR S3 INSTALLATION
+export S3_REGION=auto
+export S3_ENDPOINT=https://localhost:9000
+export S3_ACCESSKEY=minioadmin
+export S3_SECRET=pikkunen
+export S3_BUCKET=zenith-testbucket
+
+
+COMPUTE_NODE_PGDATA=zenith-pgdata
+
+
+# 1. Initialize a cluster.
+initdb -D $COMPUTE_NODE_PGDATA -U zenith
+
+echo "port=65432" >> $COMPUTE_NODE_PGDATA/postgresql.conf
+echo "log_connections=on" >> $COMPUTE_NODE_PGDATA/postgresql.conf
+
+# Use a small shared_buffers, so that we hit the Page Server more
+# easily.
+echo "shared_buffers = 1MB" >> $COMPUTE_NODE_PGDATA/postgresql.conf
+
+# TODO: page server should use a replication slot, or some other mechanism
+# to make sure that the primary doesn't lose data that the page server still
+# needs. (The WAL safekeepers should ensure that)
+echo "wal_keep_size=10GB" >> $COMPUTE_NODE_PGDATA/postgresql.conf
+
+# Tell the Postgres server how to connect to the Page Server
+echo "page_server_connstring='host=localhost port=5430'" >> $COMPUTE_NODE_PGDATA/postgresql.conf
+
+
+# 2. Run zenith_push to push a base backup fo the database to an S3 bucket. The
+# Page Server will read it from there
+zenith_push -D $COMPUTE_NODE_PGDATA
+
+
+# 3. Launch page server
+rm -rf /tmp/pgdata-dummy
+initdb -N -D /tmp/pgdata-dummy
+PGDATA=/tmp/pgdata-dummy ./target/debug/pageserver  &
+
+# 4. Start up the Postgres server
+postgres -D $COMPUTE_NODE_PGDATA &
+
+
+echo "ALL SET! You can now connect to Postgres with something like:"
+echo ""
+echo 'psql "dbname=postgres host=localhost user=zenith port=65432"'

pageserver/src/basebackup.rs

@@ -1,290 +0,0 @@
-//!
-//! Generate a tarball with files needed to bootstrap ComputeNode.
-//!
-//! TODO: this module has nothing to do with PostgreSQL pg_basebackup.
-//! It could use a better name.
-//!
-//! Stateless Postgres compute node is launched by sending a tarball
-//! which contains non-relational data (multixacts, clog, filenodemaps, twophase files),
-//! generated pg_control and dummy segment of WAL.
-//! This module is responsible for creation of such tarball
-//! from data stored in object storage.
-//!
-use bytes::{BufMut, BytesMut};
-use log::*;
-use std::io;
-use std::io::Write;
-use std::sync::Arc;
-use std::time::SystemTime;
-use tar::{Builder, EntryType, Header};
-
-use crate::relish::*;
-use crate::repository::Timeline;
-use postgres_ffi::xlog_utils::*;
-use postgres_ffi::*;
-use zenith_utils::lsn::Lsn;
-
-/// This is short-living object only for the time of tarball creation,
-/// created mostly to avoid passing a lot of parameters between various functions
-/// used for constructing tarball.
-pub struct Basebackup<'a> {
-    ar: Builder<&'a mut dyn Write>,
-    timeline: &'a Arc<dyn Timeline>,
-    lsn: Lsn,
-    prev_record_lsn: Lsn,
-}
-
-impl<'a> Basebackup<'a> {
-    pub fn new(
-        write: &'a mut dyn Write,
-        timeline: &'a Arc<dyn Timeline>,
-        lsn: Lsn,
-        prev_record_lsn: Lsn,
-    ) -> Basebackup<'a> {
-        Basebackup {
-            ar: Builder::new(write),
-            timeline,
-            lsn,
-            prev_record_lsn,
-        }
-    }
-
-    pub fn send_tarball(&mut self) -> anyhow::Result<()> {
-        // Create pgdata subdirs structure
-        for dir in pg_constants::PGDATA_SUBDIRS.iter() {
-            info!("send subdir {:?}", *dir);
-            let header = new_tar_header_dir(*dir)?;
-            self.ar.append(&header, &mut io::empty())?;
-        }
-
-        // Send empty config files.
-        for filepath in pg_constants::PGDATA_SPECIAL_FILES.iter() {
-            if *filepath == "pg_hba.conf" {
-                let data = pg_constants::PG_HBA.as_bytes();
-                let header = new_tar_header(&filepath, data.len() as u64)?;
-                self.ar.append(&header, &data[..])?;
-            } else {
-                let header = new_tar_header(&filepath, 0)?;
-                self.ar.append(&header, &mut io::empty())?;
-            }
-        }
-
-        // Gather non-relational files from object storage pages.
-        for obj in self.timeline.list_nonrels(self.lsn)? {
-            match obj {
-                RelishTag::Slru { slru, segno } => {
-                    self.add_slru_segment(slru, segno)?;
-                }
-                RelishTag::FileNodeMap { spcnode, dbnode } => {
-                    self.add_relmap_file(spcnode, dbnode)?;
-                }
-                RelishTag::TwoPhase { xid } => {
-                    self.add_twophase_file(xid)?;
-                }
-                _ => {}
-            }
-        }
-
-        // Generate pg_control and bootstrap WAL segment.
-        self.add_pgcontrol_file()?;
-        self.ar.finish()?;
-        debug!("all tarred up!");
-        Ok(())
-    }
-
-    //
-    // Generate SLRU segment files from repository.
-    //
-    fn add_slru_segment(&mut self, slru: SlruKind, segno: u32) -> anyhow::Result<()> {
-        let seg_size = self
-            .timeline
-            .get_relish_size(RelishTag::Slru { slru, segno }, self.lsn)?;
-
-        if seg_size == None {
-            trace!(
-                "SLRU segment {}/{:>04X} was truncated",
-                slru.to_str(),
-                segno
-            );
-            return Ok(());
-        }
-
-        let nblocks = seg_size.unwrap();
-
-        let mut slru_buf: Vec<u8> =
-            Vec::with_capacity(nblocks as usize * pg_constants::BLCKSZ as usize);
-        for blknum in 0..nblocks {
-            let img = self.timeline.get_page_at_lsn_nowait(
-                RelishTag::Slru { slru, segno },
-                blknum,
-                self.lsn,
-            )?;
-            assert!(img.len() == pg_constants::BLCKSZ as usize);
-
-            slru_buf.extend_from_slice(&img);
-        }
-
-        let segname = format!("{}/{:>04X}", slru.to_str(), segno);
-        let header = new_tar_header(&segname, slru_buf.len() as u64)?;
-        self.ar.append(&header, slru_buf.as_slice())?;
-
-        trace!("Added to basebackup slru {} relsize {}", segname, nblocks);
-        Ok(())
-    }
-
-    //
-    // Extract pg_filenode.map files from repository
-    // Along with them also send PG_VERSION for each database.
-    //
-    fn add_relmap_file(&mut self, spcnode: u32, dbnode: u32) -> anyhow::Result<()> {
-        let img = self.timeline.get_page_at_lsn_nowait(
-            RelishTag::FileNodeMap { spcnode, dbnode },
-            0,
-            self.lsn,
-        )?;
-        let path = if spcnode == pg_constants::GLOBALTABLESPACE_OID {
-            let dst_path = "PG_VERSION";
-            let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
-            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
-            self.ar.append(&header, &version_bytes[..])?;
-
-            let dst_path = format!("global/PG_VERSION");
-            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
-            self.ar.append(&header, &version_bytes[..])?;
-
-            String::from("global/pg_filenode.map") // filenode map for global tablespace
-        } else {
-            // User defined tablespaces are not supported
-            assert!(spcnode == pg_constants::DEFAULTTABLESPACE_OID);
-
-            // Append dir path for each database
-            let path = format!("base/{}", dbnode);
-            let header = new_tar_header_dir(&path)?;
-            self.ar.append(&header, &mut io::empty())?;
-
-            let dst_path = format!("base/{}/PG_VERSION", dbnode);
-            let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
-            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
-            self.ar.append(&header, &version_bytes[..])?;
-
-            format!("base/{}/pg_filenode.map", dbnode)
-        };
-        assert!(img.len() == 512);
-        let header = new_tar_header(&path, img.len() as u64)?;
-        self.ar.append(&header, &img[..])?;
-        Ok(())
-    }
-
-    //
-    // Extract twophase state files
-    //
-    fn add_twophase_file(&mut self, xid: TransactionId) -> anyhow::Result<()> {
-        if let Ok(img) =
-            self.timeline
-                .get_page_at_lsn_nowait(RelishTag::TwoPhase { xid }, 0, self.lsn)
-        {
-            let mut buf = BytesMut::new();
-            buf.extend_from_slice(&img[..]);
-            let crc = crc32c::crc32c(&img[..]);
-            buf.put_u32_le(crc);
-            let path = format!("pg_twophase/{:>08X}", xid);
-            let header = new_tar_header(&path, buf.len() as u64)?;
-            self.ar.append(&header, &buf[..])?;
-        }
-        Ok(())
-    }
-
-    //
-    // Add generated pg_control file and bootstrap WAL segment.
-    // Also send zenith.signal file with extra bootstrap data.
-    //
-    fn add_pgcontrol_file(&mut self) -> anyhow::Result<()> {
-        let checkpoint_bytes =
-            self.timeline
-                .get_page_at_lsn_nowait(RelishTag::Checkpoint, 0, self.lsn)?;
-        let pg_control_bytes =
-            self.timeline
-                .get_page_at_lsn_nowait(RelishTag::ControlFile, 0, self.lsn)?;
-        let mut pg_control = ControlFileData::decode(&pg_control_bytes)?;
-        let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
-
-        // Generate new pg_control and WAL needed for bootstrap
-        let checkpoint_segno = self.lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE);
-        let checkpoint_lsn = XLogSegNoOffsetToRecPtr(
-            checkpoint_segno,
-            XLOG_SIZE_OF_XLOG_LONG_PHD as u32,
-            pg_constants::WAL_SEGMENT_SIZE,
-        );
-        checkpoint.redo = self.lsn.0 + self.lsn.calc_padding(8u32);
-
-        //reset some fields we don't want to preserve
-        //TODO Check this.
-        //We may need to determine the value from twophase data.
-        checkpoint.oldestActiveXid = 0;
-
-        //save new values in pg_control
-        pg_control.checkPoint = checkpoint_lsn;
-        pg_control.checkPointCopy = checkpoint;
-        pg_control.state = pg_constants::DB_SHUTDOWNED;
-
-        // add zenith.signal file
-        self.ar.append(
-            &new_tar_header("zenith.signal", 8)?,
-            &self.prev_record_lsn.0.to_le_bytes()[..],
-        )?;
-
-        //send pg_control
-        let pg_control_bytes = pg_control.encode();
-        let header = new_tar_header("global/pg_control", pg_control_bytes.len() as u64)?;
-        self.ar.append(&header, &pg_control_bytes[..])?;
-
-        //send wal segment
-        let wal_file_name = XLogFileName(
-            1, // FIXME: always use Postgres timeline 1
-            checkpoint_segno,
-            pg_constants::WAL_SEGMENT_SIZE,
-        );
-        let wal_file_path = format!("pg_wal/{}", wal_file_name);
-        let header = new_tar_header(&wal_file_path, pg_constants::WAL_SEGMENT_SIZE as u64)?;
-        let wal_seg = generate_wal_segment(&pg_control);
-        assert!(wal_seg.len() == pg_constants::WAL_SEGMENT_SIZE);
-        self.ar.append(&header, &wal_seg[..])?;
-        Ok(())
-    }
-}
-
-//
-// Create new tarball entry header
-//
-fn new_tar_header(path: &str, size: u64) -> anyhow::Result<Header> {
-    let mut header = Header::new_gnu();
-    header.set_size(size);
-    header.set_path(path)?;
-    header.set_mode(0b110000000); // -rw-------
-    header.set_mtime(
-        // use currenttime as last modified time
-        SystemTime::now()
-            .duration_since(SystemTime::UNIX_EPOCH)
-            .unwrap()
-            .as_secs(),
-    );
-    header.set_cksum();
-    Ok(header)
-}
-
-fn new_tar_header_dir(path: &str) -> anyhow::Result<Header> {
-    let mut header = Header::new_gnu();
-    header.set_size(0);
-    header.set_path(path)?;
-    header.set_mode(0o755); // -rw-------
-    header.set_entry_type(EntryType::dir());
-    header.set_mtime(
-        // use currenttime as last modified time
-        SystemTime::now()
-            .duration_since(SystemTime::UNIX_EPOCH)
-            .unwrap()
-            .as_secs(),
-    );
-    header.set_cksum();
-    Ok(header)
-}

pageserver/src/bin/cli/main.rs

@@ -0,0 +1,43 @@
+use anyhow::Result;
+use clap::{App, AppSettings};
+
+pub mod pg;
+pub mod snapshot;
+pub mod storage;
+mod subcommand;
+
+fn main() -> Result<()> {
+    let cli_commands = subcommand::ClapCommands {
+        commands: vec![
+            Box::new(pg::PgCmd {
+                clap_cmd: clap::SubCommand::with_name("pg"),
+            }),
+            Box::new(storage::StorageCmd {
+                clap_cmd: clap::SubCommand::with_name("storage"),
+            }),
+            Box::new(snapshot::SnapshotCmd {
+                clap_cmd: clap::SubCommand::with_name("snapshot"),
+            }),
+        ],
+    };
+
+    let matches = App::new("zenith")
+        .about("Zenith CLI")
+        .version("1.0")
+        .setting(AppSettings::SubcommandRequiredElseHelp)
+        .subcommands(cli_commands.generate())
+        .get_matches();
+
+    if let Some(subcommand) = matches.subcommand_name() {
+        println!("'git {}' was used", subcommand);
+    }
+
+    match matches.subcommand() {
+        ("pg", Some(sub_args)) => cli_commands.commands[0].run(sub_args.clone())?,
+        ("storage", Some(sub_args)) => cli_commands.commands[1].run(sub_args.clone())?,
+        ("snapshot", Some(sub_args)) => cli_commands.commands[2].run(sub_args.clone())?,
+        ("", None) => println!("No subcommand"),
+        _ => unreachable!(),
+    }
+    Ok(())
+}

pageserver/src/bin/cli/pg.rs

@@ -0,0 +1,105 @@
+use anyhow::Result;
+use clap::{App, AppSettings, Arg};
+
+use crate::subcommand;
+
+pub struct PgCmd<'a> {
+    pub clap_cmd: clap::App<'a, 'a>,
+}
+
+impl subcommand::SubCommand for PgCmd<'_> {
+    fn gen_clap_command(&self) -> clap::App {
+        let c = self.clap_cmd.clone();
+        c.about("Operations with zenith compute nodes")
+            .setting(AppSettings::SubcommandRequiredElseHelp)
+            .subcommand(App::new("list").about("List existing compute nodes"))
+            .subcommand(
+                App::new("create")
+                    .about(
+                        "Create (init) new data directory using given storage and start postgres",
+                    )
+                    .arg(
+                        Arg::with_name("name")
+                            .short("n")
+                            .long("name")
+                            .takes_value(true)
+                            .help("Name of the compute node"),
+                    )
+                    .arg(
+                        Arg::with_name("storage")
+                            .short("s")
+                            .long("storage")
+                            .takes_value(true)
+                            .help("Name of the storage node to use"),
+                    )
+                    //TODO should it be just name of uploaded snapshot or some path?
+                    .arg(
+                        Arg::with_name("snapshot")
+                            .long("snapshot")
+                            .takes_value(true)
+                            .help("Name of the snapshot to use"),
+                    )
+                    .arg(
+                        Arg::with_name("nostart")
+                            .long("no-start")
+                            .takes_value(false)
+                            .help("Don't start postgres on the created node"),
+                    ),
+            )
+            .subcommand(
+                App::new("destroy")
+                    .about("Stop postgres and destroy node's data directory")
+                    .arg(
+                        Arg::with_name("name")
+                            .short("n")
+                            .long("name")
+                            .takes_value(true)
+                            .help("Name of the compute node"),
+                    ),
+            )
+            .subcommand(
+                App::new("start")
+                    .about("Start postgres on the given node")
+                    .arg(
+                        Arg::with_name("name")
+                            .short("n")
+                            .long("name")
+                            .takes_value(true)
+                            .help("Name of the compute node"),
+                    )
+                    .arg(
+                        Arg::with_name("replica")
+                            .long("replica")
+                            .takes_value(false)
+                            .help("Start the compute node as replica"),
+                    ),
+            )
+            .subcommand(
+                App::new("stop")
+                    .about("Stop postgres on the given node")
+                    .arg(
+                        Arg::with_name("name")
+                            .short("n")
+                            .long("name")
+                            .takes_value(true)
+                            .help("Name of the compute node"),
+                    ),
+            )
+            .subcommand(
+                App::new("show")
+                    .about("Show info about the given node")
+                    .arg(
+                        Arg::with_name("name")
+                            .short("n")
+                            .long("name")
+                            .takes_value(true)
+                            .help("Name of the compute node"),
+                    ),
+            )
+    }
+
+    fn run(&self, args: clap::ArgMatches) -> Result<()> {
+        println!("Run PgCmd with args {:?}", args);
+        Ok(())
+    }
+}

pageserver/src/bin/cli/snapshot.rs

@@ -0,0 +1,27 @@
+use anyhow::Result;
+use clap::{App, AppSettings, Arg};
+
+use crate::subcommand;
+
+pub struct SnapshotCmd<'a> {
+    pub clap_cmd: clap::App<'a, 'a>,
+}
+
+impl subcommand::SubCommand for SnapshotCmd<'_> {
+    fn gen_clap_command(&self) -> clap::App {
+        let c = self.clap_cmd.clone();
+        c.about("Operations with zenith snapshots")
+            .setting(AppSettings::SubcommandRequiredElseHelp)
+            .subcommand(App::new("list"))
+            .subcommand(App::new("create").arg(Arg::with_name("pgdata").required(true)))
+            .subcommand(App::new("destroy"))
+            .subcommand(App::new("start"))
+            .subcommand(App::new("stop"))
+            .subcommand(App::new("show"))
+    }
+
+    fn run(&self, args: clap::ArgMatches) -> Result<()> {
+        println!("Run SnapshotCmd with args {:?}", args);
+        Ok(())
+    }
+}

pageserver/src/bin/cli/storage.rs

@@ -0,0 +1,25 @@
+use anyhow::Result;
+use clap::{App, AppSettings};
+
+use crate::subcommand;
+
+pub struct StorageCmd<'a> {
+    pub clap_cmd: clap::App<'a, 'a>,
+}
+
+impl subcommand::SubCommand for StorageCmd<'_> {
+    fn gen_clap_command(&self) -> clap::App {
+        let c = self.clap_cmd.clone();
+        c.about("Operations with zenith storage nodes")
+            .setting(AppSettings::SubcommandRequiredElseHelp)
+            .subcommand(App::new("list"))
+            .subcommand(App::new("attach"))
+            .subcommand(App::new("detach"))
+            .subcommand(App::new("show"))
+    }
+
+    fn run(&self, args: clap::ArgMatches) -> Result<()> {
+        println!("Run StorageCmd with args {:?}", args);
+        Ok(())
+    }
+}

pageserver/src/bin/cli/subcommand.rs

@@ -0,0 +1,29 @@
+use anyhow::Result;
+
+/// All subcommands need to implement this interface.
+pub trait SubCommand {
+    /// Generates the cli-config that Clap requires for the subcommand.
+    fn gen_clap_command(&self) -> clap::App;
+
+    /// Runs the body of the subcommand.
+    fn run(&self, args: clap::ArgMatches) -> Result<()>;
+}
+
+/// A struct which holds a vector of heap-allocated `Box`es of trait objects all of which must
+/// implement the `SubCommand` trait, but other than that, can be of any type.
+pub struct ClapCommands {
+    pub commands: Vec<Box<dyn SubCommand>>,
+}
+
+impl ClapCommands {
+    /// Generates a vector of `clap::Apps` that can be passed into clap's `.subcommands()` method in
+    /// order to generate the full CLI.
+    pub fn generate(&self) -> Vec<clap::App> {
+        let mut v: Vec<clap::App> = Vec::new();
+
+        for command in self.commands.iter() {
+            v.push(command.gen_clap_command());
+        }
+        v
+    }
+}

pageserver/src/bin/pageserver.rs

@@ -3,317 +3,146 @@
 //
 
 use log::*;
-use serde::{Deserialize, Serialize};
-use std::{
-    env,
-    net::TcpListener,
-    path::{Path, PathBuf},
-    process::exit,
-    str::FromStr,
-    sync::Arc,
-    thread,
-    time::Duration,
-};
-use zenith_utils::{auth::JwtAuth, postgres_backend::AuthType};
+use std::fs;
+use std::io;
+use std::path::PathBuf;
+use std::thread;
+use std::{fs::File, fs::OpenOptions, str::FromStr};
 
-use anyhow::{ensure, Result};
-use clap::{App, Arg, ArgMatches};
+use clap::{App, Arg};
 use daemonize::Daemonize;
 
-use pageserver::{branches, logger, page_cache, page_service, PageServerConf, RepositoryFormat};
-use zenith_utils::http_endpoint;
+use slog;
+use slog::Drain;
+use slog_scope;
+use slog_stdlog;
 
-const DEFAULT_LISTEN_ADDR: &str = "127.0.0.1:64000";
-const DEFAULT_HTTP_ENDPOINT_ADDR: &str = "127.0.0.1:9898";
+use pageserver::page_service;
+use pageserver::restore_s3;
+use pageserver::tui;
+use pageserver::walreceiver;
+use pageserver::PageServerConf;
 
-const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
-const DEFAULT_GC_PERIOD: Duration = Duration::from_secs(10);
-
-const DEFAULT_SUPERUSER: &str = "zenith_admin";
-
-/// String arguments that can be declared via CLI or config file
-#[derive(Serialize, Deserialize)]
-struct CfgFileParams {
-    listen_addr: Option<String>,
-    http_endpoint_addr: Option<String>,
-    gc_horizon: Option<String>,
-    gc_period: Option<String>,
-    pg_distrib_dir: Option<String>,
-    auth_validation_public_key_path: Option<String>,
-    auth_type: Option<String>,
-    repository_format: Option<String>,
-}
-
-impl CfgFileParams {
-    /// Extract string arguments from CLI
-    fn from_args(arg_matches: &ArgMatches) -> Self {
-        let get_arg = |arg_name: &str| -> Option<String> {
-            arg_matches.value_of(arg_name).map(str::to_owned)
-        };
-
-        Self {
-            listen_addr: get_arg("listen"),
-            http_endpoint_addr: get_arg("http_endpoint"),
-            gc_horizon: get_arg("gc_horizon"),
-            gc_period: get_arg("gc_period"),
-            pg_distrib_dir: get_arg("postgres-distrib"),
-            auth_validation_public_key_path: get_arg("auth-validation-public-key-path"),
-            auth_type: get_arg("auth-type"),
-            repository_format: get_arg("repository-format"),
-        }
-    }
-
-    /// Fill missing values in `self` with `other`
-    fn or(self, other: CfgFileParams) -> Self {
-        // TODO cleaner way to do this
-        Self {
-            listen_addr: self.listen_addr.or(other.listen_addr),
-            http_endpoint_addr: self.http_endpoint_addr.or(other.http_endpoint_addr),
-            gc_horizon: self.gc_horizon.or(other.gc_horizon),
-            gc_period: self.gc_period.or(other.gc_period),
-            pg_distrib_dir: self.pg_distrib_dir.or(other.pg_distrib_dir),
-            auth_validation_public_key_path: self
-                .auth_validation_public_key_path
-                .or(other.auth_validation_public_key_path),
-            auth_type: self.auth_type.or(other.auth_type),
-            repository_format: self.repository_format.or(other.repository_format),
-        }
-    }
-
-    /// Create a PageServerConf from these string parameters
-    fn try_into_config(&self) -> Result<PageServerConf> {
-        let workdir = PathBuf::from(".");
-
-        let listen_addr = match self.listen_addr.as_ref() {
-            Some(addr) => addr.clone(),
-            None => DEFAULT_LISTEN_ADDR.to_owned(),
-        };
-
-        let http_endpoint_addr = match self.http_endpoint_addr.as_ref() {
-            Some(addr) => addr.clone(),
-            None => DEFAULT_HTTP_ENDPOINT_ADDR.to_owned(),
-        };
-
-        let gc_horizon: u64 = match self.gc_horizon.as_ref() {
-            Some(horizon_str) => horizon_str.parse()?,
-            None => DEFAULT_GC_HORIZON,
-        };
-        let gc_period = match self.gc_period.as_ref() {
-            Some(period_str) => humantime::parse_duration(period_str)?,
-            None => DEFAULT_GC_PERIOD,
-        };
-
-        let pg_distrib_dir = match self.pg_distrib_dir.as_ref() {
-            Some(pg_distrib_dir_str) => PathBuf::from(pg_distrib_dir_str),
-            None => env::current_dir()?.join("tmp_install"),
-        };
-
-        let auth_validation_public_key_path = self
-            .auth_validation_public_key_path
-            .as_ref()
-            .map(PathBuf::from);
-
-        let auth_type = self
-            .auth_type
-            .as_ref()
-            .map_or(Ok(AuthType::Trust), |auth_type| {
-                AuthType::from_str(&auth_type)
-            })?;
-
-        if !pg_distrib_dir.join("bin/postgres").exists() {
-            anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
-        }
-
-        if auth_type == AuthType::ZenithJWT {
-            ensure!(
-                auth_validation_public_key_path.is_some(),
-                "Missing auth_validation_public_key_path when auth_type is ZenithJWT"
-            );
-            let path_ref = auth_validation_public_key_path.as_ref().unwrap();
-            ensure!(
-                path_ref.exists(),
-                format!("Can't find auth_validation_public_key at {:?}", path_ref)
-            );
-        }
-
-        let repository_format = match self.repository_format.as_ref() {
-            Some(repo_format_str) if repo_format_str == "rocksdb" => RepositoryFormat::RocksDb,
-            Some(repo_format_str) if repo_format_str == "layered" => RepositoryFormat::Layered,
-            Some(repo_format_str) => anyhow::bail!(
-                "invalid --repository-format '{}', must be 'rocksdb' or 'layered'",
-                repo_format_str
-            ),
-            None => RepositoryFormat::Layered, // default
-        };
-
-        Ok(PageServerConf {
-            daemonize: false,
-
-            listen_addr,
-            http_endpoint_addr,
-            gc_horizon,
-            gc_period,
-
-            superuser: String::from(DEFAULT_SUPERUSER),
-
-            workdir,
-
-            pg_distrib_dir,
-
-            auth_validation_public_key_path,
-            auth_type,
-
-            repository_format,
-        })
-    }
-}
-
-fn main() -> Result<()> {
+fn main() -> Result<(), io::Error> {
     let arg_matches = App::new("Zenith page server")
         .about("Materializes WAL stream to pages and serves them to the postgres")
-        .arg(
-            Arg::with_name("listen")
-                .short("l")
-                .long("listen")
-                .takes_value(true)
-                .help("listen for incoming page requests on ip:port (default: 127.0.0.1:5430)"),
-        )
-        .arg(
-            Arg::with_name("daemonize")
-                .short("d")
-                .long("daemonize")
-                .takes_value(false)
-                .help("Run in the background"),
-        )
-        .arg(
-            Arg::with_name("init")
-                .long("init")
-                .takes_value(false)
-                .help("Initialize pageserver repo"),
-        )
-        .arg(
-            Arg::with_name("gc_horizon")
-                .long("gc_horizon")
-                .takes_value(true)
-                .help("Distance from current LSN to perform all wal records cleanup"),
-        )
-        .arg(
-            Arg::with_name("gc_period")
-                .long("gc_period")
-                .takes_value(true)
-                .help("Interval between garbage collector iterations"),
-        )
-        .arg(
-            Arg::with_name("workdir")
-                .short("D")
-                .long("workdir")
-                .takes_value(true)
-                .help("Working directory for the pageserver"),
-        )
-        .arg(
-            Arg::with_name("postgres-distrib")
-                .long("postgres-distrib")
-                .takes_value(true)
-                .help("Postgres distribution directory"),
-        )
-        .arg(
-            Arg::with_name("create-tenant")
-                .long("create-tenant")
-                .takes_value(true)
-                .help("Create tenant during init")
-                .requires("init"),
-        )
-        .arg(
-            Arg::with_name("auth-validation-public-key-path")
-                .long("auth-validation-public-key-path")
-                .takes_value(true)
-                .help("Path to public key used to validate jwt signature"),
-        )
-        .arg(
-            Arg::with_name("auth-type")
-                .long("auth-type")
-                .takes_value(true)
-                .help("Authentication scheme type. One of: Trust, MD5, ZenithJWT"),
-        )
-        .arg(
-            Arg::with_name("repository-format")
-                .long("repository-format")
-                .takes_value(true)
-                .help("Which repository implementation to use, 'rocksdb' or 'layered'"),
-        )
+        .arg(Arg::with_name("datadir")
+                 .short("D")
+                 .long("dir")
+                 .takes_value(true)
+                 .help("Path to the page server data directory"))
+        .arg(Arg::with_name("wal_producer")
+                 .short("w")
+                 .long("wal-producer")
+                 .takes_value(true)
+                 .help("connect to the WAL sender (postgres or wal_acceptor) on connstr (default: 'host=127.0.0.1 port=65432 user=zenith')"))
+        .arg(Arg::with_name("listen")
+                 .short("l")
+                 .long("listen")
+                 .takes_value(true)
+                 .help("listen for incoming page requests on ip:port (default: 127.0.0.1:5430)"))
+        .arg(Arg::with_name("interactive")
+                 .short("i")
+                 .long("interactive")
+                 .takes_value(false)
+                 .help("Interactive mode"))
+        .arg(Arg::with_name("daemonize")
+                 .short("d")
+                 .long("daemonize")
+                 .takes_value(false)
+                 .help("Run in the background"))
+        .arg(Arg::with_name("skip_recovery")
+                 .long("skip-recovery")
+                 .takes_value(false)
+                 .help("Skip S3 recovery procedy and start empty"))
         .get_matches();
 
-    let workdir = Path::new(arg_matches.value_of("workdir").unwrap_or(".zenith"));
-    let cfg_file_path = workdir.canonicalize()?.join("pageserver.toml");
-
-    let args_params = CfgFileParams::from_args(&arg_matches);
-
-    let init = arg_matches.is_present("init");
-    let create_tenant = arg_matches.value_of("create-tenant");
-
-    let params = if init {
-        // We're initializing the repo, so there's no config file yet
-        args_params
-    } else {
-        // Supplement the CLI arguments with the config file
-        let cfg_file_contents = std::fs::read_to_string(&cfg_file_path)?;
-        let file_params: CfgFileParams = toml::from_str(&cfg_file_contents)?;
-        args_params.or(file_params)
+    let mut conf = PageServerConf {
+        data_dir: PathBuf::from("./"),
+        daemonize: false,
+        interactive: false,
+        wal_producer_connstr: None,
+        listen_addr: "127.0.0.1:5430".parse().unwrap(),
+        skip_recovery: false,
     };
 
-    // Set CWD to workdir for non-daemon modes
-    env::set_current_dir(&workdir)?;
-
-    // Ensure the config is valid, even if just init-ing
-    let mut conf = params.try_into_config()?;
-
-    conf.daemonize = arg_matches.is_present("daemonize");
-
-    if init && conf.daemonize {
-        eprintln!("--daemonize cannot be used with --init");
-        exit(1);
+    if let Some(dir) = arg_matches.value_of("datadir") {
+        conf.data_dir = PathBuf::from(dir);
     }
 
-    // The configuration is all set up now. Turn it into a 'static
-    // that can be freely stored in structs and passed across threads
-    // as a ref.
-    let conf: &'static PageServerConf = Box::leak(Box::new(conf));
+    if arg_matches.is_present("daemonize") {
+        conf.daemonize = true;
+    }
 
-    // Create repo and exit if init was requested
-    if init {
-        branches::init_pageserver(conf, create_tenant)?;
-        // write the config file
-        let cfg_file_contents = toml::to_string_pretty(&params)?;
-        // TODO support enable-auth flag
-        std::fs::write(&cfg_file_path, cfg_file_contents)?;
+    if arg_matches.is_present("interactive") {
+        conf.interactive = true;
+    }
 
-        return Ok(());
+    if conf.daemonize && conf.interactive {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidInput,
+            "--daemonize is not allowed with --interactive: choose one",
+        ));
+    }
+
+    if arg_matches.is_present("skip_recovery") {
+        conf.skip_recovery = true;
+    }
+
+    if let Some(addr) = arg_matches.value_of("wal_producer") {
+        conf.wal_producer_connstr = Some(String::from_str(addr).unwrap());
+    }
+
+    if let Some(addr) = arg_matches.value_of("listen") {
+        conf.listen_addr = addr.parse().unwrap();
     }
 
     start_pageserver(conf)
 }
 
-fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
+fn start_pageserver(conf: PageServerConf) -> Result<(), io::Error> {
     // Initialize logger
-    let (_scope_guard, log_file) = logger::init_logging(&conf, "pageserver.log")?;
-    let _log_guard = slog_stdlog::init()?;
+    let _scope_guard = init_logging(&conf);
+    let _log_guard = slog_stdlog::init().unwrap();
 
     // Note: this `info!(...)` macro comes from `log` crate
     info!("standard logging redirected to slog");
 
-    // TODO: Check that it looks like a valid repository before going further
+    let tui_thread: Option<thread::JoinHandle<()>>;
+    if conf.interactive {
+        // Initialize the UI
+        tui_thread = Some(
+            thread::Builder::new()
+                .name("UI thread".into())
+                .spawn(|| {
+                    let _ = tui::ui_main();
+                })
+                .unwrap(),
+        );
+        //threads.push(tui_thread);
+    } else {
+        tui_thread = None;
+    }
 
     if conf.daemonize {
         info!("daemonizing...");
 
         // There should'n be any logging to stdin/stdout. Redirect it to the main log so
-        // that we will see any accidental manual fprintf's or backtraces.
-        let stdout = log_file.try_clone().unwrap();
-        let stderr = log_file;
+        // that we will see any accidental manual fpritf's or backtraces.
+        let stdout = OpenOptions::new()
+            .create(true)
+            .append(true)
+            .open(conf.data_dir.join("pageserver.log"))
+            .unwrap();
+        let stderr = OpenOptions::new()
+            .create(true)
+            .append(true)
+            .open(conf.data_dir.join("pageserver.log"))
+            .unwrap();
 
         let daemonize = Daemonize::new()
-            .pid_file("pageserver.pid")
-            .working_directory(".")
+            .pid_file(conf.data_dir.join("pageserver.pid"))
+            .working_directory(conf.data_dir.clone())
             .stdout(stdout)
             .stderr(stderr);
 
@@ -323,40 +152,106 @@ fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
         }
     }
 
-    // Spawn a new thread for the http endpoint
-    thread::Builder::new()
-        .name("Metrics thread".into())
-        .spawn(move || http_endpoint::thread_main(conf.http_endpoint_addr.clone()))?;
+    let mut threads = Vec::new();
 
-    // Check that we can bind to address before starting threads to simplify shutdown
-    // sequence if port is occupied.
-    info!("Starting pageserver on {}", conf.listen_addr);
-    let pageserver_listener = TcpListener::bind(conf.listen_addr.clone())?;
+    info!("starting...");
 
-    // Initialize page cache, this will spawn walredo_thread
-    page_cache::init(conf);
+    // Before opening up for connections, restore the latest base backup from S3.
+    // (We don't persist anything to local disk at the moment, so we need to do
+    // this at every startup)
+    // TODO move it to a separate function
+    if !conf.skip_recovery {
+        restore_s3::restore_main(&conf);
+    }
 
-    // initialize authentication for incoming connections
-    let auth = match &conf.auth_type {
-        AuthType::Trust | AuthType::MD5 => Arc::new(None),
-        AuthType::ZenithJWT => {
-            // unwrap is ok because check is performed when creating config, so path is set and file exists
-            let key_path = conf.auth_validation_public_key_path.as_ref().unwrap();
-            Arc::new(Some(JwtAuth::from_key_path(key_path)?))
-        }
-    };
-    info!("Using auth: {:#?}", conf.auth_type);
-    // Spawn a thread to listen for connections. It will spawn further threads
-    // for each connection.
-    let page_service_thread = thread::Builder::new()
+    // Create directory for wal-redo datadirs
+    match fs::create_dir(conf.data_dir.join("wal-redo")) {
+        Ok(_) => {}
+        Err(e) => match e.kind() {
+            io::ErrorKind::AlreadyExists => {}
+            _ => {
+                panic!("Failed to create wal-redo data directory: {}", e);
+            }
+        },
+    }
+
+    // Launch the WAL receiver thread if pageserver was started with --wal-producer
+    // option. It will try to connect to the WAL safekeeper, and stream the WAL. If
+    // the connection is lost, it will reconnect on its own. We just fire and forget
+    // it here.
+    //
+    // All other wal receivers are started on demand by "callmemaybe" command
+    // sent to pageserver.
+    let conf_copy = conf.clone();
+    if let Some(wal_producer) = conf.wal_producer_connstr {
+        let conf = conf_copy.clone();
+        let walreceiver_thread = thread::Builder::new()
+            .name("static WAL receiver thread".into())
+            .spawn(move || {
+                walreceiver::thread_main(conf, &wal_producer);
+            })
+            .unwrap();
+        threads.push(walreceiver_thread);
+    }
+
+    // GetPage@LSN requests are served by another thread. (It uses async I/O,
+    // but the code in page_service sets up it own thread pool for that)
+    let conf = conf_copy.clone();
+    let page_server_thread = thread::Builder::new()
         .name("Page Service thread".into())
-        .spawn(move || {
-            page_service::thread_main(conf, auth, pageserver_listener, conf.auth_type)
-        })?;
-
-    page_service_thread
-        .join()
-        .expect("Page service thread has panicked")?;
+        .spawn(|| {
+            // thread code
+            page_service::thread_main(conf);
+        })
+        .unwrap();
+    threads.push(page_server_thread);
 
+    if tui_thread.is_some() {
+        // The TUI thread exits when the user asks to Quit.
+        tui_thread.unwrap().join().unwrap();
+    } else {
+        // In non-interactive mode, wait forever.
+        for t in threads {
+            t.join().unwrap()
+        }
+    }
     Ok(())
 }
+
+fn init_logging(conf: &PageServerConf) -> slog_scope::GlobalLoggerGuard {
+    if conf.interactive {
+        tui::init_logging()
+    } else if conf.daemonize {
+        let log = conf.data_dir.join("pageserver.log");
+        let log_file = File::create(log).unwrap_or_else(|_| panic!("Could not create log file"));
+        let decorator = slog_term::PlainSyncDecorator::new(log_file);
+        let drain = slog_term::CompactFormat::new(decorator).build();
+        let drain = slog::Filter::new(drain, |record: &slog::Record| {
+            if record.level().is_at_least(slog::Level::Info) {
+                return true;
+            }
+            return false;
+        });
+        let drain = std::sync::Mutex::new(drain).fuse();
+        let logger = slog::Logger::root(drain, slog::o!());
+        slog_scope::set_global_logger(logger)
+    } else {
+        let decorator = slog_term::TermDecorator::new().build();
+        let drain = slog_term::FullFormat::new(decorator).build().fuse();
+        let drain = slog_async::Async::new(drain).chan_size(1000).build().fuse();
+        let drain = slog::Filter::new(drain, |record: &slog::Record| {
+            if record.level().is_at_least(slog::Level::Info) {
+                return true;
+            }
+            if record.level().is_at_least(slog::Level::Debug)
+                && record.module().starts_with("pageserver")
+            {
+                return true;
+            }
+            return false;
+        })
+        .fuse();
+        let logger = slog::Logger::root(drain, slog::o!());
+        slog_scope::set_global_logger(logger)
+    }
+}

pageserver/src/branches.rs

@@ -1,404 +0,0 @@
-//!
-//! Branch management code
-//!
-// TODO: move all paths construction to conf impl
-//
-
-use anyhow::{bail, ensure, Context, Result};
-use postgres_ffi::ControlFileData;
-use serde::{Deserialize, Serialize};
-use std::{
-    fs,
-    path::Path,
-    process::{Command, Stdio},
-    str::FromStr,
-    sync::Arc,
-};
-use zenith_utils::zid::{ZTenantId, ZTimelineId};
-
-use log::*;
-use zenith_utils::lsn::Lsn;
-
-use crate::logger;
-use crate::object_repository::ObjectRepository;
-use crate::page_cache;
-use crate::restore_local_repo;
-use crate::walredo::WalRedoManager;
-use crate::{repository::Repository, PageServerConf, RepositoryFormat};
-
-#[derive(Serialize, Deserialize, Clone)]
-pub struct BranchInfo {
-    pub name: String,
-    pub timeline_id: ZTimelineId,
-    pub latest_valid_lsn: Option<Lsn>,
-    pub ancestor_id: Option<String>,
-    pub ancestor_lsn: Option<String>,
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct PointInTime {
-    pub timelineid: ZTimelineId,
-    pub lsn: Lsn,
-}
-
-pub fn init_pageserver(conf: &'static PageServerConf, create_tenant: Option<&str>) -> Result<()> {
-    // Initialize logger
-    let (_scope_guard, _log_file) = logger::init_logging(&conf, "pageserver.log")?;
-    let _log_guard = slog_stdlog::init()?;
-
-    if let Some(tenantid) = create_tenant {
-        let tenantid = ZTenantId::from_str(tenantid)?;
-        println!("initializing tenantid {}", tenantid);
-        create_repo(
-            conf,
-            tenantid,
-            Arc::new(crate::walredo::DummyRedoManager {}),
-        )
-        .with_context(|| "failed to create repo")?;
-    }
-    fs::create_dir_all(conf.tenants_path())?;
-
-    println!("pageserver init succeeded");
-    Ok(())
-}
-
-pub fn create_repo(
-    conf: &'static PageServerConf,
-    tenantid: ZTenantId,
-    wal_redo_manager: Arc<dyn WalRedoManager + Send + Sync>,
-) -> Result<Arc<dyn Repository>> {
-    let repo_dir = conf.tenant_path(&tenantid);
-    if repo_dir.exists() {
-        bail!("repo for {} already exists", tenantid)
-    }
-
-    // top-level dir may exist if we are creating it through CLI
-    fs::create_dir_all(&repo_dir)
-        .with_context(|| format!("could not create directory {}", repo_dir.display()))?;
-
-    // Note: this `info!(...)` macro comes from `log` crate
-    info!("standard logging redirected to slog");
-
-    fs::create_dir(conf.timelines_path(&tenantid))?;
-    fs::create_dir_all(conf.branches_path(&tenantid))?;
-    fs::create_dir_all(conf.tags_path(&tenantid))?;
-
-    info!("created directory structure in {}", repo_dir.display());
-
-    let tli = create_timeline(conf, None, &tenantid)?;
-
-    // We don't use page_cache here, because we don't want to spawn the WAL redo thread during
-    // repository initialization.
-    //
-    // FIXME: That caused trouble, because the WAL redo thread launched initdb in the background,
-    // and it kept running even after the "zenith init" had exited. In tests, we started the
-    // page server immediately after that, so that initdb was still running in the background,
-    // and we failed to run initdb again in the same directory. This has been solved for the
-    // rapid init+start case now, but the general race condition remains if you restart the
-    // server quickly.
-    let repo: Arc<dyn Repository + Sync + Send> =
-        match conf.repository_format {
-            RepositoryFormat::Layered => Arc::new(
-                crate::layered_repository::LayeredRepository::new(conf, wal_redo_manager, tenantid),
-            ),
-            RepositoryFormat::RocksDb => {
-                let obj_store = crate::rocksdb_storage::RocksObjectStore::create(conf, &tenantid)?;
-
-                Arc::new(ObjectRepository::new(
-                    conf,
-                    Arc::new(obj_store),
-                    wal_redo_manager,
-                    tenantid,
-                ))
-            }
-        };
-
-    // Load data into pageserver
-    // TODO To implement zenith import we need to
-    //      move data loading out of create_repo()
-    bootstrap_timeline(conf, tenantid, tli, &*repo)?;
-
-    Ok(repo)
-}
-
-// Returns checkpoint LSN from controlfile
-fn get_lsn_from_controlfile(path: &Path) -> Result<Lsn> {
-    // Read control file to extract the LSN
-    let controlfile_path = path.join("global").join("pg_control");
-    let controlfile = ControlFileData::decode(&fs::read(controlfile_path)?)?;
-    let lsn = controlfile.checkPoint;
-
-    Ok(Lsn(lsn))
-}
-
-// Create the cluster temporarily in a initdbpath directory inside the repository
-// to get bootstrap data for timeline initialization.
-//
-fn run_initdb(conf: &'static PageServerConf, initdbpath: &Path) -> Result<()> {
-    info!("running initdb... ");
-
-    let initdb_path = conf.pg_bin_dir().join("initdb");
-    let initdb_output = Command::new(initdb_path)
-        .args(&["-D", initdbpath.to_str().unwrap()])
-        .args(&["-U", &conf.superuser])
-        .arg("--no-instructions")
-        .env_clear()
-        .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
-        .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
-        .stdout(Stdio::null())
-        .output()
-        .with_context(|| "failed to execute initdb")?;
-    if !initdb_output.status.success() {
-        anyhow::bail!(
-            "initdb failed: '{}'",
-            String::from_utf8_lossy(&initdb_output.stderr)
-        );
-    }
-    info!("initdb succeeded");
-
-    Ok(())
-}
-
-//
-// - run initdb to init temporary instance and get bootstrap data
-// - after initialization complete, remove the temp dir.
-//
-fn bootstrap_timeline(
-    conf: &'static PageServerConf,
-    tenantid: ZTenantId,
-    tli: ZTimelineId,
-    repo: &dyn Repository,
-) -> Result<()> {
-    let initdb_path = conf.tenant_path(&tenantid).join("tmp");
-
-    // Init temporarily repo to get bootstrap data
-    run_initdb(conf, &initdb_path)?;
-    let pgdata_path = initdb_path;
-
-    let lsn = get_lsn_from_controlfile(&pgdata_path)?;
-
-    info!("bootstrap_timeline {:?} at lsn {}", pgdata_path, lsn);
-
-    let timeline = repo.create_empty_timeline(tli, lsn)?;
-    restore_local_repo::import_timeline_from_postgres_datadir(&pgdata_path, &*timeline, lsn)?;
-
-    let wal_dir = pgdata_path.join("pg_wal");
-    restore_local_repo::import_timeline_wal(&wal_dir, &*timeline, timeline.get_last_record_lsn())?;
-
-    println!(
-        "created initial timeline {} timeline.lsn {}",
-        tli,
-        timeline.get_last_record_lsn()
-    );
-
-    let data = tli.to_string();
-    fs::write(conf.branch_path("main", &tenantid), data)?;
-    println!("created main branch");
-
-    // Remove temp dir. We don't need it anymore
-    fs::remove_dir_all(pgdata_path)?;
-
-    Ok(())
-}
-
-pub(crate) fn get_tenants(conf: &PageServerConf) -> Result<Vec<String>> {
-    let tenants_dir = conf.tenants_path();
-
-    std::fs::read_dir(&tenants_dir)?
-        .map(|dir_entry_res| {
-            let dir_entry = dir_entry_res?;
-            ensure!(dir_entry.file_type()?.is_dir());
-            Ok(dir_entry.file_name().to_str().unwrap().to_owned())
-        })
-        .collect()
-}
-
-pub(crate) fn get_branches(conf: &PageServerConf, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
-    let repo = page_cache::get_repository_for_tenant(tenantid)?;
-
-    // Each branch has a corresponding record (text file) in the refs/branches
-    // with timeline_id.
-    let branches_dir = conf.branches_path(tenantid);
-
-    std::fs::read_dir(&branches_dir)?
-        .map(|dir_entry_res| {
-            let dir_entry = dir_entry_res?;
-            let name = dir_entry.file_name().to_str().unwrap().to_string();
-            let timeline_id = std::fs::read_to_string(dir_entry.path())?.parse::<ZTimelineId>()?;
-
-            let latest_valid_lsn = repo
-                .get_timeline(timeline_id)
-                .map(|timeline| timeline.get_last_valid_lsn())
-                .ok();
-
-            let ancestor_path = conf.ancestor_path(&timeline_id, tenantid);
-            let mut ancestor_id: Option<String> = None;
-            let mut ancestor_lsn: Option<String> = None;
-
-            if ancestor_path.exists() {
-                let ancestor = std::fs::read_to_string(ancestor_path)?;
-                let mut strings = ancestor.split('@');
-
-                ancestor_id = Some(
-                    strings
-                        .next()
-                        .with_context(|| "wrong branch ancestor point in time format")?
-                        .to_owned(),
-                );
-                ancestor_lsn = Some(
-                    strings
-                        .next()
-                        .with_context(|| "wrong branch ancestor point in time format")?
-                        .to_owned(),
-                );
-            }
-
-            Ok(BranchInfo {
-                name,
-                timeline_id,
-                latest_valid_lsn,
-                ancestor_id,
-                ancestor_lsn,
-            })
-        })
-        .collect()
-}
-
-pub(crate) fn create_branch(
-    conf: &PageServerConf,
-    branchname: &str,
-    startpoint_str: &str,
-    tenantid: &ZTenantId,
-) -> Result<BranchInfo> {
-    let repo = page_cache::get_repository_for_tenant(tenantid)?;
-
-    if conf.branch_path(branchname, tenantid).exists() {
-        anyhow::bail!("branch {} already exists", branchname);
-    }
-
-    let mut startpoint = parse_point_in_time(conf, startpoint_str, tenantid)?;
-
-    if startpoint.lsn == Lsn(0) {
-        // Find end of WAL on the old timeline
-        let end_of_wal = repo
-            .get_timeline(startpoint.timelineid)?
-            .get_last_record_lsn();
-        println!("branching at end of WAL: {}", end_of_wal);
-        startpoint.lsn = end_of_wal;
-    }
-
-    // create a new timeline directory for it
-    let newtli = create_timeline(conf, Some(startpoint), tenantid)?;
-
-    // Let the Repository backend do its initialization
-    repo.branch_timeline(startpoint.timelineid, newtli, startpoint.lsn)?;
-
-    // Remember the human-readable branch name for the new timeline.
-    // FIXME: there's a race condition, if you create a branch with the same
-    // name concurrently.
-    let data = newtli.to_string();
-    fs::write(conf.branch_path(&branchname, tenantid), data)?;
-
-    Ok(BranchInfo {
-        name: branchname.to_string(),
-        timeline_id: newtli,
-        latest_valid_lsn: Some(startpoint.lsn),
-        ancestor_id: None,
-        ancestor_lsn: None,
-    })
-}
-
-//
-// Parse user-given string that represents a point-in-time.
-//
-// We support multiple variants:
-//
-// Raw timeline id in hex, meaning the end of that timeline:
-//    bc62e7d612d0e6fe8f99a6dd2f281f9d
-//
-// A specific LSN on a timeline:
-//    bc62e7d612d0e6fe8f99a6dd2f281f9d@2/15D3DD8
-//
-// Same, with a human-friendly branch name:
-//    main
-//    main@2/15D3DD8
-//
-// Human-friendly tag name:
-//    mytag
-//
-//
-fn parse_point_in_time(
-    conf: &PageServerConf,
-    s: &str,
-    tenantid: &ZTenantId,
-) -> Result<PointInTime> {
-    let mut strings = s.split('@');
-    let name = strings.next().unwrap();
-
-    let lsn: Option<Lsn>;
-    if let Some(lsnstr) = strings.next() {
-        lsn = Some(
-            Lsn::from_str(lsnstr).with_context(|| "invalid LSN in point-in-time specification")?,
-        );
-    } else {
-        lsn = None
-    }
-
-    // Check if it's a tag
-    if lsn.is_none() {
-        let tagpath = conf.tag_path(name, &tenantid);
-        if tagpath.exists() {
-            let pointstr = fs::read_to_string(tagpath)?;
-
-            return parse_point_in_time(conf, &pointstr, &tenantid);
-        }
-    }
-
-    // Check if it's a branch
-    // Check if it's branch @ LSN
-    let branchpath = conf.branch_path(name, &tenantid);
-    if branchpath.exists() {
-        let pointstr = fs::read_to_string(branchpath)?;
-
-        let mut result = parse_point_in_time(conf, &pointstr, &tenantid)?;
-
-        result.lsn = lsn.unwrap_or(Lsn(0));
-        return Ok(result);
-    }
-
-    // Check if it's a timelineid
-    // Check if it's timelineid @ LSN
-    if let Ok(timelineid) = ZTimelineId::from_str(name) {
-        let tlipath = conf.timeline_path(&timelineid, &tenantid);
-        if tlipath.exists() {
-            return Ok(PointInTime {
-                timelineid,
-                lsn: lsn.unwrap_or(Lsn(0)),
-            });
-        }
-    }
-
-    bail!("could not parse point-in-time {}", s);
-}
-
-fn create_timeline(
-    conf: &PageServerConf,
-    ancestor: Option<PointInTime>,
-    tenantid: &ZTenantId,
-) -> Result<ZTimelineId> {
-    // Create initial timeline
-
-    let timelineid = ZTimelineId::generate();
-
-    let timelinedir = conf.timeline_path(&timelineid, tenantid);
-
-    fs::create_dir(&timelinedir)?;
-    fs::create_dir(&timelinedir.join("wal"))?;
-
-    if let Some(ancestor) = ancestor {
-        let data = format!("{}@{}", ancestor.timelineid, ancestor.lsn);
-        fs::write(timelinedir.join("ancestor"), data)?;
-    }
-
-    Ok(timelineid)
-}

pageserver/src/controlfile.rs

@@ -0,0 +1,218 @@
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+
+use std::fs::File;
+use std::io::prelude::*;
+use std::io::SeekFrom;
+
+use bytes::{Buf, Bytes};
+
+use log::*;
+
+type XLogRecPtr = u64;
+
+#[repr(C)]
+#[derive(Debug, Clone)]
+/*
+ * Body of CheckPoint XLOG records.  This is declared here because we keep
+ * a copy of the latest one in pg_control for possible disaster recovery.
+ * Changing this struct requires a PG_CONTROL_VERSION bump.
+ */
+pub struct CheckPoint {
+    pub redo: XLogRecPtr,    /* next RecPtr available when we began to
+                              * create CheckPoint (i.e. REDO start point) */
+    pub ThisTimeLineID: u32, /* current TLI */
+    pub PrevTimeLineID: u32, /* previous TLI, if this record begins a new
+                              * timeline (equals ThisTimeLineID otherwise) */
+    pub fullPageWrites: bool, /* current full_page_writes */
+    pub nextXid: u64,         /* next free transaction ID */
+    pub nextOid: u32,         /* next free OID */
+    pub nextMulti: u32,       /* next free MultiXactId */
+    pub nextMultiOffset: u32, /* next free MultiXact offset */
+    pub oldestXid: u32,       /* cluster-wide minimum datfrozenxid */
+    pub oldestXidDB: u32,     /* database with minimum datfrozenxid */
+    pub oldestMulti: u32,     /* cluster-wide minimum datminmxid */
+    pub oldestMultiDB: u32,   /* database with minimum datminmxid */
+    pub time: u64,            /* time stamp of checkpoint */
+    pub oldestCommitTsXid: u32, /* oldest Xid with valid commit
+                               * timestamp */
+    pub newestCommitTsXid: u32, /* newest Xid with valid commit
+                                 * timestamp */
+
+    /*
+     * Oldest XID still running. This is only needed to initialize hot standby
+     * mode from an online checkpoint, so we only bother calculating this for
+     * online checkpoints and only when wal_level is replica. Otherwise it's
+     * set to InvalidTransactionId.
+     */
+    pub oldestActiveXid: u32,
+}
+
+#[repr(C)]
+#[derive(Debug, Clone)]
+pub struct ControlFileDataZenith {
+    pub system_identifier: u64,
+    pg_control_version: u32, /* PG_CONTROL_VERSION */
+    catalog_version_no: u32, /* see catversion.h */
+
+    state: i32, /* see enum above */
+    time: i64,  /* time stamp of last pg_control update */
+    pub checkPoint: XLogRecPtr,
+    checkPointCopy: CheckPoint, /* copy of last check point record */
+    unloggedLSN: XLogRecPtr,    /* current fake LSN value, for unlogged rels */
+    minRecoveryPoint: XLogRecPtr,
+    minRecoveryPointTLI: u32,
+    backupStartPoint: XLogRecPtr,
+    backupEndPoint: XLogRecPtr,
+    backupEndRequired: bool,
+}
+
+impl ControlFileDataZenith {
+    pub fn new() -> ControlFileDataZenith {
+        ControlFileDataZenith {
+            system_identifier: 0,
+            pg_control_version: 0,
+            catalog_version_no: 0,
+            state: 0,
+            time: 0,
+            checkPoint: 0,
+            checkPointCopy: {
+                CheckPoint {
+                    redo: 0,
+                    ThisTimeLineID: 0,
+                    PrevTimeLineID: 0,
+                    fullPageWrites: false,
+                    nextXid: 0,
+                    nextOid: 0,
+                    nextMulti: 0,
+                    nextMultiOffset: 0,
+                    oldestXid: 0,
+                    oldestXidDB: 0,
+                    oldestMulti: 0,
+                    oldestMultiDB: 0,
+                    time: 0,
+                    oldestCommitTsXid: 0,
+                    newestCommitTsXid: 0,
+                    oldestActiveXid: 0,
+                }
+            },
+            unloggedLSN: 0,
+            minRecoveryPoint: 0,
+            minRecoveryPointTLI: 0,
+            backupStartPoint: 0,
+            backupEndPoint: 0,
+            backupEndRequired: false,
+        }
+    }
+}
+
+pub fn decode_pg_control(mut buf: Bytes) -> ControlFileDataZenith {
+    info!("decode pg_control");
+
+    let controlfile: ControlFileDataZenith = ControlFileDataZenith {
+        system_identifier: buf.get_u64_le(),
+        pg_control_version: buf.get_u32_le(),
+        catalog_version_no: buf.get_u32_le(),
+        state: buf.get_i32_le(),
+        time: {
+            buf.advance(4);
+            buf.get_i64_le()
+        },
+        checkPoint: buf.get_u64_le(),
+        checkPointCopy: {
+            CheckPoint {
+                redo: buf.get_u64_le(),
+                ThisTimeLineID: buf.get_u32_le(),
+                PrevTimeLineID: buf.get_u32_le(),
+                fullPageWrites: buf.get_u8() != 0,
+                nextXid: {
+                    buf.advance(7);
+                    buf.get_u64_le()
+                },
+                nextOid: buf.get_u32_le(),
+                nextMulti: buf.get_u32_le(),
+                nextMultiOffset: buf.get_u32_le(),
+                oldestXid: buf.get_u32_le(),
+                oldestXidDB: buf.get_u32_le(),
+                oldestMulti: buf.get_u32_le(),
+                oldestMultiDB: buf.get_u32_le(),
+                time: {
+                    buf.advance(4);
+                    buf.get_u64_le()
+                },
+                oldestCommitTsXid: buf.get_u32_le(),
+                newestCommitTsXid: buf.get_u32_le(),
+                oldestActiveXid: buf.get_u32_le(),
+            }
+        },
+        unloggedLSN: buf.get_u64_le(),
+        minRecoveryPoint: buf.get_u64_le(),
+        minRecoveryPointTLI: buf.get_u32_le(),
+        backupStartPoint: {
+            buf.advance(4);
+            buf.get_u64_le()
+        },
+        backupEndPoint: buf.get_u64_le(),
+        backupEndRequired: buf.get_u8() != 0,
+    };
+
+    return controlfile;
+}
+
+pub fn parse_controlfile(b: Bytes) {
+    let controlfile = decode_pg_control(b);
+
+    info!(
+        "controlfile {:X}/{:X}",
+        controlfile.checkPoint >> 32,
+        controlfile.checkPoint
+    );
+    info!("controlfile {:?}", controlfile);
+}
+
+const MAX_MAPPINGS: usize = 62;
+
+#[derive(Debug)]
+struct RelMapping {
+    mapoid: u32,      /* OID of a catalog */
+    mapfilenode: u32, /* its filenode number */
+}
+
+#[derive(Debug)]
+pub struct RelMapFile {
+    magic: i32,        /* always RELMAPPER_FILEMAGIC */
+    num_mappings: i32, /* number of valid RelMapping entries */
+    mappings: [u8; MAX_MAPPINGS * 8],
+    crc: u32, /* CRC of all above */
+    pad: i32, /* to make the struct size be 512 exactly */
+}
+
+pub fn decode_filemapping(mut buf: Bytes) -> RelMapFile {
+    info!("decode filemap");
+
+    let file: RelMapFile = RelMapFile {
+        magic: buf.get_i32_le(),        /* always RELMAPPER_FILEMAGIC */
+        num_mappings: buf.get_i32_le(), /* number of valid RelMapping entries */
+        mappings: {
+            let mut arr = [0 as u8; MAX_MAPPINGS * 8];
+            buf.copy_to_slice(&mut arr);
+            arr
+        },
+        crc: buf.get_u32_le(), /* CRC of all above */
+        pad: buf.get_i32_le(),
+    };
+
+    info!("decode filemap {:?}", file);
+    file
+}
+
+pub fn write_buf_to_file(filepath: String, buf: Bytes, blkno: u32) {
+    info!("write_buf_to_file {}", filepath.clone());
+
+    let mut buffer = File::create(filepath.clone()).unwrap();
+    buffer.seek(SeekFrom::Start(8192 * blkno as u64)).unwrap();
+
+    buffer.write_all(&buf).unwrap();
+
+    info!("DONE write_buf_to_file {}", filepath);
+}

pageserver/src/layered_repository/README.md

@@ -1,298 +0,0 @@
-# Overview
-
-The on-disk format is based on immutable files. The page server
-receives a stream of incoming WAL, parses the WAL records to determine
-which pages they apply to, and accumulates the incoming changes in
-memory. Every now and then, the accumulated changes are written out to
-new files.
-
-The files are called "snapshot files". Each snapshot file corresponds
-to one 10 MB slice of a PostgreSQL relation fork. The snapshot files
-for each timeline are stored in the timeline's subdirectory under
-.zenith/tenants/<tenantid>/timelines.
-
-The files are named like this:
-
-    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>_<end LSN>
-
-For example:
-
-    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000
-
-Some non-relation files are also stored in repository. For example,
-a CLOG segment would be named like this:
-
-    pg_xact_0000_0_00000000198B06B0_00000000198C2550
-
-There is no difference in how the relation and non-relation files are
-managed, except that the first part of file names is different.
-Internally, the relations and non-relation files that are managed in
-the versioned store are together called "relishes".
-
-Each snapshot file contains a full snapshot, that is, full copy of all
-pages in the relation, as of the "start LSN". It also contains all WAL
-records applicable to the relation between the start and end
-LSNs. With this information, the page server can reconstruct any page
-version of the relation in the LSN range.
-
-If a file has been dropped, the last snapshot file for it is created
-with the _DROPPED suffix, e.g.
-
-    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000_DROPPED
-
-In addition to the relations, with "rel_*" prefix, we use the same
-format for storing various smaller files from the PostgreSQL data
-directory. They will use different suffixes and the naming scheme
-up to the LSN range varies. The Zenith source code uses the term
-"relish" to mean "a relation, or other file that's treated like a
-relation in the storage"
-
-## Notation used in this document
-
-The full path of a snapshot file looks like this:
-
-    .zenith/tenants/941ddc8604413b88b3d208bddf90396c/timelines/4af489b06af8eed9e27a841775616962/rel_1663_13990_2609_0_10_000000000169C348_0000000001702000
-
-For simplicity, the examples below use a simplified notation for the
-paths.  The tenant ID is left out, the timeline ID is replaced with
-the human-readable branch name, and spcnode+dbnode+relnode+forkum+segno
-with a human-readable table name. The LSNs are also shorter. For
-example, a snapshot file for 'orders' table on 'main' branch, with LSN
-range 100-200 would be:
-
-    main/orders_100_200
-
-
-# Creating snapshot files
-
-Let's start with a simple example with a system that contains one
-branch called 'main' and two tables, 'orders' and 'customers'. The end
-of WAL is currently at LSN 250. In this starting situation, you would
-have two files on disk:
-
-	main/orders_100_200
-	main/customers_100_200
-
-In addition to those files, the recent changes between LSN 200 and the
-end of WAL at 250 are kept in memory. If the page server crashes, the
-latest records between 200-250 need to be re-read from the WAL.
-
-Whenever enough WAL has been accumulated in memory, the page server
-writes out the changes in memory into new snapshot files. This process
-is called "checkpointing" (not to be confused with the PostgreSQL
-checkpoints, that's a different thing). The page server only creates
-snapshot files for relations that have been modified since the last
-checkpoint. For example, if the current end of WAL is at LSN 450, and
-the last checkpoint happened at LSN 400 but there hasn't been any
-recent changes to 'customers' table, you would have these files on
-disk:
-
-	main/orders_100_200
-	main/orders_200_300
-	main/orders_300_400
-	main/customers_100_200
-
-If the customers table is modified later, a new file is created for it
-at the next checkpoint. The new file will cover the "gap" from the
-last snapshot file, so the LSN ranges are always contiguous:
-
-	main/orders_100_200
-	main/orders_200_300
-	main/orders_300_400
-	main/customers_100_200
-	main/customers_200_500
-
-## Reading page versions
-
-Whenever a GetPage@LSN request comes in from the compute node, the
-page server needs to reconstruct the requested page, as it was at the
-requested LSN. To do that, the page server first checks the recent
-in-memory layer; if the requested page version is found there, it can
-be returned immediatedly without looking at the files on
-disk. Otherwise the page server needs to locate the snapshot file that
-contains the requested page version.
-
-For example, if a request comes in for table 'orders' at LSN 250, the
-page server would load the 'main/orders_200_300' file into memory, and
-reconstruct and return the requested page from it, as it was at
-LSN 250. Because the snapshot file consists of a full image of the
-relation at the start LSN and the WAL, reconstructing the page
-involves replaying any WAL records applicable to the page between LSNs
-200-250, starting from the base image at LSN 200.
-
-A request at a file boundary can be satisfied using either file. For
-example, if there are two files on disk:
-
-	main/orders_100_200
-	main/orders_200_300
-
-And a request comes with LSN 200, either file can be used for it. It
-is better to use the later file, however, because it contains an
-already materialized version of all the pages at LSN 200. Using the
-first file, you would need to apply any WAL records between 100 and
-200 to reconstruct the requested page.
-
-# Multiple branches
-
-Imagine that a child branch is created at LSN 250:
-
-            @250
-    ----main--+-------------------------->
-               \
-                +---child-------------->
-
-
-Then, the 'orders' table is updated differently on the 'main' and
-'child' branches. You now have this situation on disk:
-
-    main/orders_100_200
-    main/orders_200_300
-    main/orders_300_400
-    main/customers_100_200
-    child/orders_250_300
-    child/orders_300_400
-
-Because the 'customers' table hasn't been modified on the child
-branch, there is no file for it there. If you request a page for it on
-the 'child' branch, the page server will not find any snapshot file
-for it in the 'child' directory, so it will recurse to look into the
-parent 'main' branch instead.
-
-From the 'child' branch's point of view, the history for each relation
-is linear, and the request's LSN identifies unambiguously which file
-you need to look at. For example, the history for the 'orders' table
-on the 'main' branch consists of these files:
-
-    main/orders_100_200
-    main/orders_200_300
-    main/orders_300_400
-
-And from the 'child' branch's point of view, it consists of these
-files:
-
-    main/orders_100_200
-    main/orders_200_300
-    child/orders_250_300
-    child/orders_300_400
-
-The branch metadata includes the point where the child branch was
-created, LSN 250. If a page request comes with LSN 275, we read the
-page version from the 'child/orders_250_300' file. If the request LSN
-is 225, we read it from the 'main/orders_200_300' file instead.  The
-page versions between 250-300 in the 'main/orders_200_300' file are
-ignored when operating on the child branch.
-
-Note: It doesn't make any difference if the child branch is created
-when the end of the main branch was at LSN 250, or later when the tip of
-the main branch had already moved on. The latter case, creating a
-branch at a historic LSN, is how we support PITR in Zenith.
-
-
-# Garbage collection
-
-In this scheme, we keep creating new snapshot files over time. We also
-need a mechanism to remove old files that are no longer needed,
-because disk space isn't infinite.
-
-What files are still needed? Currently, the page server supports PITR
-and branching from any branch at any LSN that is "recent enough" from
-the tip of the branch.  "Recent enough" is defined as an LSN horizon,
-which by default is 64 MB.  (See DEFAULT_GC_HORIZON). For this
-example, let's assume that the LSN horizon is 150 units.
-
-Let's look at the single branch scenario again. Imagine that the end
-of the branch is LSN 525, so that the GC horizon is currently at
-525-150 = 375
-
-	main/orders_100_200
-	main/orders_200_300
-	main/orders_300_400
-	main/orders_400_500
-	main/customers_100_200
-
-We can remove files 'main/orders_100_200' and 'main/orders_200_300',
-because the end LSNs of those files are older than GC horizon 375, and
-there are more recent snapshot files for the table. 'main/orders_300_400'
-and 'main/orders_400_500' are still within the horizon, so they must be
-retained. 'main/customers_100_200' is old enough, but it cannot be
-removed because there is no newer snapshot file for the table.
-
-Things get slightly more complicated with multiple branches. All of
-the above still holds, but in addition to recent files we must also
-retain older shapshot files that are still needed by child branches.
-For example, if child branch is created at LSN 150, and the 'customers'
-table is updated on the branch, you would have these files:
-
-	main/orders_100_200
-	main/orders_200_300
-	main/orders_300_400
-	main/orders_400_500
-	main/customers_100_200
-	child/customers_150_300
-
-In this situation, the 'main/orders_100_200' file cannot be removed,
-even though it is older than the GC horizon, because it is still
-needed by the child branch.  'main/orders_200_300' can still be
-removed. So after garbage collection, these files would remain:
-
-	main/orders_100_200
-
-	main/orders_300_400
-	main/orders_400_500
-	main/customers_100_200
-	child/customers_150_300
-
-If 'orders' is modified later on the 'child' branch, we will create a
-snapshot file for it on the child:
-
-	main/orders_100_200
-
-	main/orders_300_400
-	main/orders_400_500
-	main/customers_100_200
-	child/customers_150_300
-	child/orders_150_400
-
-After this, the 'main/orders_100_200' file can be removed. It is no
-longer needed by the child branch, because there is a newer snapshot
-file there. TODO: This optimization hasn't been implemented! The GC
-algorithm will currently keep the file on the 'main' branch anyway, for
-as long as the child branch exists.
-
-
-# TODO: On LSN ranges
-
-In principle, each relation can be checkpointed separately, i.e. the
-LSN ranges of the files don't need to line up. So this would be legal:
-
-	main/orders_100_200
-	main/orders_200_300
-	main/orders_300_400
-	main/customers_150_250
-	main/customers_250_500
-
-However, the code currently always checkpoints all relations together.
-So that situation doesn't arise in practice.
-
-It would also be OK to have overlapping LSN ranges for the same relation:
-
-	main/orders_100_200
-	main/orders_200_300
-	main/orders_250_350
-	main/orders_300_400
-
-The code that reads the snapshot files should cope with this, but this
-situation doesn't arise either, because the checkpointing code never
-does that.  It could be useful, however, as a transient state when
-garbage collecting around branch points, or explicit recovery
-points. For example, if we start with this:
-
-	main/orders_100_200
-	main/orders_200_300
-	main/orders_300_400
-
-And there is a branch or explicit recovery point at LSN 150, we could
-replace 'main/orders_100_200' with 'main/orders_150_150' to keep a
-snapshot only at that exact point that's still needed, removing the
-other page versions around it. But such compaction has not been
-implemented yet.

pageserver/src/layered_repository/inmemory_layer.rs

@@ -1,491 +0,0 @@
-//!
-//! An in-memory layer stores recently received page versions in memory. The page versions
-//! are held in a BTreeMap, and there's another BTreeMap to track the size of the relation.
-//!
-use crate::layered_repository::storage_layer::{
-    Layer, PageReconstructData, PageVersion, SegmentTag, RELISH_SEG_SIZE,
-};
-use crate::layered_repository::LayeredTimeline;
-use crate::layered_repository::SnapshotLayer;
-use crate::repository::WALRecord;
-use crate::PageServerConf;
-use crate::{ZTenantId, ZTimelineId};
-use anyhow::{bail, Result};
-use bytes::Bytes;
-use log::*;
-use std::collections::BTreeMap;
-use std::ops::Bound::Included;
-use std::sync::{Arc, Mutex};
-
-use zenith_utils::lsn::Lsn;
-
-pub struct InMemoryLayer {
-    conf: &'static PageServerConf,
-    tenantid: ZTenantId,
-    timelineid: ZTimelineId,
-    seg: SegmentTag,
-
-    ///
-    /// This layer contains all the changes from 'start_lsn'. The
-    /// start is inclusive. There is no end LSN; we only use in-memory
-    /// layer at the end of a timeline.
-    ///
-    start_lsn: Lsn,
-
-    /// The above fields never change. The parts that do change are in 'inner',
-    /// and protected by mutex.
-    inner: Mutex<InMemoryLayerInner>,
-}
-
-pub struct InMemoryLayerInner {
-    /// If this relation was dropped, remember when that happened.
-    drop_lsn: Option<Lsn>,
-
-    ///
-    /// All versions of all pages in the layer are are kept here.
-    /// Indexed by block number and LSN.
-    ///
-    page_versions: BTreeMap<(u32, Lsn), PageVersion>,
-
-    ///
-    /// `segsizes` tracks the size of the segment at different points in time.
-    ///
-    segsizes: BTreeMap<Lsn, u32>,
-}
-
-impl Layer for InMemoryLayer {
-    fn get_timeline_id(&self) -> ZTimelineId {
-        return self.timelineid;
-    }
-
-    fn get_seg_tag(&self) -> SegmentTag {
-        return self.seg;
-    }
-
-    fn get_start_lsn(&self) -> Lsn {
-        return self.start_lsn;
-    }
-
-    fn get_end_lsn(&self) -> Lsn {
-        let inner = self.inner.lock().unwrap();
-
-        if let Some(drop_lsn) = inner.drop_lsn {
-            drop_lsn
-        } else {
-            Lsn(u64::MAX)
-        }
-    }
-
-    fn is_dropped(&self) -> bool {
-        let inner = self.inner.lock().unwrap();
-        inner.drop_lsn.is_some()
-    }
-
-    /// Look up given page in the cache.
-    fn get_page_reconstruct_data(
-        &self,
-        blknum: u32,
-        lsn: Lsn,
-        reconstruct_data: &mut PageReconstructData,
-    ) -> Result<Option<Lsn>> {
-        // Scan the BTreeMap backwards, starting from reconstruct_data.lsn.
-        let mut need_base_image_lsn: Option<Lsn> = Some(lsn);
-
-        assert!(self.seg.blknum_in_seg(blknum));
-
-        {
-            let inner = self.inner.lock().unwrap();
-            let minkey = (blknum, Lsn(0));
-            let maxkey = (blknum, lsn);
-            let mut iter = inner
-                .page_versions
-                .range((Included(&minkey), Included(&maxkey)));
-            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
-                if let Some(img) = &entry.page_image {
-                    reconstruct_data.page_img = Some(img.clone());
-                    need_base_image_lsn = None;
-                    break;
-                } else if let Some(rec) = &entry.record {
-                    reconstruct_data.records.push(rec.clone());
-                    if rec.will_init {
-                        // This WAL record initializes the page, so no need to go further back
-                        need_base_image_lsn = None;
-                        break;
-                    } else {
-                        need_base_image_lsn = Some(*entry_lsn);
-                    }
-                } else {
-                    // No base image, and no WAL record. Huh?
-                    bail!("no page image or WAL record for requested page");
-                }
-            }
-
-            // release lock on 'page_versions'
-        }
-
-        Ok(need_base_image_lsn)
-    }
-
-    /// Get size of the relation at given LSN
-    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let inner = self.inner.lock().unwrap();
-        let mut iter = inner.segsizes.range((Included(&Lsn(0)), Included(&lsn)));
-
-        if let Some((_entry_lsn, entry)) = iter.next_back() {
-            let result = *entry;
-            drop(inner);
-            trace!("get_seg_size: {} at {} -> {}", self.seg, lsn, result);
-            Ok(result)
-        } else {
-            bail!("No size found for {} at {} in memory", self.seg, lsn);
-        }
-    }
-
-    /// Does this segment exist at given LSN?
-    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
-        let inner = self.inner.lock().unwrap();
-
-        // Is the requested LSN after the segment was dropped?
-        if let Some(drop_lsn) = inner.drop_lsn {
-            if lsn >= drop_lsn {
-                return Ok(false);
-            }
-        }
-
-        // Otherwise, it exists
-        Ok(true)
-    }
-}
-
-impl InMemoryLayer {
-    ///
-    /// Create a new, empty, in-memory layer
-    ///
-    pub fn create(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        seg: SegmentTag,
-        start_lsn: Lsn,
-    ) -> Result<InMemoryLayer> {
-        trace!(
-            "initializing new empty InMemoryLayer for writing {} on timeline {} at {}",
-            seg,
-            timelineid,
-            start_lsn
-        );
-
-        Ok(InMemoryLayer {
-            conf,
-            timelineid,
-            tenantid,
-            seg,
-            start_lsn,
-            inner: Mutex::new(InMemoryLayerInner {
-                drop_lsn: None,
-                page_versions: BTreeMap::new(),
-                segsizes: BTreeMap::new(),
-            }),
-        })
-    }
-
-    // Write operations
-
-    /// Remember new page version, as a WAL record over previous version
-    pub fn put_wal_record(&self, blknum: u32, rec: WALRecord) -> Result<()> {
-        self.put_page_version(
-            blknum,
-            rec.lsn,
-            PageVersion {
-                page_image: None,
-                record: Some(rec),
-            },
-        )
-    }
-
-    /// Remember new page version, as a full page image
-    pub fn put_page_image(&self, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()> {
-        self.put_page_version(
-            blknum,
-            lsn,
-            PageVersion {
-                page_image: Some(img),
-                record: None,
-            },
-        )
-    }
-
-    /// Common subroutine of the public put_wal_record() and put_page_image() functions.
-    /// Adds the page version to the in-memory tree
-    pub fn put_page_version(&self, blknum: u32, lsn: Lsn, pv: PageVersion) -> Result<()> {
-        assert!(self.seg.blknum_in_seg(blknum));
-
-        trace!(
-            "put_page_version blk {} of {} at {}/{}",
-            blknum,
-            self.seg.rel,
-            self.timelineid,
-            lsn
-        );
-        let mut inner = self.inner.lock().unwrap();
-
-        let old = inner.page_versions.insert((blknum, lsn), pv);
-
-        if old.is_some() {
-            // We already had an entry for this LSN. That's odd..
-            warn!(
-                "Page version of rel {} blk {} at {} already exists",
-                self.seg.rel, blknum, lsn
-            );
-        }
-
-        // Also update the relation size, if this extended the relation.
-        if self.seg.rel.is_blocky() {
-            let newsize = blknum - self.seg.segno * RELISH_SEG_SIZE + 1;
-
-            let mut iter = inner.segsizes.range((Included(&Lsn(0)), Included(&lsn)));
-
-            let oldsize;
-            if let Some((_entry_lsn, entry)) = iter.next_back() {
-                oldsize = *entry;
-            } else {
-                oldsize = 0;
-                //bail!("No old size found for {} at {}", self.tag, lsn);
-            }
-            if newsize > oldsize {
-                trace!(
-                    "enlarging segment {} from {} to {} blocks at {}",
-                    self.seg,
-                    oldsize,
-                    newsize,
-                    lsn
-                );
-                inner.segsizes.insert(lsn, newsize);
-            }
-        }
-
-        Ok(())
-    }
-
-    /// Remember that the relation was truncated at given LSN
-    pub fn put_truncation(&self, lsn: Lsn, segsize: u32) -> anyhow::Result<()> {
-        let mut inner = self.inner.lock().unwrap();
-        let old = inner.segsizes.insert(lsn, segsize);
-
-        if old.is_some() {
-            // We already had an entry for this LSN. That's odd..
-            warn!("Inserting truncation, but had an entry for the LSN already");
-        }
-
-        Ok(())
-    }
-
-    /// Remember that the segment was dropped at given LSN
-    pub fn put_unlink(&self, lsn: Lsn) -> anyhow::Result<()> {
-        let mut inner = self.inner.lock().unwrap();
-
-        assert!(inner.drop_lsn.is_none());
-        inner.drop_lsn = Some(lsn);
-
-        info!("dropped segment {} at {}", self.seg, lsn);
-
-        Ok(())
-    }
-
-    ///
-    /// Initialize a new InMemoryLayer for, by copying the state at the given
-    /// point in time from given existing layer.
-    ///
-    pub fn copy_snapshot(
-        conf: &'static PageServerConf,
-        timeline: &LayeredTimeline,
-        src: &dyn Layer,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        lsn: Lsn,
-    ) -> Result<InMemoryLayer> {
-        trace!(
-            "initializing new InMemoryLayer for writing {} on timeline {} at {}",
-            src.get_seg_tag(),
-            timelineid,
-            lsn
-        );
-        let mut page_versions = BTreeMap::new();
-        let mut segsizes = BTreeMap::new();
-
-        let seg = src.get_seg_tag();
-
-        let startblk;
-        let size;
-        if seg.rel.is_blocky() {
-            size = src.get_seg_size(lsn)?;
-            segsizes.insert(lsn, size);
-            startblk = seg.segno * RELISH_SEG_SIZE;
-        } else {
-            size = 1;
-            startblk = 0;
-        }
-
-        for blknum in startblk..(startblk + size) {
-            let img = timeline.materialize_page(seg, blknum, lsn, src)?;
-            let pv = PageVersion {
-                page_image: Some(img),
-                record: None,
-            };
-            page_versions.insert((blknum, lsn), pv);
-        }
-
-        Ok(InMemoryLayer {
-            conf,
-            timelineid,
-            tenantid,
-            seg: src.get_seg_tag(),
-            start_lsn: lsn,
-            inner: Mutex::new(InMemoryLayerInner {
-                drop_lsn: None,
-                page_versions: page_versions,
-                segsizes: segsizes,
-            }),
-        })
-    }
-
-    ///
-    /// Write the this in-memory layer to disk, as a snapshot layer.
-    ///
-    /// The cutoff point for the layer that's written to disk is 'end_lsn'.
-    ///
-    /// Returns new layers that replace this one. Always returns a
-    /// SnapshotLayer containing the page versions that were written to disk,
-    /// but if there were page versions newer than 'end_lsn', also return a new
-    /// in-memory layer containing those page versions. The caller replaces
-    /// this layer with the returned layers in the layer map.
-    ///
-    pub fn freeze(
-        &self,
-        cutoff_lsn: Lsn,
-        // This is needed just to call materialize_page()
-        timeline: &LayeredTimeline,
-    ) -> Result<(Option<Arc<SnapshotLayer>>, Option<Arc<InMemoryLayer>>)> {
-        info!(
-            "freezing in memory layer for {} on timeline {} at {}",
-            self.seg, self.timelineid, cutoff_lsn
-        );
-
-        let inner = self.inner.lock().unwrap();
-
-        // Normally, use the cutoff LSN as the end of the frozen layer.
-        // But if the relation was dropped, we know that there are no
-        // more changes coming in for it, and in particular we know that
-        // there are no changes "in flight" for the LSN anymore, so we use
-        // the drop LSN instead. The drop-LSN could be ahead of the
-        // caller-specified LSN!
-        let dropped = inner.drop_lsn.is_some();
-        let end_lsn = if dropped {
-            inner.drop_lsn.unwrap()
-        } else {
-            cutoff_lsn
-        };
-
-        // Divide all the page versions into old and new at the 'end_lsn' cutoff point.
-        let mut before_page_versions;
-        let mut before_segsizes;
-        let mut after_page_versions;
-        let mut after_segsizes;
-        if !dropped {
-            before_segsizes = BTreeMap::new();
-            after_segsizes = BTreeMap::new();
-            for (lsn, size) in inner.segsizes.iter() {
-                if *lsn > end_lsn {
-                    after_segsizes.insert(*lsn, *size);
-                } else {
-                    before_segsizes.insert(*lsn, *size);
-                }
-            }
-
-            before_page_versions = BTreeMap::new();
-            after_page_versions = BTreeMap::new();
-            for ((blknum, lsn), pv) in inner.page_versions.iter() {
-                if *lsn > end_lsn {
-                    after_page_versions.insert((*blknum, *lsn), pv.clone());
-                } else {
-                    before_page_versions.insert((*blknum, *lsn), pv.clone());
-                }
-            }
-        } else {
-            before_page_versions = inner.page_versions.clone();
-            before_segsizes = inner.segsizes.clone();
-            after_segsizes = BTreeMap::new();
-            after_page_versions = BTreeMap::new();
-        }
-
-        // we can release the lock now.
-        drop(inner);
-
-        // Write the page versions before the cutoff to disk.
-        let snapfile = SnapshotLayer::create(
-            self.conf,
-            self.timelineid,
-            self.tenantid,
-            self.seg,
-            self.start_lsn,
-            end_lsn,
-            dropped,
-            before_page_versions,
-            before_segsizes,
-        )?;
-
-        // If there were any "new" page versions, initialize a new in-memory layer to hold
-        // them
-        let new_open = if !after_segsizes.is_empty() || !after_page_versions.is_empty() {
-            info!("created new in-mem layer for {} {}-", self.seg, end_lsn);
-
-            let new_open = Self::copy_snapshot(
-                self.conf,
-                timeline,
-                &snapfile,
-                self.timelineid,
-                self.tenantid,
-                end_lsn,
-            )?;
-            let mut new_inner = new_open.inner.lock().unwrap();
-            new_inner.page_versions.append(&mut after_page_versions);
-            new_inner.segsizes.append(&mut after_segsizes);
-            drop(new_inner);
-
-            Some(Arc::new(new_open))
-        } else {
-            None
-        };
-
-        let new_historic = Some(Arc::new(snapfile));
-
-        Ok((new_historic, new_open))
-    }
-
-    /// debugging function to print out the contents of the layer
-    #[allow(unused)]
-    pub fn dump(&self) -> String {
-        let mut result = format!(
-            "----- inmemory layer for {} {}-> ----\n",
-            self.seg, self.start_lsn
-        );
-
-        let inner = self.inner.lock().unwrap();
-
-        for (k, v) in inner.segsizes.iter() {
-            result += &format!("{}: {}\n", k, v);
-        }
-        for (k, v) in inner.page_versions.iter() {
-            result += &format!(
-                "blk {} at {}: {}/{}\n",
-                k.0,
-                k.1,
-                v.page_image.is_some(),
-                v.record.is_some()
-            );
-        }
-
-        result
-    }
-}

pageserver/src/layered_repository/layer_map.rs

@@ -1,281 +0,0 @@
-//!
-//! The layer map tracks what layers exist for all the relations in a timeline.
-//!
-//! When the timeline is first accessed, the server lists of all snapshot files
-//! in the timelines/<timelineid> directory, and populates this map with
-//! SnapshotLayers corresponding to each file. When new WAL is received,
-//! we create InMemoryLayers to hold the incoming records. Now and then,
-//! in the checkpoint() function, the in-memory layers are frozen, forming
-//! new snapshot layers and corresponding files are written to disk.
-//!
-
-use crate::layered_repository::storage_layer::{Layer, SegmentTag};
-use crate::layered_repository::{InMemoryLayer, SnapshotLayer};
-use crate::relish::*;
-use anyhow::Result;
-use log::*;
-use std::collections::HashSet;
-use std::collections::{BTreeMap, HashMap};
-use std::ops::Bound::Included;
-use std::sync::Arc;
-use zenith_utils::lsn::Lsn;
-
-///
-/// LayerMap tracks what layers exist or a timeline. The last layer that is
-/// open for writes is always an InMemoryLayer, and is tracked separately
-/// because there can be only one for each segment. The older layers,
-/// stored on disk, are kept in a BTreeMap keyed by the layer's start LSN.
-///
-pub struct LayerMap {
-    segs: HashMap<SegmentTag, SegEntry>,
-}
-
-struct SegEntry {
-    pub open: Option<Arc<InMemoryLayer>>,
-    pub historic: BTreeMap<Lsn, Arc<SnapshotLayer>>,
-}
-
-impl LayerMap {
-    ///
-    /// Look up using the given segment tag and LSN. This differs from a plain
-    /// key-value lookup in that if there is any layer that covers the
-    /// given LSN, or precedes the given LSN, it is returned. In other words,
-    /// you don't need to know the exact start LSN of the layer.
-    ///
-    pub fn get(&self, tag: &SegmentTag, lsn: Lsn) -> Option<Arc<dyn Layer>> {
-        let segentry = self.segs.get(tag)?;
-
-        if let Some(open) = &segentry.open {
-            if open.get_start_lsn() <= lsn {
-                let x: Arc<dyn Layer> = Arc::clone(&open) as _;
-                return Some(x);
-            }
-        }
-
-        if let Some((_k, v)) = segentry
-            .historic
-            .range((Included(Lsn(0)), Included(lsn)))
-            .next_back()
-        {
-            let x: Arc<dyn Layer> = Arc::clone(&v) as _;
-            Some(x)
-        } else {
-            None
-        }
-    }
-
-    ///
-    /// Get the open layer for given segment for writing. Or None if no open
-    /// layer exists.
-    ///
-    pub fn get_open(&self, tag: &SegmentTag) -> Option<Arc<InMemoryLayer>> {
-        let segentry = self.segs.get(tag)?;
-
-        if let Some(open) = &segentry.open {
-            Some(Arc::clone(open))
-        } else {
-            None
-        }
-    }
-
-    ///
-    /// Insert an open in-memory layer
-    ///
-    pub fn insert_open(&mut self, layer: Arc<InMemoryLayer>) {
-        let tag = layer.get_seg_tag();
-
-        if let Some(segentry) = self.segs.get_mut(&tag) {
-            if let Some(_old) = &segentry.open {
-                // FIXME: shouldn't exist, but check
-            }
-            segentry.open = Some(layer);
-        } else {
-            let segentry = SegEntry {
-                open: Some(layer),
-                historic: BTreeMap::new(),
-            };
-            self.segs.insert(tag, segentry);
-        }
-    }
-
-    ///
-    /// Insert an on-disk layer
-    ///
-    pub fn insert_historic(&mut self, layer: Arc<SnapshotLayer>) {
-        let tag = layer.get_seg_tag();
-        let start_lsn = layer.get_start_lsn();
-
-        if let Some(segentry) = self.segs.get_mut(&tag) {
-            segentry.historic.insert(start_lsn, layer);
-        } else {
-            let mut historic = BTreeMap::new();
-            historic.insert(start_lsn, layer);
-
-            let segentry = SegEntry {
-                open: None,
-                historic,
-            };
-            self.segs.insert(tag, segentry);
-        }
-    }
-
-    ///
-    /// Remove an on-disk layer from the map.
-    ///
-    /// This should be called when the corresponding file on disk has been deleted.
-    ///
-    pub fn remove_historic(&mut self, layer: &SnapshotLayer) {
-        let tag = layer.get_seg_tag();
-        let start_lsn = layer.get_start_lsn();
-
-        if let Some(segentry) = self.segs.get_mut(&tag) {
-            segentry.historic.remove(&start_lsn);
-        }
-    }
-
-    pub fn list_rels(&self, spcnode: u32, dbnode: u32) -> Result<HashSet<RelTag>> {
-        let mut rels: HashSet<RelTag> = HashSet::new();
-
-        for (seg, _entry) in self.segs.iter() {
-            if let RelishTag::Relation(reltag) = seg.rel {
-                // FIXME: skip if it was dropped before the requested LSN. But there is no
-                // LSN argument
-
-                if (spcnode == 0 || reltag.spcnode == spcnode)
-                    && (dbnode == 0 || reltag.dbnode == dbnode)
-                {
-                    rels.insert(reltag);
-                }
-            }
-        }
-        Ok(rels)
-    }
-
-    pub fn list_nonrels(&self, _lsn: Lsn) -> Result<HashSet<RelishTag>> {
-        let mut rels: HashSet<RelishTag> = HashSet::new();
-
-        // Scan the timeline directory to get all rels in this timeline.
-        for (seg, _entry) in self.segs.iter() {
-            // FIXME: skip if it was dropped before the requested LSN.
-
-            if let RelishTag::Relation(_) = seg.rel {
-            } else {
-                rels.insert(seg.rel);
-            }
-        }
-        Ok(rels)
-    }
-
-    /// Is there a newer layer for given segment?
-    pub fn newer_layer_exists(&self, seg: SegmentTag, lsn: Lsn) -> bool {
-        if let Some(segentry) = self.segs.get(&seg) {
-            if let Some(_open) = &segentry.open {
-                return true;
-            }
-
-            for (newer_lsn, layer) in segentry
-                .historic
-                .range((Included(lsn), Included(Lsn(u64::MAX))))
-            {
-                if layer.get_end_lsn() > lsn {
-                    trace!(
-                        "found later layer for {}, {} {}-{}",
-                        seg,
-                        lsn,
-                        newer_lsn,
-                        layer.get_end_lsn()
-                    );
-                    return true;
-                } else {
-                    trace!("found singleton layer for {}, {} {}", seg, lsn, newer_lsn);
-                    continue;
-                }
-            }
-        }
-        trace!("no later layer found for {}, {}", seg, lsn);
-        false
-    }
-
-    pub fn iter_open_layers(&mut self) -> OpenLayerIter {
-        OpenLayerIter {
-            last: None,
-            segiter: self.segs.iter_mut(),
-        }
-    }
-
-    pub fn iter_historic_layers(&self) -> HistoricLayerIter {
-        HistoricLayerIter {
-            segiter: self.segs.iter(),
-            iter: None,
-        }
-    }
-}
-
-impl Default for LayerMap {
-    fn default() -> Self {
-        LayerMap {
-            segs: HashMap::new(),
-        }
-    }
-}
-
-pub struct OpenLayerIter<'a> {
-    last: Option<&'a mut SegEntry>,
-
-    segiter: std::collections::hash_map::IterMut<'a, SegmentTag, SegEntry>,
-}
-
-impl<'a> OpenLayerIter<'a> {
-    pub fn replace(&mut self, replacement: Option<Arc<InMemoryLayer>>) {
-        let segentry = self.last.as_mut().unwrap();
-        segentry.open = replacement;
-    }
-
-    pub fn insert_historic(&mut self, new_layer: Arc<SnapshotLayer>) {
-        let start_lsn = new_layer.get_start_lsn();
-
-        let segentry = self.last.as_mut().unwrap();
-        segentry.historic.insert(start_lsn, new_layer);
-    }
-}
-
-impl<'a> Iterator for OpenLayerIter<'a> {
-    type Item = Arc<InMemoryLayer>;
-
-    fn next(&mut self) -> std::option::Option<<Self as std::iter::Iterator>::Item> {
-        while let Some((_seg, entry)) = self.segiter.next() {
-            if let Some(open) = &entry.open {
-                let op = Arc::clone(&open);
-                self.last = Some(entry);
-                return Some(op);
-            }
-        }
-        self.last = None;
-        None
-    }
-}
-
-pub struct HistoricLayerIter<'a> {
-    segiter: std::collections::hash_map::Iter<'a, SegmentTag, SegEntry>,
-    iter: Option<std::collections::btree_map::Iter<'a, Lsn, Arc<SnapshotLayer>>>,
-}
-
-impl<'a> Iterator for HistoricLayerIter<'a> {
-    type Item = Arc<SnapshotLayer>;
-
-    fn next(&mut self) -> std::option::Option<<Self as std::iter::Iterator>::Item> {
-        loop {
-            if let Some(x) = &mut self.iter {
-                if let Some(x) = x.next() {
-                    return Some(Arc::clone(&*x.1));
-                }
-            }
-            if let Some(seg) = self.segiter.next() {
-                self.iter = Some(seg.1.historic.iter());
-                continue;
-            } else {
-                return None;
-            }
-        }
-    }
-}

pageserver/src/layered_repository/snapshot_layer.rs

@@ -1,547 +0,0 @@
-//!
-//! A SnapshotLayer represents one snapshot file on disk. One file holds all page
-//! version and size information of one relation, in a range of LSN.
-//! The name "snapshot file" is a bit of a misnomer because a snapshot file doesn't
-//! contain a snapshot at a specific LSN, but rather all the page versions in a range
-//! of LSNs.
-//!
-//! Currently, a snapshot file contains full information needed to reconstruct any
-//! page version in the LSN range, without consulting any other snapshot files. When
-//! a new snapshot file is created for writing, the full contents of relation are
-//! materialized as it is at the beginning of the LSN range. That can be very expensive,
-//! we should find a way to store differential files. But this keeps the read-side
-//! of things simple. You can find the correct snapshot file based on RelishTag and
-//! timeline+LSN, and once you've located it, you have all the data you need to in that
-//! file.
-//!
-//! When a snapshot file needs to be accessed, we slurp the whole file into memory, into
-//! the SnapshotLayer struct. See load() and unload() functions.
-//!
-//! On disk, the snapshot files are stored in timelines/<timelineid> directory.
-//! Currently, there are no subdirectories, and each snapshot file is named like this:
-//!
-//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<start LSN>_<end LSN>
-//!
-//! For example:
-//!
-//!    1663_13990_2609_0_000000000169C348_000000000169C349
-//!
-//! If a relation is dropped, we add a '_DROPPED' to the end of the filename to indicate that.
-//! So the above example would become:
-//!
-//!    1663_13990_2609_0_000000000169C348_000000000169C349_DROPPED
-//!
-//! The end LSN indicates when it was dropped in that case, we don't store it in the
-//! file contents in any way.
-//!
-//! A snapshot file is constructed using the 'bookfile' crate. Each file consists of two
-//! parts: the page versions and the relation sizes. They are stored as separate chapters.
-//!
-use crate::layered_repository::storage_layer::{
-    Layer, PageReconstructData, PageVersion, SegmentTag,
-};
-use crate::relish::*;
-use crate::PageServerConf;
-use crate::{ZTenantId, ZTimelineId};
-use anyhow::{bail, Result};
-use log::*;
-use std::collections::BTreeMap;
-use std::fmt;
-use std::fs;
-use std::fs::File;
-use std::io::Write;
-use std::ops::Bound::Included;
-use std::path::PathBuf;
-use std::sync::{Arc, Mutex, MutexGuard};
-
-use bookfile::{Book, BookWriter};
-
-use zenith_utils::bin_ser::BeSer;
-use zenith_utils::lsn::Lsn;
-
-// Magic constant to identify a Zenith snapshot file
-static SNAPSHOT_FILE_MAGIC: u32 = 0x5A616E01;
-
-static PAGE_VERSIONS_CHAPTER: u64 = 1;
-static REL_SIZES_CHAPTER: u64 = 2;
-
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
-struct SnapshotFileName {
-    seg: SegmentTag,
-    start_lsn: Lsn,
-    end_lsn: Lsn,
-    dropped: bool,
-}
-
-impl SnapshotFileName {
-    fn from_str(fname: &str) -> Option<Self> {
-        // Split the filename into parts
-        //
-        //    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>
-        //
-        // or if it was dropped:
-        //
-        //    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>_DROPPED
-        //
-        let rel;
-        let mut parts;
-        if let Some(rest) = fname.strip_prefix("rel_") {
-            parts = rest.split('_');
-            rel = RelishTag::Relation(RelTag {
-                spcnode: parts.next()?.parse::<u32>().ok()?,
-                dbnode: parts.next()?.parse::<u32>().ok()?,
-                relnode: parts.next()?.parse::<u32>().ok()?,
-                forknum: parts.next()?.parse::<u8>().ok()?,
-            });
-        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::Clog,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::MultiXactMembers,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::MultiXactOffsets,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
-            parts = rest.split('_');
-            rel = RelishTag::FileNodeMap {
-                spcnode: parts.next()?.parse::<u32>().ok()?,
-                dbnode: parts.next()?.parse::<u32>().ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
-            parts = rest.split('_');
-            rel = RelishTag::TwoPhase {
-                xid: parts.next()?.parse::<u32>().ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
-            parts = rest.split('_');
-            rel = RelishTag::Checkpoint;
-        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
-            parts = rest.split('_');
-            rel = RelishTag::ControlFile;
-        } else {
-            return None;
-        }
-
-        let segno = parts.next()?.parse::<u32>().ok()?;
-
-        let seg = SegmentTag { rel, segno };
-
-        let start_lsn = Lsn::from_hex(parts.next()?).ok()?;
-        let end_lsn = Lsn::from_hex(parts.next()?).ok()?;
-
-        let mut dropped = false;
-        if let Some(suffix) = parts.next() {
-            if suffix == "DROPPED" {
-                dropped = true;
-            } else {
-                warn!("unrecognized filename in timeline dir: {}", fname);
-                return None;
-            }
-        }
-        if parts.next().is_some() {
-            warn!("unrecognized filename in timeline dir: {}", fname);
-            return None;
-        }
-
-        Some(SnapshotFileName {
-            seg,
-            start_lsn,
-            end_lsn,
-            dropped,
-        })
-    }
-
-    fn to_string(&self) -> String {
-        let basename = match self.seg.rel {
-            RelishTag::Relation(reltag) => format!(
-                "rel_{}_{}_{}_{}",
-                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
-            ),
-            RelishTag::Slru {
-                slru: SlruKind::Clog,
-                segno,
-            } => format!("pg_xact_{:04X}", segno),
-            RelishTag::Slru {
-                slru: SlruKind::MultiXactMembers,
-                segno,
-            } => format!("pg_multixact_members_{:04X}", segno),
-            RelishTag::Slru {
-                slru: SlruKind::MultiXactOffsets,
-                segno,
-            } => format!("pg_multixact_offsets_{:04X}", segno),
-            RelishTag::FileNodeMap { spcnode, dbnode } => {
-                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
-            }
-            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
-            RelishTag::Checkpoint => format!("pg_control_checkpoint"),
-            RelishTag::ControlFile => format!("pg_control"),
-        };
-
-        format!(
-            "{}_{}_{:016X}_{:016X}{}",
-            basename,
-            self.seg.segno,
-            u64::from(self.start_lsn),
-            u64::from(self.end_lsn),
-            if self.dropped { "_DROPPED" } else { "" }
-        )
-    }
-}
-
-impl fmt::Display for SnapshotFileName {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}", self.to_string())
-    }
-}
-
-///
-/// SnapshotLayer is the in-memory data structure associated with an
-/// on-disk snapshot file.  We keep a SnapshotLayer in memory for each
-/// file, in the LayerMap. If a layer is in "loaded" state, we have a
-/// copy of the file in memory, in 'inner'. Otherwise the struct is
-/// just a placeholder for a file that exists on disk, and it needs to
-/// be loaded before using it in queries.
-///
-pub struct SnapshotLayer {
-    conf: &'static PageServerConf,
-    pub tenantid: ZTenantId,
-    pub timelineid: ZTimelineId,
-    pub seg: SegmentTag,
-
-    //
-    // This entry contains all the changes from 'start_lsn' to 'end_lsn'. The
-    // start is inclusive, and end is exclusive.
-    pub start_lsn: Lsn,
-    pub end_lsn: Lsn,
-
-    dropped: bool,
-
-    inner: Mutex<SnapshotLayerInner>,
-}
-
-pub struct SnapshotLayerInner {
-    /// If false, the 'page_versions' and 'relsizes' have not been
-    /// loaded into memory yet.
-    loaded: bool,
-
-    /// All versions of all pages in the file are are kept here.
-    /// Indexed by block number and LSN.
-    page_versions: BTreeMap<(u32, Lsn), PageVersion>,
-
-    /// `relsizes` tracks the size of the relation at different points in time.
-    relsizes: BTreeMap<Lsn, u32>,
-}
-
-impl Layer for SnapshotLayer {
-    fn get_timeline_id(&self) -> ZTimelineId {
-        return self.timelineid;
-    }
-
-    fn get_seg_tag(&self) -> SegmentTag {
-        return self.seg;
-    }
-
-    fn is_dropped(&self) -> bool {
-        return self.dropped;
-    }
-
-    fn get_start_lsn(&self) -> Lsn {
-        return self.start_lsn;
-    }
-
-    fn get_end_lsn(&self) -> Lsn {
-        return self.end_lsn;
-    }
-
-    /// Look up given page in the cache.
-    fn get_page_reconstruct_data(
-        &self,
-        blknum: u32,
-        lsn: Lsn,
-        reconstruct_data: &mut PageReconstructData,
-    ) -> Result<Option<Lsn>> {
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let mut need_base_image_lsn: Option<Lsn> = Some(lsn);
-        {
-            let inner = self.load()?;
-            let minkey = (blknum, Lsn(0));
-            let maxkey = (blknum, lsn);
-            let mut iter = inner
-                .page_versions
-                .range((Included(&minkey), Included(&maxkey)));
-            while let Some(((_blknum, entry_lsn), entry)) = iter.next_back() {
-                if let Some(img) = &entry.page_image {
-                    reconstruct_data.page_img = Some(img.clone());
-                    need_base_image_lsn = None;
-                    break;
-                } else if let Some(rec) = &entry.record {
-                    reconstruct_data.records.push(rec.clone());
-                    if rec.will_init {
-                        // This WAL record initializes the page, so no need to go further back
-                        need_base_image_lsn = None;
-                        break;
-                    } else {
-                        need_base_image_lsn = Some(*entry_lsn);
-                    }
-                } else {
-                    // No base image, and no WAL record. Huh?
-                    bail!("no page image or WAL record for requested page");
-                }
-            }
-
-            // release lock on 'inner'
-        }
-
-        Ok(need_base_image_lsn)
-    }
-
-    /// Get size of the relation at given LSN
-    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let inner = self.load()?;
-        let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn)));
-
-        if let Some((_entry_lsn, entry)) = iter.next_back() {
-            let result = *entry;
-            drop(inner);
-            trace!("get_seg_size: {} at {} -> {}", self.seg, lsn, result);
-            Ok(result)
-        } else {
-            error!(
-                "No size found for {} at {} in snapshot layer {} {}-{}",
-                self.seg, lsn, self.seg, self.start_lsn, self.end_lsn
-            );
-            bail!(
-                "No size found for {} at {} in snapshot layer",
-                self.seg,
-                lsn
-            );
-        }
-    }
-
-    /// Does this segment exist at given LSN?
-    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
-        // Is the requested LSN after the rel was dropped?
-        if self.dropped && lsn >= self.end_lsn {
-            return Ok(false);
-        }
-
-        // Otherwise, it exists.
-        Ok(true)
-    }
-}
-
-impl SnapshotLayer {
-    fn path(&self) -> PathBuf {
-        Self::path_for(
-            self.conf,
-            self.timelineid,
-            self.tenantid,
-            &SnapshotFileName {
-                seg: self.seg,
-                start_lsn: self.start_lsn,
-                end_lsn: self.end_lsn,
-                dropped: self.dropped,
-            },
-        )
-    }
-
-    fn path_for(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        fname: &SnapshotFileName,
-    ) -> PathBuf {
-        conf.timeline_path(&timelineid, &tenantid)
-            .join(fname.to_string())
-    }
-
-    /// Create a new snapshot file, using the given btreemaps containing the page versions and
-    /// relsizes.
-    ///
-    /// This is used to write the in-memory layer to disk. The in-memory layer uses the same
-    /// data structure with two btreemaps as we do, so passing the btreemaps is currently
-    /// expedient.
-    pub fn create(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        seg: SegmentTag,
-        start_lsn: Lsn,
-        end_lsn: Lsn,
-        dropped: bool,
-        page_versions: BTreeMap<(u32, Lsn), PageVersion>,
-        relsizes: BTreeMap<Lsn, u32>,
-    ) -> Result<SnapshotLayer> {
-        let snapfile = SnapshotLayer {
-            conf: conf,
-            timelineid: timelineid,
-            tenantid: tenantid,
-            seg: seg,
-            start_lsn: start_lsn,
-            end_lsn,
-            dropped,
-            inner: Mutex::new(SnapshotLayerInner {
-                loaded: true,
-                page_versions: page_versions,
-                relsizes: relsizes,
-            }),
-        };
-        let inner = snapfile.inner.lock().unwrap();
-
-        // Write the in-memory btreemaps into a file
-        let path = snapfile.path();
-
-        // Note: This overwrites any existing file. There shouldn't be any.
-        // FIXME: throw an error instead?
-        let file = File::create(&path)?;
-        let book = BookWriter::new(file, SNAPSHOT_FILE_MAGIC)?;
-
-        // Write out page versions
-        let mut chapter = book.new_chapter(PAGE_VERSIONS_CHAPTER);
-        let buf = BTreeMap::ser(&inner.page_versions)?;
-        chapter.write_all(&buf)?;
-        let book = chapter.close()?;
-
-        // and relsizes to separate chapter
-        let mut chapter = book.new_chapter(REL_SIZES_CHAPTER);
-        let buf = BTreeMap::ser(&inner.relsizes)?;
-        chapter.write_all(&buf)?;
-        let book = chapter.close()?;
-
-        book.close()?;
-
-        trace!("saved {}", &path.display());
-
-        drop(inner);
-
-        Ok(snapfile)
-    }
-
-    ///
-    /// Load the contents of the file into memory
-    ///
-    fn load(&self) -> Result<MutexGuard<SnapshotLayerInner>> {
-        // quick exit if already loaded
-        let mut inner = self.inner.lock().unwrap();
-
-        if inner.loaded {
-            return Ok(inner);
-        }
-
-        let path = Self::path_for(
-            self.conf,
-            self.timelineid,
-            self.tenantid,
-            &SnapshotFileName {
-                seg: self.seg,
-                start_lsn: self.start_lsn,
-                end_lsn: self.end_lsn,
-                dropped: self.dropped,
-            },
-        );
-
-        let file = File::open(&path)?;
-        let book = Book::new(file)?;
-
-        let chapter = book.read_chapter(PAGE_VERSIONS_CHAPTER)?;
-        let page_versions = BTreeMap::des(&chapter)?;
-
-        let chapter = book.read_chapter(REL_SIZES_CHAPTER)?;
-        let relsizes = BTreeMap::des(&chapter)?;
-
-        debug!("loaded from {}", &path.display());
-
-        *inner = SnapshotLayerInner {
-            loaded: true,
-            page_versions,
-            relsizes,
-        };
-
-        Ok(inner)
-    }
-
-    /// Create SnapshotLayers representing all files on disk
-    ///
-    // TODO: returning an Iterator would be more idiomatic
-    pub fn list_snapshot_files(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-    ) -> Result<Vec<Arc<SnapshotLayer>>> {
-        let path = conf.timeline_path(&timelineid, &tenantid);
-
-        let mut snapfiles: Vec<Arc<SnapshotLayer>> = Vec::new();
-        for direntry in fs::read_dir(path)? {
-            let fname = direntry?.file_name();
-            let fname = fname.to_str().unwrap();
-
-            if let Some(snapfilename) = SnapshotFileName::from_str(fname) {
-                let snapfile = SnapshotLayer {
-                    conf,
-                    timelineid,
-                    tenantid,
-                    seg: snapfilename.seg,
-                    start_lsn: snapfilename.start_lsn,
-                    end_lsn: snapfilename.end_lsn,
-                    dropped: snapfilename.dropped,
-                    inner: Mutex::new(SnapshotLayerInner {
-                        loaded: false,
-                        page_versions: BTreeMap::new(),
-                        relsizes: BTreeMap::new(),
-                    }),
-                };
-
-                snapfiles.push(Arc::new(snapfile));
-            }
-        }
-        return Ok(snapfiles);
-    }
-
-    pub fn delete(&self) -> Result<()> {
-        // delete underlying file
-        fs::remove_file(self.path())?;
-        Ok(())
-    }
-
-    ///
-    /// Release most of the memory used by this layer. If it's accessed again later,
-    /// it will need to be loaded back.
-    ///
-    pub fn unload(&self) -> Result<()> {
-        let mut inner = self.inner.lock().unwrap();
-        inner.page_versions = BTreeMap::new();
-        inner.relsizes = BTreeMap::new();
-        inner.loaded = false;
-        Ok(())
-    }
-
-    /// debugging function to print out the contents of the layer
-    #[allow(unused)]
-    pub fn dump(&self) -> String {
-        let mut result = format!(
-            "----- snapshot layer for {} {}-{} ----\n",
-            self.seg, self.start_lsn, self.end_lsn
-        );
-
-        let inner = self.inner.lock().unwrap();
-        for (k, v) in inner.relsizes.iter() {
-            result += &format!("{}: {}\n", k, v);
-        }
-        //for (k, v) in inner.page_versions.iter() {
-        //    result += &format!("blk {} at {}: {}/{}\n", k.0, k.1, v.page_image.is_some(), v.record.is_some());
-        //}
-
-        result
-    }
-}

pageserver/src/layered_repository/storage_layer.rs

@@ -1,128 +0,0 @@
-//!
-//! Common traits and structs for layers
-//!
-
-use crate::relish::RelishTag;
-use crate::repository::WALRecord;
-use crate::ZTimelineId;
-use anyhow::Result;
-use bytes::Bytes;
-use serde::{Deserialize, Serialize};
-use std::fmt;
-
-use zenith_utils::lsn::Lsn;
-
-// Size of one segment in pages (10 MB)
-pub const RELISH_SEG_SIZE: u32 = 10 * 1024 * 1024 / 8192;
-
-///
-/// Each relish stored in the repository is divided into fixed-sized "segments",
-/// with 10 MB of key-space, or 1280 8k pages each.
-///
-#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy)]
-pub struct SegmentTag {
-    pub rel: RelishTag,
-    pub segno: u32,
-}
-
-impl fmt::Display for SegmentTag {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}.{}", self.rel, self.segno)
-    }
-}
-
-impl SegmentTag {
-    pub const fn from_blknum(rel: RelishTag, blknum: u32) -> SegmentTag {
-        SegmentTag {
-            rel,
-            segno: blknum / RELISH_SEG_SIZE,
-        }
-    }
-
-    pub fn blknum_in_seg(&self, blknum: u32) -> bool {
-        blknum / RELISH_SEG_SIZE == self.segno
-    }
-}
-
-///
-/// Represents a version of a page at a specific LSN. The LSN is the key of the
-/// entry in the 'page_versions' hash, it is not duplicated here.
-///
-/// A page version can be stored as a full page image, or as WAL record that needs
-/// to be applied over the previous page version to reconstruct this version.
-///
-/// It's also possible to have both a WAL record and a page image in the same
-/// PageVersion. That happens if page version is originally stored as a WAL record
-/// but it is later reconstructed by a GetPage@LSN request by performing WAL
-/// redo. The get_page_at_lsn() code will store the reconstructed pag image next to
-/// the WAL record in that case. TODO: That's pretty accidental, not the result
-/// of any grand design. If we want to keep reconstructed page versions around, we
-/// probably should have a separate buffer cache so that we could control the
-/// replacement policy globally. Or if we keep a reconstructed page image, we
-/// could throw away the WAL record.
-///
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct PageVersion {
-    /// an 8kb page image
-    pub page_image: Option<Bytes>,
-    /// WAL record to get from previous page version to this one.
-    pub record: Option<WALRecord>,
-}
-
-///
-/// Data needed to reconstruct a page version
-///
-/// 'page_img' is the old base image of the page to start the WAL replay with.
-/// It can be None, if the first WAL record initializes the page (will_init)
-/// 'records' contains the records to apply over the base image.
-///
-pub struct PageReconstructData {
-    pub records: Vec<WALRecord>,
-    pub page_img: Option<Bytes>,
-}
-
-///
-/// A Layer holds all page versions for one segment of a relish, in a range of LSNs.
-/// There are two kinds of layers, in-memory and snapshot layers. In-memory
-/// layers are used to ingest incoming WAL, and provide fast access
-/// to the recent page versions. Snaphot layers are stored on disk, and
-/// are immutable. This trait presents the common functionality of
-/// in-memory and snapshot layers.
-///
-/// Each layer contains a full snapshot of the segment at the start
-/// LSN. In addition to that, it contains WAL (or more page images)
-/// needed to recontruct any page version up to the end LSN.
-///
-pub trait Layer: Send + Sync {
-    // These functions identify the relish segment and the LSN range
-    // that this Layer holds.
-    fn get_timeline_id(&self) -> ZTimelineId;
-    fn get_seg_tag(&self) -> SegmentTag;
-    fn get_start_lsn(&self) -> Lsn;
-    fn get_end_lsn(&self) -> Lsn;
-    fn is_dropped(&self) -> bool;
-
-    ///
-    /// Return data needed to reconstruct given page at LSN.
-    ///
-    /// It is up to the caller to collect more data from previous layer and
-    /// perform WAL redo, if necessary.
-    ///
-    /// If returns Some, the returned data is not complete. The caller needs
-    /// to continue with the returned 'lsn'.
-    ///
-    /// Note that the 'blknum' is the offset of the page from the beginning
-    /// of the *relish*, not the beginning of the segment. The requested
-    /// 'blknum' must be covered by this segment.
-    fn get_page_reconstruct_data(
-        &self,
-        blknum: u32,
-        lsn: Lsn,
-        reconstruct_data: &mut PageReconstructData,
-    ) -> Result<Option<Lsn>>;
-
-    // Functions that correspond to the Timeline trait functions.
-    fn get_seg_size(&self, lsn: Lsn) -> Result<u32>;
-
-    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool>;
-}

pageserver/src/lib.rs

@@ -1,124 +1,26 @@
-use zenith_utils::postgres_backend::AuthType;
-use zenith_utils::zid::{ZTenantId, ZTimelineId};
-
+use std::net::SocketAddr;
 use std::path::PathBuf;
-use std::time::Duration;
 
-use lazy_static::lazy_static;
-use zenith_metrics::{register_int_gauge_vec, IntGaugeVec};
-
-pub mod basebackup;
-pub mod branches;
-pub mod layered_repository;
-pub mod logger;
-pub mod object_key;
-pub mod object_repository;
-pub mod object_store;
+pub mod controlfile;
 pub mod page_cache;
 pub mod page_service;
-pub mod relish;
-pub mod repository;
-pub mod restore_local_repo;
-pub mod rocksdb_storage;
+#[allow(dead_code)]
+pub mod pg_constants;
+pub mod restore_s3;
+pub mod tui;
+pub mod tui_event;
+mod tui_logger;
 pub mod waldecoder;
 pub mod walreceiver;
 pub mod walredo;
 
-lazy_static! {
-    static ref LIVE_CONNECTIONS_COUNT: IntGaugeVec = register_int_gauge_vec!(
-        "pageserver_live_connections_count",
-        "Number of live network connections",
-        &["pageserver_connection_kind"]
-    )
-    .expect("failed to define a metric");
-}
-
+#[allow(dead_code)]
 #[derive(Debug, Clone)]
 pub struct PageServerConf {
+    pub data_dir: PathBuf,
     pub daemonize: bool,
-    pub listen_addr: String,
-    pub http_endpoint_addr: String,
-    pub gc_horizon: u64,
-    pub gc_period: Duration,
-    pub superuser: String,
-
-    // Repository directory, relative to current working directory.
-    // Normally, the page server changes the current working directory
-    // to the repository, and 'workdir' is always '.'. But we don't do
-    // that during unit testing, because the current directory is global
-    // to the process but different unit tests work on different
-    // repositories.
-    pub workdir: PathBuf,
-
-    pub pg_distrib_dir: PathBuf,
-
-    pub auth_type: AuthType,
-
-    pub auth_validation_public_key_path: Option<PathBuf>,
-
-    pub repository_format: RepositoryFormat,
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub enum RepositoryFormat {
-    Layered,
-    RocksDb,
-}
-
-impl PageServerConf {
-    //
-    // Repository paths, relative to workdir.
-    //
-
-    fn tenants_path(&self) -> PathBuf {
-        self.workdir.join("tenants")
-    }
-
-    fn tenant_path(&self, tenantid: &ZTenantId) -> PathBuf {
-        self.tenants_path().join(tenantid.to_string())
-    }
-
-    fn tags_path(&self, tenantid: &ZTenantId) -> PathBuf {
-        self.tenant_path(tenantid).join("refs").join("tags")
-    }
-
-    fn tag_path(&self, tag_name: &str, tenantid: &ZTenantId) -> PathBuf {
-        self.tags_path(tenantid).join(tag_name)
-    }
-
-    fn branches_path(&self, tenantid: &ZTenantId) -> PathBuf {
-        self.tenant_path(tenantid).join("refs").join("branches")
-    }
-
-    fn branch_path(&self, branch_name: &str, tenantid: &ZTenantId) -> PathBuf {
-        self.branches_path(tenantid).join(branch_name)
-    }
-
-    fn timelines_path(&self, tenantid: &ZTenantId) -> PathBuf {
-        self.tenant_path(tenantid).join("timelines")
-    }
-
-    fn timeline_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
-        self.timelines_path(tenantid).join(timelineid.to_string())
-    }
-
-    fn ancestor_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
-        self.timeline_path(timelineid, tenantid).join("ancestor")
-    }
-
-    fn wal_dir_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
-        self.timeline_path(timelineid, tenantid).join("wal")
-    }
-
-    //
-    // Postgres distribution paths
-    //
-
-    pub fn pg_bin_dir(&self) -> PathBuf {
-        self.pg_distrib_dir.join("bin")
-    }
-
-    pub fn pg_lib_dir(&self) -> PathBuf {
-        self.pg_distrib_dir.join("lib")
-    }
+    pub interactive: bool,
+    pub wal_producer_connstr: Option<String>,
+    pub listen_addr: SocketAddr,
+    pub skip_recovery: bool,
 }

pageserver/src/logger.rs

@@ -1,45 +0,0 @@
-use crate::PageServerConf;
-
-use anyhow::{Context, Result};
-use slog::{Drain, FnValue};
-use std::fs::{File, OpenOptions};
-
-pub fn init_logging(
-    _conf: &PageServerConf,
-    log_filename: &str,
-) -> Result<(slog_scope::GlobalLoggerGuard, File)> {
-    // Don't open the same file for output multiple times;
-    // the different fds could overwrite each other's output.
-    let log_file = OpenOptions::new()
-        .create(true)
-        .append(true)
-        .open(&log_filename)
-        .with_context(|| format!("failed to open {:?}", &log_filename))?;
-
-    let logger_file = log_file.try_clone().unwrap();
-
-    let decorator = slog_term::PlainSyncDecorator::new(logger_file);
-    let drain = slog_term::FullFormat::new(decorator).build();
-    let drain = slog::Filter::new(drain, |record: &slog::Record| {
-        if record.level().is_at_least(slog::Level::Info) {
-            return true;
-        }
-        false
-    });
-    let drain = std::sync::Mutex::new(drain).fuse();
-    let logger = slog::Logger::root(
-        drain,
-        slog::o!(
-            "location" =>
-                FnValue(move |record| {
-                    format!("{}, {}:{}",
-                            record.module(),
-                            record.file(),
-                            record.line()
-                    )
-                }
-                )
-        ),
-    );
-    Ok((slog_scope::set_global_logger(logger), log_file))
-}

pageserver/src/object_key.rs

@@ -1,49 +0,0 @@
-//!
-//! Common structs shared by object_repository.rs and object_store.rs.
-//!
-
-use crate::relish::RelishTag;
-use serde::{Deserialize, Serialize};
-use zenith_utils::zid::ZTimelineId;
-
-///
-/// ObjectKey is the key type used to identify objects stored in an object
-/// repository. It is shared between object_repository.rs and object_store.rs.
-/// It is mostly opaque to ObjectStore, it just stores and retrieves objects
-/// using the key given by the caller.
-///
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ObjectKey {
-    pub timeline: ZTimelineId,
-    pub tag: ObjectTag,
-}
-
-///
-/// ObjectTag is a part of ObjectKey that is specific to the type of
-/// the stored object.
-///
-/// NB: the order of the enum values is significant!  In particular,
-/// rocksdb_storage.rs assumes that TimelineMetadataTag is first
-///
-/// Buffer is the kind of object that is accessible by the public
-/// get_page_at_lsn() / put_page_image() / put_wal_record() functions in
-/// the repository.rs interface. The rest are internal objects stored in
-/// the key-value store, to store various metadata. They're not directly
-/// accessible outside object_repository.rs
-///
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
-pub enum ObjectTag {
-    // dummy tag preceeding all other keys
-    FirstTag,
-
-    // Metadata about a timeline. Not versioned.
-    TimelineMetadataTag,
-
-    // These objects store metadata about one relish. Currently it's used
-    // just to track the relish's size. It's not used for non-blocky relishes
-    // at all.
-    RelationMetadata(RelishTag),
-
-    // These are the pages exposed in the public Repository/Timeline interface.
-    Buffer(RelishTag, u32),
-}

pageserver/src/object_store.rs

@@ -1,92 +0,0 @@
-//! Low-level key-value storage abstraction.
-//!
-use crate::object_key::*;
-use crate::relish::*;
-use anyhow::Result;
-use std::collections::HashSet;
-use std::iter::Iterator;
-use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::ZTimelineId;
-
-///
-/// Low-level storage abstraction.
-///
-/// All the data in the repository is stored in a key-value store. This trait
-/// abstracts the details of the key-value store.
-///
-/// A simple key-value store would support just GET and PUT operations with
-/// a key, but the upper layer needs slightly complicated read operations
-///
-/// The most frequently used function is 'object_versions'. It is used
-/// to look up a page version. It is LSN aware, in that the caller
-/// specifies an LSN, and the function returns all values for that
-/// block with the same or older LSN.
-///
-pub trait ObjectStore: Send + Sync {
-    ///
-    /// Store a value with given key.
-    ///
-    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()>;
-
-    /// Read entry with the exact given key.
-    ///
-    /// This is used for retrieving metadata with special key that doesn't
-    /// correspond to any real relation.
-    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>>;
-
-    /// Read key greater or equal than specified
-    fn get_next_key(&self, key: &ObjectKey) -> Result<Option<ObjectKey>>;
-
-    /// Iterate through all page versions of one object.
-    ///
-    /// Returns all page versions in descending LSN order, along with the LSN
-    /// of each page version.
-    fn object_versions<'a>(
-        &'a self,
-        key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>>;
-
-    /// Iterate through versions of all objects in a timeline.
-    ///
-    /// Returns objects in increasing key-version order.
-    /// Returns all versions up to and including the specified LSN.
-    fn objects<'a>(
-        &'a self,
-        timeline: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = Result<(ObjectTag, Lsn, Vec<u8>)>> + 'a>>;
-
-    /// Iterate through all keys with given tablespace and database ID, and LSN <= 'lsn'.
-    /// Both dbnode and spcnode can be InvalidId (0) which means get all relations in tablespace/cluster
-    ///
-    /// This is used to implement 'create database'
-    fn list_rels(
-        &self,
-        timelineid: ZTimelineId,
-        spcnode: u32,
-        dbnode: u32,
-        lsn: Lsn,
-    ) -> Result<HashSet<RelTag>>;
-
-    /// Iterate through non-rel relishes
-    ///
-    /// This is used to prepare tarball for new node startup.
-    fn list_nonrels<'a>(&'a self, timelineid: ZTimelineId, lsn: Lsn) -> Result<HashSet<RelishTag>>;
-
-    /// Iterate through objects tags. If nonrel_only, then only non-relationa data is iterated.
-    ///
-    /// This is used to implement GC and preparing tarball for new node startup
-    /// Returns objects in increasing key-version order.
-    fn list_objects<'a>(
-        &'a self,
-        timelineid: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = ObjectTag> + 'a>>;
-
-    /// Unlink object (used by GC). This mehod may actually delete object or just mark it for deletion.
-    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()>;
-
-    // Compact storage and remove versions marged for deletion
-    fn compact(&self);
-}

pageserver/src/page_cache.rs

@@ -1,91 +1,793 @@
-//! This module acts as a switchboard to access different repositories managed by this
-//! page server.
+//
+// Page Cache holds all the different page versions and WAL records
+//
+// The Page Cache is a BTreeMap, keyed by the RelFileNode an blocknumber, and the LSN.
+// The BTreeMap is protected by a Mutex, and each cache entry is protected by another
+// per-entry mutex.
+//
 
-use crate::branches;
-use crate::layered_repository::LayeredRepository;
-use crate::object_repository::ObjectRepository;
-use crate::repository::Repository;
-use crate::rocksdb_storage::RocksObjectStore;
-use crate::walredo::PostgresRedoManager;
-use crate::{PageServerConf, RepositoryFormat};
-use anyhow::{anyhow, bail, Result};
+use core::ops::Bound::Included;
+use std::collections::{BTreeMap, HashMap};
+use std::{convert::TryInto, ops::AddAssign};
+
+use std::error::Error;
+use std::sync::atomic::AtomicU64;
+use std::sync::atomic::Ordering;
+use std::sync::{Arc, Condvar, Mutex};
+use std::thread;
+use std::time::Duration;
+// use tokio::sync::RwLock;
+use bytes::Bytes;
 use lazy_static::lazy_static;
-use log::info;
-use std::collections::HashMap;
-use std::fs;
-use std::str::FromStr;
-use std::sync::{Arc, Mutex};
-use zenith_utils::zid::ZTenantId;
+use log::*;
+use rand::Rng;
+
+use crate::{controlfile, walredo, PageServerConf};
+
+use crossbeam_channel::unbounded;
+use crossbeam_channel::{Receiver, Sender};
+
+// Timeout when waiting or WAL receiver to catch up to an LSN given in a GetPage@LSN call.
+static TIMEOUT: Duration = Duration::from_secs(60);
+
+pub struct PageCache {
+    shared: Mutex<PageCacheShared>,
+
+    // Channel for communicating with the WAL redo process here.
+    pub walredo_sender: Sender<Arc<CacheEntry>>,
+    pub walredo_receiver: Receiver<Arc<CacheEntry>>,
+
+    valid_lsn_condvar: Condvar,
+
+    // Counters, for metrics collection.
+    pub num_entries: AtomicU64,
+    pub num_page_images: AtomicU64,
+    pub num_wal_records: AtomicU64,
+    pub num_getpage_requests: AtomicU64,
+
+    // copies of shared.first/last_valid_lsn fields (copied here so
+    // that they can be read without acquiring the mutex).
+    pub first_valid_lsn: AtomicU64,
+    pub last_valid_lsn: AtomicU64,
+    pub last_record_lsn: AtomicU64,
+}
+
+#[derive(Clone)]
+pub struct PageCacheStats {
+    pub num_entries: u64,
+    pub num_page_images: u64,
+    pub num_wal_records: u64,
+    pub num_getpage_requests: u64,
+    pub first_valid_lsn: u64,
+    pub last_valid_lsn: u64,
+    pub last_record_lsn: u64,
+}
+
+impl AddAssign for PageCacheStats {
+    fn add_assign(&mut self, other: Self) {
+        *self = Self {
+            num_entries: self.num_entries + other.num_entries,
+            num_page_images: self.num_page_images + other.num_page_images,
+            num_wal_records: self.num_wal_records + other.num_wal_records,
+            num_getpage_requests: self.num_getpage_requests + other.num_getpage_requests,
+            first_valid_lsn: self.first_valid_lsn + other.first_valid_lsn,
+            last_valid_lsn: self.last_valid_lsn + other.last_valid_lsn,
+            last_record_lsn: self.last_record_lsn + other.last_record_lsn,
+        }
+    }
+}
+
+//
+// Shared data structure, holding page cache and related auxiliary information
+//
+struct PageCacheShared {
+    // The actual page cache
+    pagecache: BTreeMap<CacheKey, Arc<CacheEntry>>,
+
+    // Relation n_blocks cache
+    //
+    // This hashtable should be updated together with the pagecache. Now it is
+    // accessed unreasonably often through the smgr_nblocks(). It is better to just
+    // cache it in postgres smgr and ask only on restart.
+    relsize_cache: HashMap<RelTag, u32>,
+
+    // What page versions do we hold in the cache? If we get GetPage with
+    // LSN < first_valid_lsn, that's an error because we (no longer) hold that
+    // page version. If we get a request > last_valid_lsn, we need to wait until
+    // we receive all the WAL up to the request.
+    //
+    // last_record_lsn points to the end of last processed WAL record.
+    // It can lag behind last_valid_lsn, if the WAL receiver has received some WAL
+    // after the end of last record, but not the whole next record yet. In the
+    // page cache, we care about last_valid_lsn, but if the WAL receiver needs to
+    // restart the streaming, it needs to restart at the end of last record, so
+    // we track them separately. last_record_lsn should perhaps be in
+    // walreceiver.rs instead of here, but it seems convenient to keep all three
+    // values together.
+    //
+    first_valid_lsn: u64,
+    last_valid_lsn: u64,
+    last_record_lsn: u64,
+
+    controldata: controlfile::ControlFileDataZenith,
+}
 
 lazy_static! {
-    pub static ref REPOSITORY: Mutex<HashMap<ZTenantId, Arc<dyn Repository>>> =
-        Mutex::new(HashMap::new());
+    pub static ref PAGECACHES: Mutex<HashMap<u64, Arc<PageCache>>> = Mutex::new(HashMap::new());
 }
 
-pub fn init(conf: &'static PageServerConf) {
-    let mut m = REPOSITORY.lock().unwrap();
+pub fn get_pagecache(conf: PageServerConf, sys_id: u64) -> Arc<PageCache> {
+    let mut pcaches = PAGECACHES.lock().unwrap();
 
-    for dir_entry in fs::read_dir(conf.tenants_path()).unwrap() {
-        let tenantid =
-            ZTenantId::from_str(dir_entry.unwrap().file_name().to_str().unwrap()).unwrap();
+    if !pcaches.contains_key(&sys_id) {
+        pcaches.insert(sys_id, Arc::new(init_page_cache()));
 
-        // Set up a WAL redo manager, for applying WAL records.
-        let walredo_mgr = PostgresRedoManager::new(conf, tenantid);
+        // Initialize the WAL redo thread
+        //
+        // Now join_handle is not saved any where and we won'try restart tharead
+        // if it is dead. We may later stop that treads after some inactivity period
+        // and restart them on demand.
+        let _walredo_thread = thread::Builder::new()
+            .name("WAL redo thread".into())
+            .spawn(move || {
+                walredo::wal_redo_main(conf, sys_id);
+            })
+            .unwrap();
+    }
 
-        // Set up an object repository, for actual data storage.
-        let repo: Arc<dyn Repository + Sync + Send> = match conf.repository_format {
-            RepositoryFormat::Layered => {
-                let repo = Arc::new(LayeredRepository::new(
-                    conf,
-                    Arc::new(walredo_mgr),
-                    tenantid,
-                ));
-                LayeredRepository::launch_checkpointer_thread(conf, repo.clone());
-                repo
+    pcaches.get(&sys_id).unwrap().clone()
+}
+
+fn init_page_cache() -> PageCache {
+    // Initialize the channel between the page cache and the WAL applicator
+    let (s, r) = unbounded();
+
+    PageCache {
+        shared: Mutex::new(PageCacheShared {
+            pagecache: BTreeMap::new(),
+            relsize_cache: HashMap::new(),
+            first_valid_lsn: 0,
+            last_valid_lsn: 0,
+            last_record_lsn: 0,
+            controldata: controlfile::ControlFileDataZenith::new(),
+        }),
+        valid_lsn_condvar: Condvar::new(),
+
+        walredo_sender: s,
+        walredo_receiver: r,
+
+        num_entries: AtomicU64::new(0),
+        num_page_images: AtomicU64::new(0),
+        num_wal_records: AtomicU64::new(0),
+        num_getpage_requests: AtomicU64::new(0),
+
+        first_valid_lsn: AtomicU64::new(0),
+        last_valid_lsn: AtomicU64::new(0),
+        last_record_lsn: AtomicU64::new(0),
+    }
+}
+
+//
+// We store two kinds of entries in the page cache:
+//
+// 1. Ready-made images of the block
+// 2. WAL records, to be applied on top of the "previous" entry
+//
+// Some WAL records will initialize the page from scratch. For such records,
+// the 'will_init' flag is set. They don't need the previous page image before
+// applying. The 'will_init' flag is set for records containing a full-page image,
+// and for records with the BKPBLOCK_WILL_INIT flag. These differ from PageImages
+// stored directly in the cache entry in that you still need to run the WAL redo
+// routine to generate the page image.
+//
+#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug)]
+pub struct CacheKey {
+    pub tag: BufferTag,
+    pub lsn: u64,
+}
+
+pub struct CacheEntry {
+    pub key: CacheKey,
+
+    pub content: Mutex<CacheEntryContent>,
+
+    // Condition variable used by the WAL redo service, to wake up
+    // requester.
+    //
+    // FIXME: this takes quite a lot of space. Consider using parking_lot::Condvar
+    // or something else.
+    pub walredo_condvar: Condvar,
+}
+
+pub struct CacheEntryContent {
+    pub page_image: Option<Bytes>,
+    pub wal_record: Option<WALRecord>,
+    pub apply_pending: bool,
+}
+
+impl CacheEntry {
+    fn new(key: CacheKey) -> CacheEntry {
+        CacheEntry {
+            key: key,
+            content: Mutex::new(CacheEntryContent {
+                page_image: None,
+                wal_record: None,
+                apply_pending: false,
+            }),
+            walredo_condvar: Condvar::new(),
+        }
+    }
+}
+
+#[derive(Eq, PartialEq, Hash, Clone, Copy, Debug)]
+pub struct RelTag {
+    pub spcnode: u32,
+    pub dbnode: u32,
+    pub relnode: u32,
+    pub forknum: u8,
+}
+
+#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug)]
+pub struct BufferTag {
+    pub spcnode: u32,
+    pub dbnode: u32,
+    pub relnode: u32,
+    pub forknum: u8,
+    pub blknum: u32,
+}
+
+#[derive(Clone)]
+pub struct WALRecord {
+    pub lsn: u64, // LSN at the *end* of the record
+    pub will_init: bool,
+    pub rec: Bytes,
+}
+
+// Public interface functions
+
+impl PageCache {
+    pub fn get_nonrel_page(&self, tag: BufferTag, _reqlsn: u64) -> Result<Bytes, Box<dyn Error>> {
+        self.num_getpage_requests.fetch_add(1, Ordering::Relaxed);
+
+        // Now we don't have versioning for non-rel pages.
+        // Also at bootstrap we don't know lsn for some files.
+        // So always request the very latest version
+        // let lsn = reqlsn;
+
+        let lsn = u64::MAX;
+
+        let minkey = CacheKey { tag: tag, lsn: 0 };
+        // Look up to the largest lsn
+        let maxkey = CacheKey { tag: tag, lsn: lsn };
+
+        let entry_rc: Arc<CacheEntry>;
+        {
+            let shared = self.shared.lock().unwrap();
+
+            let pagecache = &shared.pagecache;
+            info!("got pagecache {}", pagecache.len());
+
+            let mut entries = pagecache.range((Included(&minkey), Included(&maxkey)));
+
+            let entry_opt = entries.next_back();
+
+            if entry_opt.is_none() {
+                return Err(format!(
+                    "not found non-rel page with LSN {} for {}/{}/{}.{} blk {}",
+                    lsn, tag.spcnode, tag.dbnode, tag.relnode, tag.forknum, tag.blknum
+                ))?;
             }
-            RepositoryFormat::RocksDb => {
-                let obj_store = RocksObjectStore::open(conf, &tenantid).unwrap();
 
-                Arc::new(ObjectRepository::new(
-                    conf,
-                    Arc::new(obj_store),
-                    Arc::new(walredo_mgr),
-                    tenantid,
-                ))
+            info!(
+                "found non-rel page with LSN {} for {}/{}/{}.{} blk {}",
+                lsn, tag.spcnode, tag.dbnode, tag.relnode, tag.forknum, tag.blknum
+            );
+
+            let (_key, entry) = entry_opt.unwrap();
+            entry_rc = entry.clone();
+
+            // Now that we have a reference to the cache entry, drop the lock on the map.
+            // It's important to do this before waiting on the condition variable below,
+            // and better to do it as soon as possible to maximize concurrency.
+        }
+
+        // Lock the cache entry and dig the page image out of it.
+        let page_img: Bytes;
+        {
+            let entry_content = entry_rc.content.lock().unwrap();
+
+            if let Some(img) = &entry_content.page_image {
+                assert!(!entry_content.apply_pending);
+                page_img = img.clone();
+            } else if entry_content.wal_record.is_some() {
+                return Err("non-rel WAL redo is not implemented yet".into());
+                //
+                // If this page needs to be reconstructed by applying some WAL,
+                // send a request to the WAL redo thread.
+                //
+                // if !entry_content.apply_pending {
+                //     assert!(!entry_content.apply_pending);
+                //     entry_content.apply_pending = true;
+
+                //     let s = &self.walredo_sender;
+                //     s.send(entry_rc.clone())?;
+                // }
+
+                // while entry_content.apply_pending {
+                //     entry_content = entry_rc.walredo_condvar.wait(entry_content).unwrap();
+                //}
+
+                // We should now have a page image. If we don't, it means that WAL redo
+                // failed to reconstruct it. WAL redo should've logged that error already.
+                // page_img = match &entry_content.page_image {
+                //     Some(p) => p.clone(),
+                //     None => {
+                //         error!("could not apply WAL to reconstruct page image for GetPage@LSN request");
+                //         return Err("could not apply WAL to reconstruct page image".into());
+                //     }
+                // };
+            } else {
+                // No base image, and no WAL record. Huh?
+                return Err(format!("no page image or WAL record for requested page"))?;
             }
+        }
+
+        trace!(
+            "Returning page for {}/{}/{}.{} blk {}",
+            tag.spcnode,
+            tag.dbnode,
+            tag.relnode,
+            tag.forknum,
+            tag.blknum
+        );
+
+        return Ok(page_img);
+    }
+
+    //
+    // GetPage@LSN
+    //
+    // Returns an 8k page image
+    //
+    pub fn get_page_at_lsn(&self, tag: BufferTag, reqlsn: u64) -> Result<Bytes, Box<dyn Error>> {
+        let mut lsn = reqlsn;
+
+        if tag.forknum > 40 {
+            info!(
+                "get_page_at_lsn got request for page with LSN {} for {}/{}/{}.{} blk {}",
+                lsn, tag.spcnode, tag.dbnode, tag.relnode, tag.forknum, tag.blknum
+            );
+
+            return self.get_nonrel_page(tag, lsn);
+        }
+
+        if reqlsn == 0 {
+            let c = self.get_controldata();
+            lsn = c.checkPoint;
+
+            info!("update reqlsn get_page_at_lsn got request for page with LSN {} for {}/{}/{}.{} blk {}", lsn,
+        tag.spcnode, tag.dbnode, tag.relnode, tag.forknum, tag.blknum);
+        }
+
+        self.num_getpage_requests.fetch_add(1, Ordering::Relaxed);
+
+        // Look up cache entry. If it's a page image, return that. If it's a WAL record,
+        // ask the WAL redo service to reconstruct the page image from the WAL records.
+        let minkey = CacheKey { tag: tag, lsn: 0 };
+        let maxkey = CacheKey { tag: tag, lsn: lsn };
+        let entry_rc: Arc<CacheEntry>;
+        {
+            let mut shared = self.shared.lock().unwrap();
+
+            let mut waited = false;
+
+            // When server just started and created checkpoint lsn,
+            // but we have not yet established connection,
+            // requested lsn will be larger than the one we have
+            while lsn > shared.last_valid_lsn + 500 {
+                // TODO: Wait for the WAL receiver to catch up
+                waited = true;
+                trace!(
+                    "not caught up yet: {}, requested {}",
+                    shared.last_valid_lsn,
+                    lsn
+                );
+                let wait_result = self
+                    .valid_lsn_condvar
+                    .wait_timeout(shared, TIMEOUT)
+                    .unwrap();
+
+                shared = wait_result.0;
+                if wait_result.1.timed_out() {
+                    return Err(format!(
+                        "Timed out while waiting for WAL record at LSN {} to arrive",
+                        lsn
+                    ))?;
+                }
+            }
+            if waited {
+                trace!("caught up now, continuing");
+            }
+
+            if lsn < shared.first_valid_lsn {
+                return Err(format!("LSN {} has already been removed", lsn))?;
+            }
+
+            let pagecache = &shared.pagecache;
+
+            let mut entries = pagecache.range((Included(&minkey), Included(&maxkey)));
+
+            let entry_opt = entries.next_back();
+
+            if entry_opt.is_none() {
+                //static ZERO_PAGE:[u8; 8192] = [0 as u8; 8192];
+                //return Ok(Bytes::from_static(&ZERO_PAGE));
+                return Err("could not find page image")?;
+            }
+            let (_key, entry) = entry_opt.unwrap();
+            entry_rc = entry.clone();
+
+            // Now that we have a reference to the cache entry, drop the lock on the map.
+            // It's important to do this before waiting on the condition variable below,
+            // and better to do it as soon as possible to maximize concurrency.
+        }
+
+        // Lock the cache entry and dig the page image out of it.
+        let page_img: Bytes;
+        {
+            let mut entry_content = entry_rc.content.lock().unwrap();
+
+            if let Some(img) = &entry_content.page_image {
+                assert!(!entry_content.apply_pending);
+                page_img = img.clone();
+            } else if entry_content.wal_record.is_some() {
+                //
+                // If this page needs to be reconstructed by applying some WAL,
+                // send a request to the WAL redo thread.
+                //
+                if !entry_content.apply_pending {
+                    assert!(!entry_content.apply_pending);
+                    entry_content.apply_pending = true;
+
+                    let s = &self.walredo_sender;
+                    s.send(entry_rc.clone())?;
+                }
+
+                while entry_content.apply_pending {
+                    entry_content = entry_rc.walredo_condvar.wait(entry_content).unwrap();
+                }
+
+                // We should now have a page image. If we don't, it means that WAL redo
+                // failed to reconstruct it. WAL redo should've logged that error already.
+                page_img = match &entry_content.page_image {
+                    Some(p) => p.clone(),
+                    None => {
+                        error!(
+                            "could not apply WAL to reconstruct page image for GetPage@LSN request"
+                        );
+                        return Err("could not apply WAL to reconstruct page image".into());
+                    }
+                };
+            } else {
+                // No base image, and no WAL record. Huh?
+                return Err(format!("no page image or WAL record for requested page"))?;
+            }
+        }
+
+        // FIXME: assumes little-endian. Only used for the debugging log though
+        let page_lsn_hi = u32::from_le_bytes(page_img.get(0..4).unwrap().try_into().unwrap());
+        let page_lsn_lo = u32::from_le_bytes(page_img.get(4..8).unwrap().try_into().unwrap());
+        trace!(
+            "Returning page with LSN {:X}/{:X} for {}/{}/{}.{} blk {}",
+            page_lsn_hi,
+            page_lsn_lo,
+            tag.spcnode,
+            tag.dbnode,
+            tag.relnode,
+            tag.forknum,
+            tag.blknum
+        );
+
+        return Ok(page_img);
+    }
+
+    //
+    // Collect all the WAL records that are needed to reconstruct a page
+    // image for the given cache entry.
+    //
+    // Returns an old page image (if any), and a vector of WAL records to apply
+    // over it.
+    //
+    pub fn collect_records_for_apply(&self, entry: &CacheEntry) -> (Option<Bytes>, Vec<WALRecord>) {
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let shared = self.shared.lock().unwrap();
+        let pagecache = &shared.pagecache;
+
+        let minkey = CacheKey {
+            tag: entry.key.tag,
+            lsn: 0,
+        };
+        let maxkey = CacheKey {
+            tag: entry.key.tag,
+            lsn: entry.key.lsn,
+        };
+        let entries = pagecache.range((Included(&minkey), Included(&maxkey)));
+
+        // the last entry in the range should be the CacheEntry we were given
+        //let _last_entry = entries.next_back();
+        //assert!(last_entry == entry);
+
+        let mut base_img: Option<Bytes> = None;
+        let mut records: Vec<WALRecord> = Vec::new();
+
+        // Scan backwards, collecting the WAL records, until we hit an
+        // old page image.
+        for (_key, e) in entries.rev() {
+            let e = e.content.lock().unwrap();
+
+            if let Some(img) = &e.page_image {
+                // We have a base image. No need to dig deeper into the list of
+                // records
+                base_img = Some(img.clone());
+                break;
+            } else if let Some(rec) = &e.wal_record {
+                records.push(rec.clone());
+
+                // If this WAL record initializes the page, no need to dig deeper.
+                if rec.will_init {
+                    break;
+                }
+            } else {
+                panic!("no base image and no WAL record on cache entry");
+            }
+        }
+
+        records.reverse();
+        return (base_img, records);
+    }
+
+    //
+    // Adds a WAL record to the page cache
+    //
+    pub fn put_wal_record(&self, tag: BufferTag, rec: WALRecord) {
+        let key = CacheKey {
+            tag: tag,
+            lsn: rec.lsn,
         };
 
-        info!("initialized storage for tenant: {}", &tenantid);
-        m.insert(tenantid, repo);
+        let entry = CacheEntry::new(key.clone());
+        entry.content.lock().unwrap().wal_record = Some(rec);
+
+        let mut shared = self.shared.lock().unwrap();
+
+        let rel_tag = RelTag {
+            spcnode: tag.spcnode,
+            dbnode: tag.dbnode,
+            relnode: tag.relnode,
+            forknum: tag.forknum,
+        };
+        let rel_entry = shared.relsize_cache.entry(rel_tag).or_insert(0);
+        if tag.blknum >= *rel_entry {
+            *rel_entry = tag.blknum + 1;
+        }
+
+        trace!("put_wal_record lsn: {}", key.lsn);
+
+        let oldentry = shared.pagecache.insert(key, Arc::new(entry));
+        self.num_entries.fetch_add(1, Ordering::Relaxed);
+
+        if !oldentry.is_none() {
+            error!("overwriting WAL record in page cache");
+        }
+
+        self.num_wal_records.fetch_add(1, Ordering::Relaxed);
+    }
+
+    //
+    // Memorize a full image of a page version
+    //
+    pub fn put_page_image(&self, tag: BufferTag, lsn: u64, img: Bytes) {
+        let key = CacheKey { tag: tag, lsn: lsn };
+
+        let entry = CacheEntry::new(key.clone());
+        entry.content.lock().unwrap().page_image = Some(img);
+
+        let mut shared = self.shared.lock().unwrap();
+        let pagecache = &mut shared.pagecache;
+
+        let oldentry = pagecache.insert(key, Arc::new(entry));
+        self.num_entries.fetch_add(1, Ordering::Relaxed);
+        assert!(oldentry.is_none());
+
+        debug!(
+            "inserted page image for {}/{}/{}_{} blk {} at {}",
+            tag.spcnode, tag.dbnode, tag.relnode, tag.forknum, tag.blknum, lsn
+        );
+
+        self.num_page_images.fetch_add(1, Ordering::Relaxed);
+    }
+
+    //
+    pub fn advance_last_valid_lsn(&self, lsn: u64) {
+        let mut shared = self.shared.lock().unwrap();
+
+        // Can't move backwards.
+        assert!(lsn >= shared.last_valid_lsn);
+
+        shared.last_valid_lsn = lsn;
+        self.valid_lsn_condvar.notify_all();
+
+        self.last_valid_lsn.store(lsn, Ordering::Relaxed);
+    }
+
+    //
+    // NOTE: this updates last_valid_lsn as well.
+    //
+    pub fn advance_last_record_lsn(&self, lsn: u64) {
+        let mut shared = self.shared.lock().unwrap();
+
+        // Can't move backwards.
+        assert!(lsn >= shared.last_valid_lsn);
+        assert!(lsn >= shared.last_record_lsn);
+
+        shared.last_valid_lsn = lsn;
+        shared.last_record_lsn = lsn;
+        self.valid_lsn_condvar.notify_all();
+
+        self.last_valid_lsn.store(lsn, Ordering::Relaxed);
+        self.last_valid_lsn.store(lsn, Ordering::Relaxed);
+    }
+
+    //
+    pub fn _advance_first_valid_lsn(&self, lsn: u64) {
+        let mut shared = self.shared.lock().unwrap();
+
+        // Can't move backwards.
+        assert!(lsn >= shared.first_valid_lsn);
+
+        // Can't overtake last_valid_lsn (except when we're
+        // initializing the system and last_valid_lsn hasn't been set yet.
+        assert!(shared.last_valid_lsn == 0 || lsn < shared.last_valid_lsn);
+
+        shared.first_valid_lsn = lsn;
+        self.first_valid_lsn.store(lsn, Ordering::Relaxed);
+    }
+
+    pub fn init_valid_lsn(&self, lsn: u64) {
+        let mut shared = self.shared.lock().unwrap();
+
+        assert!(shared.first_valid_lsn == 0);
+        assert!(shared.last_valid_lsn == 0);
+        assert!(shared.last_record_lsn == 0);
+
+        shared.first_valid_lsn = lsn;
+        shared.last_valid_lsn = lsn;
+        shared.last_record_lsn = lsn;
+
+        self.first_valid_lsn.store(lsn, Ordering::Relaxed);
+        self.last_valid_lsn.store(lsn, Ordering::Relaxed);
+        self.last_record_lsn.store(lsn, Ordering::Relaxed);
+    }
+
+    pub fn get_last_valid_lsn(&self) -> u64 {
+        let shared = self.shared.lock().unwrap();
+
+        return shared.last_record_lsn;
+    }
+
+    pub fn set_controldata(&self, c: controlfile::ControlFileDataZenith) {
+        let mut shared = self.shared.lock().unwrap();
+        shared.controldata = c;
+    }
+
+    pub fn get_controldata(&self) -> controlfile::ControlFileDataZenith {
+        let shared = self.shared.lock().unwrap();
+        return shared.controldata.clone();
+    }
+
+    //
+    // Simple test function for the WAL redo code:
+    //
+    // 1. Pick a page from the page cache at random.
+    // 2. Request that page with GetPage@LSN, using Max LSN (i.e. get the latest page version)
+    //
+    //
+    pub fn _test_get_page_at_lsn(&self) {
+        // for quick testing of the get_page_at_lsn() funcion.
+        //
+        // Get a random page from the page cache. Apply all its WAL, by requesting
+        // that page at the highest lsn.
+
+        let mut tag: Option<BufferTag> = None;
+
+        {
+            let shared = self.shared.lock().unwrap();
+            let pagecache = &shared.pagecache;
+
+            if pagecache.is_empty() {
+                info!("page cache is empty");
+                return;
+            }
+
+            // Find nth entry in the map, where n is picked at random
+            let n = rand::thread_rng().gen_range(0..pagecache.len());
+            let mut i = 0;
+            for (key, _e) in pagecache.iter() {
+                if i == n {
+                    tag = Some(key.tag);
+                    break;
+                }
+                i += 1;
+            }
+        }
+
+        info!("testing GetPage@LSN for block {}", tag.unwrap().blknum);
+        match self.get_page_at_lsn(tag.unwrap(), 0xffff_ffff_ffff_eeee) {
+            Ok(_img) => {
+                // This prints out the whole page image.
+                //println!("{:X?}", img);
+            }
+            Err(error) => {
+                error!("GetPage@LSN failed: {}", error);
+            }
+        }
+    }
+
+    // FIXME: Shouldn't relation size also be tracked with an LSN?
+    // If a replica is lagging behind, it needs to get the size as it was on
+    // the replica's current replay LSN.
+    pub fn relsize_inc(&self, rel: &RelTag, to: Option<u32>) {
+        let mut shared = self.shared.lock().unwrap();
+        let entry = shared.relsize_cache.entry(*rel).or_insert(0);
+
+        if let Some(to) = to {
+            if to >= *entry {
+                *entry = to + 1;
+            }
+        }
+        trace!("relsize_inc {:?} to {}", rel, entry);
+    }
+
+    pub fn relsize_get(&self, rel: &RelTag) -> u32 {
+        let mut shared = self.shared.lock().unwrap();
+        let entry = shared.relsize_cache.entry(*rel).or_insert(0);
+        *entry
+    }
+
+    pub fn relsize_exist(&self, rel: &RelTag) -> bool {
+        let shared = self.shared.lock().unwrap();
+        let relsize_cache = &shared.relsize_cache;
+        relsize_cache.contains_key(rel)
+    }
+
+    pub fn get_stats(&self) -> PageCacheStats {
+        PageCacheStats {
+            num_entries: self.num_entries.load(Ordering::Relaxed),
+            num_page_images: self.num_page_images.load(Ordering::Relaxed),
+            num_wal_records: self.num_wal_records.load(Ordering::Relaxed),
+            num_getpage_requests: self.num_getpage_requests.load(Ordering::Relaxed),
+            first_valid_lsn: self.first_valid_lsn.load(Ordering::Relaxed),
+            last_valid_lsn: self.last_valid_lsn.load(Ordering::Relaxed),
+            last_record_lsn: self.last_record_lsn.load(Ordering::Relaxed),
+        }
     }
 }
 
-pub fn create_repository_for_tenant(
-    conf: &'static PageServerConf,
-    tenantid: ZTenantId,
-) -> Result<()> {
-    let mut m = REPOSITORY.lock().unwrap();
+pub fn get_stats() -> PageCacheStats {
+    let pcaches = PAGECACHES.lock().unwrap();
 
-    // First check that the tenant doesn't exist already
-    if m.get(&tenantid).is_some() {
-        bail!("tenant {} already exists", tenantid);
-    }
-    let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenantid));
-    let repo = branches::create_repo(conf, tenantid, wal_redo_manager)?;
+    let mut stats = PageCacheStats {
+        num_entries: 0,
+        num_page_images: 0,
+        num_wal_records: 0,
+        num_getpage_requests: 0,
+        first_valid_lsn: 0,
+        last_valid_lsn: 0,
+        last_record_lsn: 0,
+    };
 
-    m.insert(tenantid, repo);
-
-    Ok(())
-}
-
-pub fn insert_repository_for_tenant(tenantid: ZTenantId, repo: Arc<dyn Repository>) {
-    let o = &mut REPOSITORY.lock().unwrap();
-    o.insert(tenantid, repo);
-}
-
-pub fn get_repository_for_tenant(tenantid: &ZTenantId) -> Result<Arc<dyn Repository>> {
-    let o = &REPOSITORY.lock().unwrap();
-    o.get(tenantid)
-        .map(|repo| Arc::clone(repo))
-        .ok_or_else(|| anyhow!("repository not found for tenant name {}", tenantid))
+    pcaches.iter().for_each(|(_sys_id, pcache)| {
+        stats += pcache.get_stats();
+    });
+    stats
 }

pageserver/src/pg_constants.rs

@@ -0,0 +1,11 @@
+// From pg_tablespace_d.h
+//
+pub const DEFAULTTABLESPACE_OID: u32 = 1663;
+pub const GLOBALTABLESPACE_OID: u32 = 1664;
+//Special values for non-rel files' tags
+//TODO maybe use enum?
+pub const PG_CONTROLFILE_FORKNUM: u32 = 42;
+pub const PG_FILENODEMAP_FORKNUM: u32 = 43;
+pub const PG_XACT_FORKNUM: u32 = 44;
+pub const PG_MXACT_OFFSETS_FORKNUM: u32 = 45;
+pub const PG_MXACT_MEMBERS_FORKNUM: u32 = 46;

pageserver/src/relish.rs

@@ -1,235 +0,0 @@
-//!
-//! Zenith stores PostgreSQL relations, and some other files, in the
-//! repository.  The relations (i.e. tables and indexes) take up most
-//! of the space in a typical installation, while the other files are
-//! small. We call each relation and other file that is stored in the
-//! repository a "relish". It comes from "rel"-ish, as in "kind of a
-//! rel", because it covers relations as well as other things that are
-//! not relations, but are treated similarly for the purposes of the
-//! storage layer.
-//!
-//! This source file contains the definition of the RelishTag struct,
-//! which uniquely identifies a relish.
-//!
-//! Relishes come in two flavors: blocky and non-blocky. Relations and
-//! SLRUs are blocky, that is, they are divided into 8k blocks, and
-//! the repository tracks their size. Other relishes are non-blocky:
-//! the content of the whole relish is stored as one blob. Block
-//! number must be passed as 0 for all operations on a non-blocky
-//! relish. The one "block" that you store in a non-blocky relish can
-//! have arbitrary size, but they are expected to be small, or you
-//! will have performance issues.
-//!
-//! All relishes are versioned by LSN in the repository.
-//!
-
-use serde::{Deserialize, Serialize};
-use std::fmt;
-
-use postgres_ffi::relfile_utils::forknumber_to_name;
-use postgres_ffi::{Oid, TransactionId};
-
-///
-/// RelishTag identifies one relish.
-///
-#[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
-pub enum RelishTag {
-    // Relations correspond to PostgreSQL relation forks. Each
-    // PostgreSQL relation fork is considered a separate relish.
-    Relation(RelTag),
-
-    // SLRUs include pg_clog, pg_multixact/members, and
-    // pg_multixact/offsets. There are other SLRUs in PostgreSQL, but
-    // they don't need to be stored permanently (e.g. pg_subtrans),
-    // or we do not support them in zenith yet (pg_commit_ts).
-    //
-    // These are currently never requested directly by the compute
-    // nodes, although in principle that would be possible. However,
-    // when a new compute node is created, these are included in the
-    // tarball that we send to the compute node to initialize the
-    // PostgreSQL data directory.
-    //
-    // Each SLRU segment in PostgreSQL is considered a separate
-    // relish. For example, pg_clog/0000, pg_clog/0001, and so forth.
-    //
-    // SLRU segments are divided into blocks, like relations.
-    Slru { slru: SlruKind, segno: u32 },
-
-    // Miscellaneous other files that need to be included in the
-    // tarball at compute node creation. These are non-blocky, and are
-    // expected to be small.
-
-    //
-    // FileNodeMap represents PostgreSQL's 'pg_filenode.map'
-    // files. They are needed to map catalog table OIDs to filenode
-    // numbers. Usually the mapping is done by looking up a relation's
-    // 'relfilenode' field in the 'pg_class' system table, but that
-    // doesn't work for 'pg_class' itself and a few other such system
-    // relations. See PostgreSQL relmapper.c for details.
-    //
-    // Each database has a map file for its local mapped catalogs,
-    // and there is a separate map file for shared catalogs.
-    //
-    // These files are always 512 bytes long (although we don't check
-    // or care about that in the page server).
-    //
-    FileNodeMap { spcnode: Oid, dbnode: Oid },
-
-    //
-    // State files for prepared transactions (e.g pg_twophase/1234)
-    //
-    TwoPhase { xid: TransactionId },
-
-    // The control file, stored in global/pg_control
-    ControlFile,
-
-    // Special entry that represents PostgreSQL checkpoint. It doesn't
-    // correspond to to any physical file in PostgreSQL, but we use it
-    // to track fields needed to restore the checkpoint data in the
-    // control file, when a compute node is created.
-    Checkpoint,
-}
-
-impl RelishTag {
-    pub const fn is_blocky(&self) -> bool {
-        match self {
-            // These relishes work with blocks
-            RelishTag::Relation(_) | RelishTag::Slru { slru: _, segno: _ } => true,
-
-            // and these don't
-            RelishTag::FileNodeMap {
-                spcnode: _,
-                dbnode: _,
-            }
-            | RelishTag::TwoPhase { xid: _ }
-            | RelishTag::ControlFile
-            | RelishTag::Checkpoint => false,
-        }
-    }
-
-    // Physical relishes represent files and use
-    // RelationSizeEntry to track existing and dropped files.
-    // They can be both blocky and non-blocky.
-    pub const fn is_physical(&self) -> bool {
-        match self {
-            // These relishes represent physical files
-            RelishTag::Relation(_)
-            | RelishTag::Slru { .. }
-            | RelishTag::FileNodeMap { .. }
-            | RelishTag::TwoPhase { .. } => true,
-
-            // and these don't
-            RelishTag::ControlFile | RelishTag::Checkpoint => false,
-        }
-    }
-
-    // convenience function to check if this relish is a normal relation.
-    pub const fn is_relation(&self) -> bool {
-        if let RelishTag::Relation(_) = self {
-            true
-        } else {
-            false
-        }
-    }
-}
-
-///
-/// Relation data file segment id throughout the Postgres cluster.
-///
-/// Every data file in Postgres is uniquely identified by 4 numbers:
-/// - relation id / node (`relnode`)
-/// - database id (`dbnode`)
-/// - tablespace id (`spcnode`), in short this is a unique id of a separate
-///   directory to store data files.
-/// - forknumber (`forknum`) is used to split different kinds of data of the same relation
-///   between some set of files (`relnode`, `relnode_fsm`, `relnode_vm`).
-///
-/// In native Postgres code `RelFileNode` structure and individual `ForkNumber` value
-/// are used for the same purpose.
-/// [See more related comments here](https:///github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/relfilenode.h#L57).
-///
-#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy, Serialize, Deserialize)]
-pub struct RelTag {
-    pub forknum: u8,
-    pub spcnode: Oid,
-    pub dbnode: Oid,
-    pub relnode: Oid,
-}
-
-/// Display RelTag in the same format that's used in most PostgreSQL debug messages:
-///
-/// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
-///
-impl fmt::Display for RelTag {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        if let Some(forkname) = forknumber_to_name(self.forknum) {
-            write!(
-                f,
-                "{}/{}/{}_{}",
-                self.spcnode, self.dbnode, self.relnode, forkname
-            )
-        } else {
-            write!(f, "{}/{}/{}", self.spcnode, self.dbnode, self.relnode)
-        }
-    }
-}
-
-/// Display RelTag in the same format that's used in most PostgreSQL debug messages:
-///
-/// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
-///
-impl fmt::Display for RelishTag {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            RelishTag::Relation(rel) => rel.fmt(f),
-            RelishTag::Slru { slru, segno } => {
-                // e.g. pg_clog/0001
-                write!(f, "{}/{:04X}", slru.to_str(), segno)
-            }
-            RelishTag::FileNodeMap { spcnode, dbnode } => {
-                write!(f, "relmapper file for spc {} db {}", spcnode, dbnode)
-            }
-            RelishTag::TwoPhase { xid } => {
-                write!(f, "pg_twophase/{:08X}", xid)
-            }
-            RelishTag::ControlFile => {
-                write!(f, "control file")
-            }
-            RelishTag::Checkpoint => {
-                write!(f, "checkpoint")
-            }
-        }
-    }
-}
-
-///
-/// Non-relation transaction status files (clog (a.k.a. pg_xact) and
-/// pg_multixact) in Postgres are handled by SLRU (Simple LRU) buffer,
-/// hence the name.
-///
-/// These files are global for a postgres instance.
-///
-/// These files are divided into segments, which are divided into
-/// pages of the same BLCKSZ as used for relation files.
-///
-#[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
-pub enum SlruKind {
-    Clog,
-    MultiXactMembers,
-    MultiXactOffsets,
-}
-
-impl SlruKind {
-    pub fn to_str(&self) -> &'static str {
-        match self {
-            Self::Clog => "pg_xact",
-            Self::MultiXactMembers => "pg_multixact/members",
-            Self::MultiXactOffsets => "pg_multixact/offsets",
-        }
-    }
-}
-
-pub const FIRST_NONREL_RELISH_TAG: RelishTag = RelishTag::Slru {
-    slru: SlruKind::Clog,
-    segno: 0,
-};

pageserver/src/repository.rs

@@ -1,736 +0,0 @@
-use crate::object_key::*;
-use crate::relish::*;
-use anyhow::Result;
-use bytes::{Buf, BufMut, Bytes, BytesMut};
-use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
-use std::iter::Iterator;
-use std::ops::AddAssign;
-use std::sync::Arc;
-use std::time::Duration;
-use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::ZTimelineId;
-
-///
-/// A repository corresponds to one .zenith directory. One repository holds multiple
-/// timelines, forked off from the same initial call to 'initdb'.
-pub trait Repository: Send + Sync {
-    /// Get Timeline handle for given zenith timeline ID.
-    fn get_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>>;
-
-    /// Create a new, empty timeline. The caller is responsible for loading data into it
-    fn create_empty_timeline(
-        &self,
-        timelineid: ZTimelineId,
-        start_lsn: Lsn,
-    ) -> Result<Arc<dyn Timeline>>;
-
-    /// Branch a timeline
-    fn branch_timeline(&self, src: ZTimelineId, dst: ZTimelineId, start_lsn: Lsn) -> Result<()>;
-
-    /// perform one garbage collection iteration.
-    /// garbage collection is periodically performed by gc thread,
-    /// but it can be explicitly requested through page server api.
-    ///
-    /// 'timelineid' specifies the timeline to GC, or None for all.
-    /// `horizon` specifies delta from last lsn to preserve all object versions (pitr interval).
-    /// `compact` parameter is used to force compaction of storage.
-    /// some storage implementation are based on lsm tree and require periodic merge (compaction).
-    /// usually storage implementation determines itself when compaction should be performed.
-    /// but for gc tests it way be useful to force compaction just after completion of gc iteration
-    /// to make sure that all detected garbage is removed.
-    /// so right now `compact` is set to true when gc explicitly requested through page srver api,
-    /// and is st to false in gc threads which infinitely repeats gc iterations in loop.
-    fn gc_iteration(
-        &self,
-        timelineid: Option<ZTimelineId>,
-        horizon: u64,
-        compact: bool,
-    ) -> Result<GcResult>;
-
-    // TODO get timelines?
-    //fn get_stats(&self) -> RepositoryStats;
-}
-
-///
-/// Result of performing GC
-///
-#[derive(Default)]
-pub struct GcResult {
-    // FIXME: These counters make sense for the ObjectRepository. They are not used
-    // by the LayeredRepository.
-    pub n_relations: u64,
-    pub inspected: u64,
-    pub truncated: u64,
-    pub deleted: u64,
-    pub prep_deleted: u64,        // RelishTag::Twophase
-    pub slru_deleted: u64,        // RelishTag::Slru
-    pub chkp_deleted: u64,        // RelishTag::Checkpoint
-    pub control_deleted: u64,     // RelishTag::ControlFile
-    pub filenodemap_deleted: u64, // RelishTag::FileNodeMap
-    pub dropped: u64,
-
-    // These are used for the LayeredRepository instead
-    pub snapshot_relfiles_total: u64,
-    pub snapshot_relfiles_needed_by_cutoff: u64,
-    pub snapshot_relfiles_needed_by_branches: u64,
-    pub snapshot_relfiles_not_updated: u64,
-    pub snapshot_relfiles_removed: u64, // # of snapshot files removed because they have been made obsolete by newer snapshot files.
-    pub snapshot_relfiles_dropped: u64, // # of snapshot files removed because the relation was dropped
-
-    pub snapshot_nonrelfiles_total: u64,
-    pub snapshot_nonrelfiles_needed_by_cutoff: u64,
-    pub snapshot_nonrelfiles_needed_by_branches: u64,
-    pub snapshot_nonrelfiles_not_updated: u64,
-    pub snapshot_nonrelfiles_removed: u64, // # of snapshot files removed because they have been made obsolete by newer snapshot files.
-    pub snapshot_nonrelfiles_dropped: u64, // # of snapshot files removed because the relation was dropped
-
-    pub elapsed: Duration,
-}
-
-impl AddAssign for GcResult {
-    fn add_assign(&mut self, other: Self) {
-        self.n_relations += other.n_relations;
-        self.truncated += other.truncated;
-        self.deleted += other.deleted;
-        self.dropped += other.dropped;
-
-        self.snapshot_relfiles_total += other.snapshot_relfiles_total;
-        self.snapshot_relfiles_needed_by_cutoff += other.snapshot_relfiles_needed_by_cutoff;
-        self.snapshot_relfiles_needed_by_branches += other.snapshot_relfiles_needed_by_branches;
-        self.snapshot_relfiles_not_updated += other.snapshot_relfiles_not_updated;
-        self.snapshot_relfiles_removed += other.snapshot_relfiles_removed;
-        self.snapshot_relfiles_dropped += other.snapshot_relfiles_dropped;
-
-        self.snapshot_nonrelfiles_total += other.snapshot_nonrelfiles_total;
-        self.snapshot_nonrelfiles_needed_by_cutoff += other.snapshot_nonrelfiles_needed_by_cutoff;
-        self.snapshot_nonrelfiles_needed_by_branches +=
-            other.snapshot_nonrelfiles_needed_by_branches;
-        self.snapshot_nonrelfiles_not_updated += other.snapshot_nonrelfiles_not_updated;
-        self.snapshot_nonrelfiles_removed += other.snapshot_nonrelfiles_removed;
-        self.snapshot_nonrelfiles_dropped += other.snapshot_nonrelfiles_dropped;
-
-        self.elapsed += other.elapsed;
-    }
-}
-
-pub trait Timeline: Send + Sync {
-    //------------------------------------------------------------------------------
-    // Public GET functions
-    //------------------------------------------------------------------------------
-
-    /// Look up given page in the cache.
-    fn get_page_at_lsn(&self, tag: RelishTag, blknum: u32, lsn: Lsn) -> Result<Bytes>;
-
-    /// Look up given page in the cache.
-    fn get_page_at_lsn_nowait(&self, tag: RelishTag, blknum: u32, lsn: Lsn) -> Result<Bytes>;
-
-    /// Get size of a relish
-    fn get_relish_size(&self, tag: RelishTag, lsn: Lsn) -> Result<Option<u32>>;
-
-    /// Does relation exist?
-    fn get_rel_exists(&self, tag: RelishTag, lsn: Lsn) -> Result<bool>;
-
-    /// Get a list of all distinct relations in given tablespace and database.
-    fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashSet<RelTag>>;
-
-    /// Get a list of non-relational objects
-    fn list_nonrels<'a>(&'a self, lsn: Lsn) -> Result<HashSet<RelishTag>>;
-
-    //------------------------------------------------------------------------------
-    // Public PUT functions, to update the repository with new page versions.
-    //
-    // These are called by the WAL receiver to digest WAL records.
-    //------------------------------------------------------------------------------
-
-    /// Put a new page version that can be constructed from a WAL record
-    ///
-    /// This will implicitly extend the relation, if the page is beyond the
-    /// current end-of-file.
-    fn put_wal_record(&self, tag: RelishTag, blknum: u32, rec: WALRecord) -> Result<()>;
-
-    /// Like put_wal_record, but with ready-made image of the page.
-    fn put_page_image(
-        &self,
-        tag: RelishTag,
-        blknum: u32,
-        lsn: Lsn,
-        img: Bytes,
-        update_meta: bool,
-    ) -> Result<()>;
-
-    /// Truncate relation
-    fn put_truncation(&self, rel: RelishTag, lsn: Lsn, nblocks: u32) -> Result<()>;
-
-    /// Unlink relish.
-    /// This method is used for marking dropped relations and truncated SLRU segments
-    fn put_unlink(&self, tag: RelishTag, lsn: Lsn) -> Result<()>;
-
-    /// Put raw data
-    fn put_raw_data(&self, tag: ObjectTag, lsn: Lsn, data: &[u8]) -> Result<()>;
-
-    /// Remember the all WAL before the given LSN has been processed.
-    ///
-    /// The WAL receiver calls this after the put_* functions, to indicate that
-    /// all WAL before this point has been digested. Before that, if you call
-    /// GET on an earlier LSN, it will block.
-    fn advance_last_valid_lsn(&self, lsn: Lsn);
-    fn get_last_valid_lsn(&self) -> Lsn;
-    fn init_valid_lsn(&self, lsn: Lsn);
-
-    /// Like `advance_last_valid_lsn`, but this always points to the end of
-    /// a WAL record, not in the middle of one.
-    ///
-    /// This must be <= last valid LSN. This is tracked separately from last
-    /// valid LSN, so that the WAL receiver knows where to restart streaming.
-    fn advance_last_record_lsn(&self, lsn: Lsn);
-    fn get_last_record_lsn(&self) -> Lsn;
-
-    // Like `advance_last_record_lsn`, but points to the start position of last record
-    fn get_prev_record_lsn(&self) -> Lsn;
-
-    ///
-    /// Flush to disk all data that was written with the put_* functions
-    ///
-    /// NOTE: This has nothing to do with checkpoint in PostgreSQL. We don't
-    /// know anything about them here in the repository.
-    fn checkpoint(&self) -> Result<()>;
-
-    /// Events for all relations in the timeline.
-    /// Contains updates from start up to the last valid LSN
-    /// at time of history() call. This lsn can be read via the lsn() function.
-    ///
-    /// Relation size is increased implicitly and decreased with Truncate updates.
-    // TODO ordering guarantee?
-    fn history<'a>(&'a self) -> Result<Box<dyn History + 'a>>;
-
-    //
-    // Wait until WAL has been received up to the given LSN.
-    //
-    fn wait_lsn(&self, lsn: Lsn) -> Result<Lsn>;
-}
-
-pub trait History: Iterator<Item = Result<Modification>> {
-    /// The last_valid_lsn at the time of history() call.
-    fn lsn(&self) -> Lsn;
-}
-
-//
-// Structure representing any update operation of object storage.
-// It is used to copy object storage content in PUSH method.
-//
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Modification {
-    pub tag: ObjectTag,
-    pub lsn: Lsn,
-    pub data: Vec<u8>,
-}
-
-impl Modification {
-    pub fn new(entry: (ObjectTag, Lsn, Vec<u8>)) -> Modification {
-        Modification {
-            tag: entry.0,
-            lsn: entry.1,
-            data: entry.2,
-        }
-    }
-}
-
-#[derive(Clone)]
-pub struct RepositoryStats {
-    pub num_entries: Lsn,
-    pub num_page_images: Lsn,
-    pub num_wal_records: Lsn,
-    pub num_getpage_requests: Lsn,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub struct WALRecord {
-    pub lsn: Lsn, // LSN at the *end* of the record
-    pub will_init: bool,
-    pub rec: Bytes,
-    // Remember the offset of main_data in rec,
-    // so that we don't have to parse the record again.
-    // If record has no main_data, this offset equals rec.len().
-    pub main_data_offset: u32,
-}
-
-impl WALRecord {
-    pub fn pack(&self, buf: &mut BytesMut) {
-        buf.put_u64(self.lsn.0);
-        buf.put_u8(self.will_init as u8);
-        buf.put_u32(self.main_data_offset);
-        buf.put_u32(self.rec.len() as u32);
-        buf.put_slice(&self.rec[..]);
-    }
-    pub fn unpack(buf: &mut Bytes) -> WALRecord {
-        let lsn = Lsn::from(buf.get_u64());
-        let will_init = buf.get_u8() != 0;
-        let main_data_offset = buf.get_u32();
-        let mut dst = vec![0u8; buf.get_u32() as usize];
-        buf.copy_to_slice(&mut dst);
-        WALRecord {
-            lsn,
-            will_init,
-            rec: Bytes::from(dst),
-            main_data_offset,
-        }
-    }
-}
-
-///
-/// Tests that should work the same with any Repository/Timeline implementation.
-///
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::layered_repository::LayeredRepository;
-    use crate::object_repository::ObjectRepository;
-    use crate::object_repository::{ObjectValue, PageEntry, RelationSizeEntry};
-    use crate::rocksdb_storage::RocksObjectStore;
-    use crate::walredo::{WalRedoError, WalRedoManager};
-    use crate::{PageServerConf, RepositoryFormat};
-    use postgres_ffi::pg_constants;
-    use std::fs;
-    use std::path::PathBuf;
-    use std::str::FromStr;
-    use std::time::Duration;
-    use zenith_utils::bin_ser::BeSer;
-    use zenith_utils::postgres_backend::AuthType;
-    use zenith_utils::zid::ZTenantId;
-
-    /// Arbitrary relation tag, for testing.
-    const TESTREL_A: RelishTag = RelishTag::Relation(RelTag {
-        spcnode: 0,
-        dbnode: 111,
-        relnode: 1000,
-        forknum: 0,
-    });
-    const TESTREL_B: RelishTag = RelishTag::Relation(RelTag {
-        spcnode: 0,
-        dbnode: 111,
-        relnode: 1001,
-        forknum: 0,
-    });
-
-    /// Convenience function to create a page image with given string as the only content
-    #[allow(non_snake_case)]
-    fn TEST_IMG(s: &str) -> Bytes {
-        let mut buf = BytesMut::new();
-        buf.extend_from_slice(s.as_bytes());
-        buf.resize(8192, 0);
-
-        buf.freeze()
-    }
-
-    static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
-
-    fn get_test_repo(
-        test_name: &str,
-        repository_format: RepositoryFormat,
-    ) -> Result<Box<dyn Repository>> {
-        let repo_dir = PathBuf::from(format!("../tmp_check/test_{}", test_name));
-        let _ = fs::remove_dir_all(&repo_dir);
-        fs::create_dir_all(&repo_dir)?;
-        fs::create_dir_all(&repo_dir.join("timelines"))?;
-
-        let conf = PageServerConf {
-            daemonize: false,
-            gc_horizon: 64 * 1024 * 1024,
-            gc_period: Duration::from_secs(10),
-            listen_addr: "127.0.0.1:5430".to_string(),
-            http_endpoint_addr: "127.0.0.1:9898".to_string(),
-            superuser: "zenith_admin".to_string(),
-            workdir: repo_dir,
-            pg_distrib_dir: "".into(),
-            auth_type: AuthType::Trust,
-            auth_validation_public_key_path: None,
-            repository_format,
-        };
-        // Make a static copy of the config. This can never be free'd, but that's
-        // OK in a test.
-        let conf: &'static PageServerConf = Box::leak(Box::new(conf));
-        let tenantid = ZTenantId::generate();
-        fs::create_dir_all(conf.tenant_path(&tenantid)).unwrap();
-
-        let walredo_mgr = TestRedoManager {};
-
-        let repo: Box<dyn Repository + Sync + Send> = match conf.repository_format {
-            RepositoryFormat::Layered => Box::new(LayeredRepository::new(
-                conf,
-                Arc::new(walredo_mgr),
-                tenantid,
-            )),
-            RepositoryFormat::RocksDb => {
-                let obj_store = RocksObjectStore::create(conf, &tenantid)?;
-
-                Box::new(ObjectRepository::new(
-                    conf,
-                    Arc::new(obj_store),
-                    Arc::new(walredo_mgr),
-                    tenantid,
-                ))
-            }
-        };
-
-        Ok(repo)
-    }
-
-    /// Test get_relsize() and truncation.
-    #[test]
-    fn test_relsize_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_relsize_rocksdb", RepositoryFormat::RocksDb)?;
-        test_relsize(&*repo)
-    }
-
-    #[test]
-    fn test_relsize_layered() -> Result<()> {
-        let repo = get_test_repo("test_relsize_layered", RepositoryFormat::Layered)?;
-        test_relsize(&*repo)
-    }
-
-    fn test_relsize(repo: &dyn Repository) -> Result<()> {
-        // get_timeline() with non-existent timeline id should fail
-        //repo.get_timeline("11223344556677881122334455667788");
-
-        // Create timeline to work on
-        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
-
-        tline.init_valid_lsn(Lsn(1));
-        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"), true)?;
-        tline.put_page_image(TESTREL_A, 1, Lsn(4), TEST_IMG("foo blk 1 at 4"), true)?;
-        tline.put_page_image(TESTREL_A, 2, Lsn(5), TEST_IMG("foo blk 2 at 5"), true)?;
-
-        tline.advance_last_valid_lsn(Lsn(5));
-
-        // The relation was created at LSN 2, not visible at LSN 1 yet.
-        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(1))?, false);
-        assert!(tline.get_relish_size(TESTREL_A, Lsn(1))?.is_none());
-
-        assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(2))?, true);
-        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(2))?.unwrap(), 1);
-        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(5))?.unwrap(), 3);
-
-        // Check page contents at each LSN
-        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(2))?,
-            TEST_IMG("foo blk 0 at 2")
-        );
-
-        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(3))?,
-            TEST_IMG("foo blk 0 at 3")
-        );
-
-        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(4))?,
-            TEST_IMG("foo blk 0 at 3")
-        );
-        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(4))?,
-            TEST_IMG("foo blk 1 at 4")
-        );
-
-        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(5))?,
-            TEST_IMG("foo blk 0 at 3")
-        );
-        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(5))?,
-            TEST_IMG("foo blk 1 at 4")
-        );
-        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 2, Lsn(5))?,
-            TEST_IMG("foo blk 2 at 5")
-        );
-
-        // Truncate last block
-        tline.put_truncation(TESTREL_A, Lsn(6), 2)?;
-        tline.advance_last_valid_lsn(Lsn(6));
-
-        // Check reported size and contents after truncation
-        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(6))?.unwrap(), 2);
-        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(6))?,
-            TEST_IMG("foo blk 0 at 3")
-        );
-        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 1, Lsn(6))?,
-            TEST_IMG("foo blk 1 at 4")
-        );
-
-        // should still see the truncated block with older LSN
-        assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(5))?.unwrap(), 3);
-        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 2, Lsn(5))?,
-            TEST_IMG("foo blk 2 at 5")
-        );
-
-        Ok(())
-    }
-
-    /// Test get_relsize() and truncation with a file larger than 1 GB, so that it's
-    /// split into multiple 1 GB segments in Postgres.
-    ///
-    /// This isn't very interesting with the RocksDb implementation, as we don't pay
-    /// any attention to Postgres segment boundaries there.
-    #[test]
-    fn test_large_rel_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_large_rel_rocksdb", RepositoryFormat::RocksDb)?;
-        test_large_rel(&*repo)
-    }
-
-    #[test]
-    fn test_large_rel_layered() -> Result<()> {
-        let repo = get_test_repo("test_large_rel_layered", RepositoryFormat::Layered)?;
-        test_large_rel(&*repo)
-    }
-
-    fn test_large_rel(repo: &dyn Repository) -> Result<()> {
-        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
-
-        tline.init_valid_lsn(Lsn(1));
-
-        let mut lsn = 1;
-        for blknum in 0..pg_constants::RELSEG_SIZE + 1 {
-            let img = TEST_IMG(&format!("foo blk {} at {}", blknum, Lsn(lsn)));
-            lsn += 1;
-            tline.put_page_image(TESTREL_A, blknum as u32, Lsn(lsn), img, true)?;
-        }
-        tline.advance_last_valid_lsn(Lsn(lsn));
-
-        assert_eq!(
-            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
-            pg_constants::RELSEG_SIZE + 1
-        );
-
-        // Truncate one block
-        lsn += 1;
-        tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE)?;
-        tline.advance_last_valid_lsn(Lsn(lsn));
-        assert_eq!(
-            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
-            pg_constants::RELSEG_SIZE
-        );
-
-        // Truncate another block
-        lsn += 1;
-        tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE - 1)?;
-        tline.advance_last_valid_lsn(Lsn(lsn));
-        assert_eq!(
-            tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
-            pg_constants::RELSEG_SIZE - 1
-        );
-
-        // Truncate to 1500, and then truncate all the way down to 0, one block at a time
-        // This tests the behavior at segment boundaries
-        let mut size: i32 = 3000;
-        while size >= 0 {
-            lsn += 1;
-            tline.put_truncation(TESTREL_A, Lsn(lsn), size as u32)?;
-            tline.advance_last_valid_lsn(Lsn(lsn));
-            assert_eq!(
-                tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
-                size as u32
-            );
-
-            size -= 1;
-        }
-
-        Ok(())
-    }
-
-    fn skip_nonrel_objects<'a>(
-        snapshot: Box<dyn History + 'a>,
-    ) -> Result<impl Iterator<Item = <dyn History as Iterator>::Item> + 'a> {
-        Ok(snapshot.skip_while(|r| match r {
-            Ok(m) => match m.tag {
-                ObjectTag::RelationMetadata(_) => false,
-                _ => true,
-            },
-            _ => panic!("Iteration error"),
-        }))
-    }
-
-    #[test]
-    fn test_branch_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_branch_rocksdb", RepositoryFormat::RocksDb)?;
-        test_branch(&*repo)
-    }
-
-    #[test]
-    fn test_branch_layered() -> Result<()> {
-        let repo = get_test_repo("test_branch_layered", RepositoryFormat::Layered)?;
-        test_branch(&*repo)
-    }
-
-    ///
-    /// Test branch creation
-    ///
-    fn test_branch(repo: &dyn Repository) -> Result<()> {
-        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
-
-        // Import initial dummy checkpoint record, otherwise the get_timeline() call
-        // after branching fails below
-        tline.put_page_image(RelishTag::Checkpoint, 0, Lsn(1), ZERO_PAGE.clone(), false)?;
-
-        // Create a relation on the timeline
-        tline.init_valid_lsn(Lsn(1));
-        tline.put_page_image(TESTREL_A, 0, Lsn(2), TEST_IMG("foo blk 0 at 2"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(3), TEST_IMG("foo blk 0 at 3"), true)?;
-        tline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("foo blk 0 at 4"), true)?;
-
-        // Create another relation
-        tline.put_page_image(TESTREL_B, 0, Lsn(2), TEST_IMG("foobar blk 0 at 2"), true)?;
-
-        tline.advance_last_valid_lsn(Lsn(4));
-
-        // Branch the history, modify relation differently on the new timeline
-        let newtimelineid = ZTimelineId::from_str("AA223344556677881122334455667788").unwrap();
-        repo.branch_timeline(timelineid, newtimelineid, Lsn(3))?;
-        let newtline = repo.get_timeline(newtimelineid)?;
-
-        newtline.put_page_image(TESTREL_A, 0, Lsn(4), TEST_IMG("bar blk 0 at 4"), true)?;
-        newtline.advance_last_valid_lsn(Lsn(4));
-
-        // Check page contents on both branches
-        assert_eq!(
-            tline.get_page_at_lsn(TESTREL_A, 0, Lsn(4))?,
-            TEST_IMG("foo blk 0 at 4")
-        );
-
-        assert_eq!(
-            newtline.get_page_at_lsn(TESTREL_A, 0, Lsn(4))?,
-            TEST_IMG("bar blk 0 at 4")
-        );
-
-        assert_eq!(
-            newtline.get_page_at_lsn(TESTREL_B, 0, Lsn(4))?,
-            TEST_IMG("foobar blk 0 at 2")
-        );
-
-        assert_eq!(newtline.get_relish_size(TESTREL_B, Lsn(4))?.unwrap(), 1);
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_history_rocksdb() -> Result<()> {
-        let repo = get_test_repo("test_history_rocksdb", RepositoryFormat::RocksDb)?;
-        test_history(&*repo)
-    }
-    #[test]
-    // TODO: This doesn't work with the layered storage, the functions needed for push/pull
-    // functionality haven't been implemented yet.
-    #[ignore]
-    fn test_history_layered() -> Result<()> {
-        let repo = get_test_repo("test_history_layered", RepositoryFormat::Layered)?;
-        test_history(&*repo)
-    }
-    fn test_history(repo: &dyn Repository) -> Result<()> {
-        let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
-
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(0));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        // add a page and advance the last valid LSN
-        let rel = TESTREL_A;
-        tline.put_page_image(rel, 1, Lsn(1), TEST_IMG("blk 1 @ lsn 1"), true)?;
-        tline.advance_last_valid_lsn(Lsn(1));
-
-        let expected_page = Modification {
-            tag: ObjectTag::Buffer(rel, 1),
-            lsn: Lsn(1),
-            data: ObjectValue::ser(&ObjectValue::Page(PageEntry::Page(TEST_IMG(
-                "blk 1 @ lsn 1",
-            ))))?,
-        };
-        let expected_init_size = Modification {
-            tag: ObjectTag::RelationMetadata(rel),
-            lsn: Lsn(1),
-            data: ObjectValue::ser(&ObjectValue::RelationSize(RelationSizeEntry::Size(2)))?,
-        };
-        let expected_trunc_size = Modification {
-            tag: ObjectTag::RelationMetadata(rel),
-            lsn: Lsn(2),
-            data: ObjectValue::ser(&ObjectValue::RelationSize(RelationSizeEntry::Size(0)))?,
-        };
-
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(1));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(
-            Some(&expected_init_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        // truncate to zero, but don't advance the last valid LSN
-        tline.put_truncation(rel, Lsn(2), 0)?;
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(1));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(
-            Some(&expected_init_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        // advance the last valid LSN and the truncation should be observable
-        tline.advance_last_valid_lsn(Lsn(2));
-        let snapshot = tline.history()?;
-        assert_eq!(snapshot.lsn(), Lsn(2));
-        let mut snapshot = skip_nonrel_objects(snapshot)?;
-        assert_eq!(
-            Some(&expected_init_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(
-            Some(&expected_trunc_size),
-            snapshot.next().transpose()?.as_ref()
-        );
-        assert_eq!(Some(&expected_page), snapshot.next().transpose()?.as_ref());
-        assert_eq!(None, snapshot.next().transpose()?);
-
-        Ok(())
-    }
-
-    // Mock WAL redo manager that doesn't do much
-    struct TestRedoManager {}
-
-    impl WalRedoManager for TestRedoManager {
-        fn request_redo(
-            &self,
-            rel: RelishTag,
-            blknum: u32,
-            lsn: Lsn,
-            base_img: Option<Bytes>,
-            records: Vec<WALRecord>,
-        ) -> Result<Bytes, WalRedoError> {
-            let s = format!(
-                "redo for {} blk {} to get to {}, with {} and {} records",
-                rel,
-                blknum,
-                lsn,
-                if base_img.is_some() {
-                    "base image"
-                } else {
-                    "no base image"
-                },
-                records.len()
-            );
-            println!("{}", s);
-            Ok(TEST_IMG(&s))
-        }
-    }
-}

pageserver/src/restore_local_repo.rs

@@ -1,969 +0,0 @@
-//!
-//! Import data and WAL from a PostgreSQL data directory and WAL segments into
-//! zenith Timeline.
-//!
-use log::*;
-use std::cmp::min;
-use std::fs;
-use std::fs::File;
-use std::io::Read;
-use std::io::Seek;
-use std::io::SeekFrom;
-use std::path::{Path, PathBuf};
-
-use anyhow::Result;
-use bytes::{Buf, Bytes};
-
-use crate::relish::*;
-use crate::repository::*;
-use crate::waldecoder::*;
-use postgres_ffi::relfile_utils::*;
-use postgres_ffi::xlog_utils::*;
-use postgres_ffi::{pg_constants, CheckPoint, ControlFileData};
-use postgres_ffi::{Oid, TransactionId};
-use zenith_utils::lsn::Lsn;
-
-const MAX_MBR_BLKNO: u32 =
-    pg_constants::MAX_MULTIXACT_ID / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-
-const ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
-
-///
-/// Import all relation data pages from local disk into the repository.
-///
-pub fn import_timeline_from_postgres_datadir(
-    path: &Path,
-    timeline: &dyn Timeline,
-    lsn: Lsn,
-) -> Result<()> {
-    // Scan 'global'
-    for direntry in fs::read_dir(path.join("global"))? {
-        let direntry = direntry?;
-        match direntry.file_name().to_str() {
-            None => continue,
-
-            // These special files appear in the snapshot, but are not needed by the page server
-            Some("pg_control") => {
-                import_nonrel_file(timeline, lsn, RelishTag::ControlFile, &direntry.path())?;
-                // Extract checkpoint record from pg_control and store is as separate object
-                let pg_control_bytes =
-                    timeline.get_page_at_lsn_nowait(RelishTag::ControlFile, 0, lsn)?;
-                let pg_control = ControlFileData::decode(&pg_control_bytes)?;
-                let checkpoint_bytes = pg_control.checkPointCopy.encode();
-                timeline.put_page_image(RelishTag::Checkpoint, 0, lsn, checkpoint_bytes, false)?;
-            }
-            Some("pg_filenode.map") => import_nonrel_file(
-                timeline,
-                lsn,
-                RelishTag::FileNodeMap {
-                    spcnode: pg_constants::GLOBALTABLESPACE_OID,
-                    dbnode: 0,
-                },
-                &direntry.path(),
-            )?,
-
-            // Load any relation files into the page server
-            _ => import_relfile(
-                &direntry.path(),
-                timeline,
-                lsn,
-                pg_constants::GLOBALTABLESPACE_OID,
-                0,
-            )?,
-        }
-    }
-
-    // Scan 'base'. It contains database dirs, the database OID is the filename.
-    // E.g. 'base/12345', where 12345 is the database OID.
-    for direntry in fs::read_dir(path.join("base"))? {
-        let direntry = direntry?;
-
-        //skip all temporary files
-        if direntry.file_name().to_str().unwrap() == "pgsql_tmp" {
-            continue;
-        }
-
-        let dboid = direntry.file_name().to_str().unwrap().parse::<u32>()?;
-
-        for direntry in fs::read_dir(direntry.path())? {
-            let direntry = direntry?;
-            match direntry.file_name().to_str() {
-                None => continue,
-
-                // These special files appear in the snapshot, but are not needed by the page server
-                Some("PG_VERSION") => continue,
-                Some("pg_filenode.map") => import_nonrel_file(
-                    timeline,
-                    lsn,
-                    RelishTag::FileNodeMap {
-                        spcnode: pg_constants::DEFAULTTABLESPACE_OID,
-                        dbnode: dboid,
-                    },
-                    &direntry.path(),
-                )?,
-
-                // Load any relation files into the page server
-                _ => import_relfile(
-                    &direntry.path(),
-                    timeline,
-                    lsn,
-                    pg_constants::DEFAULTTABLESPACE_OID,
-                    dboid,
-                )?,
-            }
-        }
-    }
-    for entry in fs::read_dir(path.join("pg_xact"))? {
-        let entry = entry?;
-        import_slru_file(timeline, lsn, SlruKind::Clog, &entry.path())?;
-    }
-    for entry in fs::read_dir(path.join("pg_multixact").join("members"))? {
-        let entry = entry?;
-        import_slru_file(timeline, lsn, SlruKind::MultiXactMembers, &entry.path())?;
-    }
-    for entry in fs::read_dir(path.join("pg_multixact").join("offsets"))? {
-        let entry = entry?;
-        import_slru_file(timeline, lsn, SlruKind::MultiXactOffsets, &entry.path())?;
-    }
-    for entry in fs::read_dir(path.join("pg_twophase"))? {
-        let entry = entry?;
-        let xid = u32::from_str_radix(&entry.path().to_str().unwrap(), 16)?;
-        import_nonrel_file(timeline, lsn, RelishTag::TwoPhase { xid }, &entry.path())?;
-    }
-    // TODO: Scan pg_tblspc
-
-    timeline.advance_last_valid_lsn(lsn);
-    timeline.checkpoint()?;
-
-    Ok(())
-}
-
-// subroutine of import_timeline_from_postgres_datadir(), to load one relation file.
-fn import_relfile(
-    path: &Path,
-    timeline: &dyn Timeline,
-    lsn: Lsn,
-    spcoid: Oid,
-    dboid: Oid,
-) -> Result<()> {
-    // Does it look like a relation file?
-
-    let p = parse_relfilename(path.file_name().unwrap().to_str().unwrap());
-    if let Err(e) = p {
-        warn!("unrecognized file in snapshot: {:?} ({})", path, e);
-        return Err(e.into());
-    }
-    let (relnode, forknum, segno) = p.unwrap();
-
-    let mut file = File::open(path)?;
-    let mut buf: [u8; 8192] = [0u8; 8192];
-
-    let mut blknum: u32 = segno * (1024 * 1024 * 1024 / pg_constants::BLCKSZ as u32);
-    loop {
-        let r = file.read_exact(&mut buf);
-        match r {
-            Ok(_) => {
-                let rel = RelTag {
-                    spcnode: spcoid,
-                    dbnode: dboid,
-                    relnode,
-                    forknum,
-                };
-                let tag = RelishTag::Relation(rel);
-                timeline.put_page_image(tag, blknum, lsn, Bytes::copy_from_slice(&buf), true)?;
-            }
-
-            // TODO: UnexpectedEof is expected
-            Err(e) => match e.kind() {
-                std::io::ErrorKind::UnexpectedEof => {
-                    // reached EOF. That's expected.
-                    // FIXME: maybe check that we read the full length of the file?
-                    break;
-                }
-                _ => {
-                    error!("error reading file: {:?} ({})", path, e);
-                    break;
-                }
-            },
-        };
-        blknum += 1;
-    }
-
-    Ok(())
-}
-
-///
-/// Import a "non-blocky" file into the repository
-///
-/// This is used for small files like the control file, twophase files etc. that
-/// are just slurped into the repository as one blob.
-///
-fn import_nonrel_file(
-    timeline: &dyn Timeline,
-    lsn: Lsn,
-    tag: RelishTag,
-    path: &Path,
-) -> Result<()> {
-    let mut file = File::open(path)?;
-    let mut buffer = Vec::new();
-    // read the whole file
-    file.read_to_end(&mut buffer)?;
-
-    info!("importing non-rel file {}", path.display());
-
-    timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buffer[..]), false)?;
-    Ok(())
-}
-
-///
-/// Import an SLRU segment file
-///
-fn import_slru_file(timeline: &dyn Timeline, lsn: Lsn, slru: SlruKind, path: &Path) -> Result<()> {
-    // Does it look like an SLRU file?
-    let mut file = File::open(path)?;
-    let mut buf: [u8; 8192] = [0u8; 8192];
-    let segno = u32::from_str_radix(path.file_name().unwrap().to_str().unwrap(), 16)?;
-
-    info!("importing slru file {}", path.display());
-
-    let mut rpageno = 0;
-    loop {
-        let r = file.read_exact(&mut buf);
-        match r {
-            Ok(_) => {
-                timeline.put_page_image(
-                    RelishTag::Slru { slru, segno },
-                    rpageno,
-                    lsn,
-                    Bytes::copy_from_slice(&buf),
-                    true,
-                )?;
-            }
-
-            // TODO: UnexpectedEof is expected
-            Err(e) => match e.kind() {
-                std::io::ErrorKind::UnexpectedEof => {
-                    // reached EOF. That's expected.
-                    // FIXME: maybe check that we read the full length of the file?
-                    break;
-                }
-                _ => {
-                    error!("error reading file: {:?} ({})", path, e);
-                    break;
-                }
-            },
-        };
-        rpageno += 1;
-
-        // TODO: Check that the file isn't unexpectedly large, not larger than SLRU_PAGES_PER_SEGMENT pages
-    }
-
-    Ok(())
-}
-
-/// Scan PostgreSQL WAL files in given directory
-/// and load all records >= 'startpoint' into the repository.
-pub fn import_timeline_wal(walpath: &Path, timeline: &dyn Timeline, startpoint: Lsn) -> Result<()> {
-    let mut waldecoder = WalStreamDecoder::new(startpoint, true);
-
-    let mut segno = startpoint.segment_number(pg_constants::WAL_SEGMENT_SIZE);
-    let mut offset = startpoint.segment_offset(pg_constants::WAL_SEGMENT_SIZE);
-    let mut last_lsn = startpoint;
-
-    let checkpoint_bytes = timeline.get_page_at_lsn_nowait(RelishTag::Checkpoint, 0, startpoint)?;
-    let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
-
-    loop {
-        // FIXME: assume postgresql tli 1 for now
-        let filename = XLogFileName(1, segno, pg_constants::WAL_SEGMENT_SIZE);
-        let mut buf = Vec::new();
-
-        //Read local file
-        let mut path = walpath.join(&filename);
-
-        // It could be as .partial
-        if !PathBuf::from(&path).exists() {
-            path = walpath.join(filename + ".partial");
-        }
-
-        // Slurp the WAL file
-        let open_result = File::open(&path);
-        if let Err(e) = &open_result {
-            if e.kind() == std::io::ErrorKind::NotFound {
-                break;
-            }
-        }
-        let mut file = open_result?;
-
-        if offset > 0 {
-            file.seek(SeekFrom::Start(offset as u64))?;
-        }
-
-        let nread = file.read_to_end(&mut buf)?;
-        if nread != pg_constants::WAL_SEGMENT_SIZE - offset as usize {
-            // Maybe allow this for .partial files?
-            error!("read only {} bytes from WAL file", nread);
-        }
-
-        waldecoder.feed_bytes(&buf);
-
-        let mut nrecords = 0;
-        loop {
-            let rec = waldecoder.poll_decode();
-            if rec.is_err() {
-                // Assume that an error means we've reached the end of
-                // a partial WAL record. So that's ok.
-                trace!("WAL decoder error {:?}", rec);
-                break;
-            }
-            if let Some((lsn, recdata)) = rec.unwrap() {
-                let decoded = decode_wal_record(recdata.clone());
-                save_decoded_record(&mut checkpoint, timeline, &decoded, recdata, lsn)?;
-                last_lsn = lsn;
-            } else {
-                break;
-            }
-            nrecords += 1;
-        }
-
-        info!("imported {} records up to {}", nrecords, last_lsn);
-
-        segno += 1;
-        offset = 0;
-    }
-
-    info!("reached end of WAL at {}", last_lsn);
-    let checkpoint_bytes = checkpoint.encode();
-    timeline.put_page_image(RelishTag::Checkpoint, 0, last_lsn, checkpoint_bytes, false)?;
-
-    timeline.advance_last_valid_lsn(last_lsn);
-    timeline.checkpoint()?;
-    Ok(())
-}
-
-///
-/// Helper function to parse a WAL record and call the Timeline's PUT functions for all the
-/// relations/pages that the record affects.
-///
-pub fn save_decoded_record(
-    checkpoint: &mut CheckPoint,
-    timeline: &dyn Timeline,
-    decoded: &DecodedWALRecord,
-    recdata: Bytes,
-    lsn: Lsn,
-) -> Result<()> {
-    checkpoint.update_next_xid(decoded.xl_xid);
-
-    // Iterate through all the blocks that the record modifies, and
-    // "put" a separate copy of the record for each block.
-    for blk in decoded.blocks.iter() {
-        let tag = RelishTag::Relation(RelTag {
-            spcnode: blk.rnode_spcnode,
-            dbnode: blk.rnode_dbnode,
-            relnode: blk.rnode_relnode,
-            forknum: blk.forknum as u8,
-        });
-
-        let rec = WALRecord {
-            lsn,
-            will_init: blk.will_init || blk.apply_image,
-            rec: recdata.clone(),
-            main_data_offset: decoded.main_data_offset as u32,
-        };
-
-        timeline.put_wal_record(tag, blk.blkno, rec)?;
-    }
-
-    let mut buf = decoded.record.clone();
-    buf.advance(decoded.main_data_offset);
-
-    // Handle a few special record types
-    if decoded.xl_rmid == pg_constants::RM_SMGR_ID
-        && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) == pg_constants::XLOG_SMGR_TRUNCATE
-    {
-        let truncate = XlSmgrTruncate::decode(&mut buf);
-        save_xlog_smgr_truncate(timeline, lsn, &truncate)?;
-    } else if decoded.xl_rmid == pg_constants::RM_DBASE_ID {
-        if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) == pg_constants::XLOG_DBASE_CREATE {
-            let createdb = XlCreateDatabase::decode(&mut buf);
-            save_xlog_dbase_create(timeline, lsn, &createdb)?;
-        } else {
-            // TODO
-            trace!("XLOG_DBASE_DROP is not handled yet");
-        }
-    } else if decoded.xl_rmid == pg_constants::RM_TBLSPC_ID {
-        trace!("XLOG_TBLSPC_CREATE/DROP is not handled yet");
-    } else if decoded.xl_rmid == pg_constants::RM_CLOG_ID {
-        let info = decoded.xl_info & !pg_constants::XLR_INFO_MASK;
-        if info == pg_constants::CLOG_ZEROPAGE {
-            let pageno = buf.get_u32_le();
-            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-            timeline.put_page_image(
-                RelishTag::Slru {
-                    slru: SlruKind::Clog,
-                    segno,
-                },
-                rpageno,
-                lsn,
-                ZERO_PAGE,
-                true,
-            )?;
-        } else {
-            assert!(info == pg_constants::CLOG_TRUNCATE);
-            let xlrec = XlClogTruncate::decode(&mut buf);
-            save_clog_truncate_record(checkpoint, timeline, lsn, &xlrec)?;
-        }
-    } else if decoded.xl_rmid == pg_constants::RM_XACT_ID {
-        let info = decoded.xl_info & pg_constants::XLOG_XACT_OPMASK;
-        if info == pg_constants::XLOG_XACT_COMMIT || info == pg_constants::XLOG_XACT_ABORT {
-            let parsed_xact = XlXactParsedRecord::decode(&mut buf, decoded.xl_xid, decoded.xl_info);
-            save_xact_record(timeline, lsn, &parsed_xact, decoded)?;
-        } else if info == pg_constants::XLOG_XACT_COMMIT_PREPARED
-            || info == pg_constants::XLOG_XACT_ABORT_PREPARED
-        {
-            let parsed_xact = XlXactParsedRecord::decode(&mut buf, decoded.xl_xid, decoded.xl_info);
-            save_xact_record(timeline, lsn, &parsed_xact, decoded)?;
-            // Remove twophase file. see RemoveTwoPhaseFile() in postgres code
-            trace!(
-                "unlink twophaseFile for xid {} parsed_xact.xid {} here at {}",
-                decoded.xl_xid,
-                parsed_xact.xid,
-                lsn
-            );
-            timeline.put_unlink(
-                RelishTag::TwoPhase {
-                    xid: parsed_xact.xid,
-                },
-                lsn,
-            )?;
-        } else if info == pg_constants::XLOG_XACT_PREPARE {
-            let mut buf = decoded.record.clone();
-            buf.advance(decoded.main_data_offset);
-
-            timeline.put_page_image(
-                RelishTag::TwoPhase {
-                    xid: decoded.xl_xid,
-                },
-                0,
-                lsn,
-                Bytes::copy_from_slice(&buf[..]),
-                true,
-            )?;
-        }
-    } else if decoded.xl_rmid == pg_constants::RM_MULTIXACT_ID {
-        let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
-
-        if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE {
-            let pageno = buf.get_u32_le();
-            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-            timeline.put_page_image(
-                RelishTag::Slru {
-                    slru: SlruKind::MultiXactOffsets,
-                    segno,
-                },
-                rpageno,
-                lsn,
-                ZERO_PAGE,
-                true,
-            )?;
-        } else if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE {
-            let pageno = buf.get_u32_le();
-            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-            timeline.put_page_image(
-                RelishTag::Slru {
-                    slru: SlruKind::MultiXactMembers,
-                    segno,
-                },
-                rpageno,
-                lsn,
-                ZERO_PAGE,
-                true,
-            )?;
-        } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
-            let xlrec = XlMultiXactCreate::decode(&mut buf);
-            save_multixact_create_record(checkpoint, timeline, lsn, &xlrec, decoded)?;
-        } else if info == pg_constants::XLOG_MULTIXACT_TRUNCATE_ID {
-            let xlrec = XlMultiXactTruncate::decode(&mut buf);
-            save_multixact_truncate_record(checkpoint, timeline, lsn, &xlrec)?;
-        }
-    } else if decoded.xl_rmid == pg_constants::RM_RELMAP_ID {
-        let xlrec = XlRelmapUpdate::decode(&mut buf);
-        save_relmap_record(timeline, lsn, &xlrec, decoded)?;
-    } else if decoded.xl_rmid == pg_constants::RM_XLOG_ID {
-        let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
-        if info == pg_constants::XLOG_NEXTOID {
-            let next_oid = buf.get_u32_le();
-            checkpoint.nextOid = next_oid;
-        } else if info == pg_constants::XLOG_CHECKPOINT_ONLINE
-            || info == pg_constants::XLOG_CHECKPOINT_SHUTDOWN
-        {
-            let mut checkpoint_bytes = [0u8; SIZEOF_CHECKPOINT];
-            let mut buf = decoded.record.clone();
-            buf.advance(decoded.main_data_offset);
-            buf.copy_to_slice(&mut checkpoint_bytes);
-            let xlog_checkpoint = CheckPoint::decode(&checkpoint_bytes).unwrap();
-            trace!(
-                "xlog_checkpoint.oldestXid={}, checkpoint.oldestXid={}",
-                xlog_checkpoint.oldestXid,
-                checkpoint.oldestXid
-            );
-            if (checkpoint.oldestXid.wrapping_sub(xlog_checkpoint.oldestXid) as i32) < 0 {
-                checkpoint.oldestXid = xlog_checkpoint.oldestXid;
-            }
-        }
-    }
-    // Now that this record has been handled, let the repository know that
-    // it is up-to-date to this LSN
-    timeline.advance_last_record_lsn(lsn);
-    Ok(())
-}
-
-/// Subroutine of save_decoded_record(), to handle an XLOG_DBASE_CREATE record.
-fn save_xlog_dbase_create(timeline: &dyn Timeline, lsn: Lsn, rec: &XlCreateDatabase) -> Result<()> {
-    let db_id = rec.db_id;
-    let tablespace_id = rec.tablespace_id;
-    let src_db_id = rec.src_db_id;
-    let src_tablespace_id = rec.src_tablespace_id;
-
-    // Creating a database is implemented by copying the template (aka. source) database.
-    // To copy all the relations, we need to ask for the state as of the same LSN, but we
-    // cannot pass 'lsn' to the Timeline.get_* functions, or they will block waiting for
-    // the last valid LSN to advance up to it. So we use the previous record's LSN in the
-    // get calls instead.
-    let req_lsn = min(timeline.get_last_record_lsn(), lsn);
-
-    let rels = timeline.list_rels(src_tablespace_id, src_db_id, req_lsn)?;
-
-    trace!("save_create_database: {} rels", rels.len());
-
-    let mut num_rels_copied = 0;
-    let mut num_blocks_copied = 0;
-    for src_rel in rels {
-        assert_eq!(src_rel.spcnode, src_tablespace_id);
-        assert_eq!(src_rel.dbnode, src_db_id);
-
-        let nblocks = timeline
-            .get_relish_size(RelishTag::Relation(src_rel), req_lsn)?
-            .unwrap_or(0);
-        let dst_rel = RelTag {
-            spcnode: tablespace_id,
-            dbnode: db_id,
-            relnode: src_rel.relnode,
-            forknum: src_rel.forknum,
-        };
-
-        // Copy content
-        for blknum in 0..nblocks {
-            let content =
-                timeline.get_page_at_lsn_nowait(RelishTag::Relation(src_rel), blknum, req_lsn)?;
-
-            debug!("copying block {} from {} to {}", blknum, src_rel, dst_rel);
-
-            timeline.put_page_image(RelishTag::Relation(dst_rel), blknum, lsn, content, true)?;
-            num_blocks_copied += 1;
-        }
-
-        if nblocks == 0 {
-            // make sure we have some trace of the relation, even if it's empty
-            timeline.put_truncation(RelishTag::Relation(dst_rel), lsn, 0)?;
-        }
-
-        num_rels_copied += 1;
-    }
-
-    // Copy relfilemap
-    // TODO This implementation is very inefficient -
-    // it scans all non-rels only to find FileNodeMaps
-    for tag in timeline.list_nonrels(req_lsn)? {
-        match tag {
-            RelishTag::FileNodeMap { spcnode, dbnode } => {
-                if spcnode == src_tablespace_id && dbnode == src_db_id {
-                    let img = timeline.get_page_at_lsn_nowait(tag, 0, req_lsn)?;
-                    let new_tag = RelishTag::FileNodeMap {
-                        spcnode: tablespace_id,
-                        dbnode: db_id,
-                    };
-                    timeline.put_page_image(new_tag, 0, lsn, img, false)?;
-                    break;
-                }
-            }
-            _ => {} // do nothing
-        }
-    }
-    info!(
-        "Created database {}/{}, copied {} blocks in {} rels at {}",
-        tablespace_id, db_id, num_blocks_copied, num_rels_copied, lsn
-    );
-    Ok(())
-}
-
-/// Subroutine of save_decoded_record(), to handle an XLOG_SMGR_TRUNCATE record.
-///
-/// This is the same logic as in PostgreSQL's smgr_redo() function.
-fn save_xlog_smgr_truncate(timeline: &dyn Timeline, lsn: Lsn, rec: &XlSmgrTruncate) -> Result<()> {
-    let spcnode = rec.rnode.spcnode;
-    let dbnode = rec.rnode.dbnode;
-    let relnode = rec.rnode.relnode;
-
-    if (rec.flags & pg_constants::SMGR_TRUNCATE_HEAP) != 0 {
-        let rel = RelTag {
-            spcnode,
-            dbnode,
-            relnode,
-            forknum: pg_constants::MAIN_FORKNUM,
-        };
-        timeline.put_truncation(RelishTag::Relation(rel), lsn, rec.blkno)?;
-    }
-    if (rec.flags & pg_constants::SMGR_TRUNCATE_FSM) != 0 {
-        let rel = RelTag {
-            spcnode,
-            dbnode,
-            relnode,
-            forknum: pg_constants::FSM_FORKNUM,
-        };
-
-        // FIXME: 'blkno' stored in the WAL record is the new size of the
-        // heap. The formula for calculating the new size of the FSM is
-        // pretty complicated (see FreeSpaceMapPrepareTruncateRel() in
-        // PostgreSQL), and we should also clear bits in the tail FSM block,
-        // and update the upper level FSM pages. None of that has been
-        // implemented. What we do instead, is always just truncate the FSM
-        // to zero blocks. That's bad for performance, but safe. (The FSM
-        // isn't needed for correctness, so we could also leave garbage in
-        // it. Seems more tidy to zap it away.)
-        if rec.blkno != 0 {
-            info!("Partial truncation of FSM is not supported");
-        }
-        let num_fsm_blocks = 0;
-        timeline.put_truncation(RelishTag::Relation(rel), lsn, num_fsm_blocks)?;
-    }
-    if (rec.flags & pg_constants::SMGR_TRUNCATE_VM) != 0 {
-        let rel = RelTag {
-            spcnode,
-            dbnode,
-            relnode,
-            forknum: pg_constants::VISIBILITYMAP_FORKNUM,
-        };
-
-        // FIXME: Like with the FSM above, the logic to truncate the VM
-        // correctly has not been implemented. Just zap it away completely,
-        // always. Unlike the FSM, the VM must never have bits incorrectly
-        // set. From a correctness point of view, it's always OK to clear
-        // bits or remove it altogether, though.
-        if rec.blkno != 0 {
-            info!("Partial truncation of VM is not supported");
-        }
-        let num_vm_blocks = 0;
-        timeline.put_truncation(RelishTag::Relation(rel), lsn, num_vm_blocks)?;
-    }
-    Ok(())
-}
-
-/// Subroutine of save_decoded_record(), to handle an XLOG_XACT_* records.
-///
-/// We are currently only interested in the dropped relations.
-fn save_xact_record(
-    timeline: &dyn Timeline,
-    lsn: Lsn,
-    parsed: &XlXactParsedRecord,
-    decoded: &DecodedWALRecord,
-) -> Result<()> {
-    // Record update of CLOG page
-    let mut pageno = parsed.xid / pg_constants::CLOG_XACTS_PER_PAGE;
-
-    let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-    let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-    let rec = WALRecord {
-        lsn,
-        will_init: false,
-        rec: decoded.record.clone(),
-        main_data_offset: decoded.main_data_offset as u32,
-    };
-    timeline.put_wal_record(
-        RelishTag::Slru {
-            slru: SlruKind::Clog,
-            segno,
-        },
-        rpageno,
-        rec.clone(),
-    )?;
-
-    for subxact in &parsed.subxacts {
-        let subxact_pageno = subxact / pg_constants::CLOG_XACTS_PER_PAGE;
-        if subxact_pageno != pageno {
-            pageno = subxact_pageno;
-            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-            timeline.put_wal_record(
-                RelishTag::Slru {
-                    slru: SlruKind::Clog,
-                    segno,
-                },
-                rpageno,
-                rec.clone(),
-            )?;
-        }
-    }
-    for xnode in &parsed.xnodes {
-        for forknum in pg_constants::MAIN_FORKNUM..=pg_constants::VISIBILITYMAP_FORKNUM {
-            let rel = RelTag {
-                forknum,
-                spcnode: xnode.spcnode,
-                dbnode: xnode.dbnode,
-                relnode: xnode.relnode,
-            };
-            timeline.put_unlink(RelishTag::Relation(rel), lsn)?;
-        }
-    }
-    Ok(())
-}
-
-// See TransactionIdPrecedes in transam.c
-fn transaction_id_precedes(id1: TransactionId, id2: TransactionId) -> bool {
-    /*
-     * If either ID is a permanent XID then we can just do unsigned
-     * comparison.  If both are normal, do a modulo-2^32 comparison.
-     */
-
-    if !(id1 >= pg_constants::FIRST_NORMAL_TRANSACTION_ID)
-        || !(id2 >= pg_constants::FIRST_NORMAL_TRANSACTION_ID)
-    {
-        return id1 < id2;
-    }
-
-    let diff = id1.wrapping_sub(id2) as i32;
-    return diff < 0;
-}
-
-// See CLOGPagePrecedes in clog.c
-fn clogpage_precedes(page1: u32, page2: u32) -> bool {
-    let mut xid1 = page1 * pg_constants::CLOG_XACTS_PER_PAGE;
-    xid1 += pg_constants::FIRST_NORMAL_TRANSACTION_ID + 1;
-    let mut xid2 = page2 * pg_constants::CLOG_XACTS_PER_PAGE;
-    xid2 += pg_constants::FIRST_NORMAL_TRANSACTION_ID + 1;
-
-    return transaction_id_precedes(xid1, xid2)
-        && transaction_id_precedes(xid1, xid2 + pg_constants::CLOG_XACTS_PER_PAGE - 1);
-}
-
-// See SlruMayDeleteSegment() in slru.c
-fn slru_may_delete_clogsegment(segpage: u32, cutoff_page: u32) -> bool {
-    let seg_last_page = segpage + pg_constants::SLRU_PAGES_PER_SEGMENT - 1;
-
-    assert_eq!(segpage % pg_constants::SLRU_PAGES_PER_SEGMENT, 0);
-
-    return clogpage_precedes(segpage, cutoff_page)
-        && clogpage_precedes(seg_last_page, cutoff_page);
-}
-
-fn save_clog_truncate_record(
-    checkpoint: &mut CheckPoint,
-    timeline: &dyn Timeline,
-    lsn: Lsn,
-    xlrec: &XlClogTruncate,
-) -> Result<()> {
-    info!(
-        "RM_CLOG_ID truncate pageno {} oldestXid {} oldestXidDB {} lsn {}",
-        xlrec.pageno, xlrec.oldest_xid, xlrec.oldest_xid_db, lsn
-    );
-
-    // Here we treat oldestXid and oldestXidDB
-    // differently from postgres redo routines.
-    // In postgres checkpoint.oldestXid lags behind xlrec.oldest_xid
-    // until checkpoint happens and updates the value.
-    // Here we can use the most recent value.
-    // It's just an optimization, though and can be deleted.
-    // TODO Figure out if there will be any issues with replica.
-    checkpoint.oldestXid = xlrec.oldest_xid;
-    checkpoint.oldestXidDB = xlrec.oldest_xid_db;
-
-    // TODO Treat AdvanceOldestClogXid() or write a comment why we don't need it
-
-    let latest_page_number = checkpoint.nextXid.value as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
-
-    // Now delete all segments containing pages between xlrec.pageno
-    // and latest_page_number.
-
-    // First, make an important safety check:
-    // the current endpoint page must not be eligible for removal.
-    // See SimpleLruTruncate() in slru.c
-    if clogpage_precedes(latest_page_number, xlrec.pageno) {
-        info!("could not truncate directory pg_xact apparent wraparound");
-        return Ok(());
-    }
-
-    // Iterate via SLRU CLOG segments and unlink segments that we're ready to truncate
-    // TODO This implementation is very inefficient -
-    // it scans all non-rels only to find Clog
-    //
-    // We cannot pass 'lsn' to the Timeline.list_nonrels(), or it
-    // will block waiting for the last valid LSN to advance up to
-    // it. So we use the previous record's LSN in the get calls
-    // instead.
-    let req_lsn = min(timeline.get_last_record_lsn(), lsn);
-    for obj in timeline.list_nonrels(req_lsn)? {
-        match obj {
-            RelishTag::Slru { slru, segno } => {
-                if slru == SlruKind::Clog {
-                    let segpage = segno * pg_constants::SLRU_PAGES_PER_SEGMENT;
-                    if slru_may_delete_clogsegment(segpage, xlrec.pageno) {
-                        timeline.put_unlink(RelishTag::Slru { slru, segno }, lsn)?;
-                        trace!("unlink CLOG segment {:>04X} at lsn {}", segno, lsn);
-                    }
-                }
-            }
-            _ => {}
-        }
-    }
-
-    Ok(())
-}
-
-fn save_multixact_create_record(
-    checkpoint: &mut CheckPoint,
-    timeline: &dyn Timeline,
-    lsn: Lsn,
-    xlrec: &XlMultiXactCreate,
-    decoded: &DecodedWALRecord,
-) -> Result<()> {
-    let rec = WALRecord {
-        lsn,
-        will_init: false,
-        rec: decoded.record.clone(),
-        main_data_offset: decoded.main_data_offset as u32,
-    };
-    let pageno = xlrec.mid / pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32;
-    let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-    let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-    timeline.put_wal_record(
-        RelishTag::Slru {
-            slru: SlruKind::MultiXactOffsets,
-            segno,
-        },
-        rpageno,
-        rec.clone(),
-    )?;
-
-    let first_mbr_pageno = xlrec.moff / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-    let last_mbr_pageno =
-        (xlrec.moff + xlrec.nmembers - 1) / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-    // The members SLRU can, in contrast to the offsets one, be filled to almost
-    // the full range at once. So we need to handle wraparound.
-    let mut pageno = first_mbr_pageno;
-    loop {
-        // Update members page
-        let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-        let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-        timeline.put_wal_record(
-            RelishTag::Slru {
-                slru: SlruKind::MultiXactMembers,
-                segno,
-            },
-            rpageno,
-            rec.clone(),
-        )?;
-
-        if pageno == last_mbr_pageno {
-            // last block inclusive
-            break;
-        }
-
-        // handle wraparound
-        if pageno == MAX_MBR_BLKNO {
-            pageno = 0;
-        } else {
-            pageno += 1;
-        }
-    }
-    if xlrec.mid >= checkpoint.nextMulti {
-        checkpoint.nextMulti = xlrec.mid + 1;
-    }
-    if xlrec.moff + xlrec.nmembers > checkpoint.nextMultiOffset {
-        checkpoint.nextMultiOffset = xlrec.moff + xlrec.nmembers;
-    }
-    let max_mbr_xid = xlrec.members.iter().fold(0u32, |acc, mbr| {
-        if mbr.xid.wrapping_sub(acc) as i32 > 0 {
-            mbr.xid
-        } else {
-            acc
-        }
-    });
-
-    checkpoint.update_next_xid(max_mbr_xid);
-    Ok(())
-}
-
-#[allow(non_upper_case_globals)]
-const MaxMultiXactOffset: u32 = 0xFFFFFFFF;
-
-#[allow(non_snake_case)]
-const fn MXOffsetToMemberPage(xid: u32) -> u32 {
-    xid / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32
-}
-#[allow(non_snake_case)]
-const fn MXOffsetToMemberSegment(xid: u32) -> i32 {
-    (MXOffsetToMemberPage(xid) / pg_constants::SLRU_PAGES_PER_SEGMENT) as i32
-}
-
-fn save_multixact_truncate_record(
-    checkpoint: &mut CheckPoint,
-    timeline: &dyn Timeline,
-    lsn: Lsn,
-    xlrec: &XlMultiXactTruncate,
-) -> Result<()> {
-    checkpoint.oldestMulti = xlrec.end_trunc_off;
-    checkpoint.oldestMultiDB = xlrec.oldest_multi_db;
-
-    // PerformMembersTruncation
-    let maxsegment: i32 = MXOffsetToMemberSegment(MaxMultiXactOffset);
-    let startsegment: i32 = MXOffsetToMemberSegment(xlrec.start_trunc_memb);
-    let endsegment: i32 = MXOffsetToMemberSegment(xlrec.end_trunc_memb);
-    let mut segment: i32 = startsegment;
-
-    // Delete all the segments except the last one. The last segment can still
-    // contain, possibly partially, valid data.
-    while segment != endsegment {
-        timeline.put_unlink(
-            RelishTag::Slru {
-                slru: SlruKind::MultiXactMembers,
-                segno: segment as u32,
-            },
-            lsn,
-        )?;
-
-        /* move to next segment, handling wraparound correctly */
-        if segment == maxsegment {
-            segment = 0;
-        } else {
-            segment += 1;
-        }
-    }
-
-    // Truncate offsets
-    // FIXME: this did not handle wraparound correctly
-
-    Ok(())
-}
-
-fn save_relmap_record(
-    timeline: &dyn Timeline,
-    lsn: Lsn,
-    xlrec: &XlRelmapUpdate,
-    decoded: &DecodedWALRecord,
-) -> Result<()> {
-    let rec = WALRecord {
-        lsn,
-        will_init: true,
-        rec: decoded.record.clone(),
-        main_data_offset: decoded.main_data_offset as u32,
-    };
-    let tag = RelishTag::FileNodeMap {
-        spcnode: xlrec.tsid,
-        dbnode: xlrec.dbid,
-    };
-    timeline.put_wal_record(tag, 0, rec)?;
-    Ok(())
-}

pageserver/src/restore_s3.rs

@@ -0,0 +1,486 @@
+//
+// Restore chunks from S3
+//
+// This runs once at Page Server startup. It loads all the "base images" from
+// S3 into the in-memory page cache. It also initializes the "last valid LSN"
+// in the page cache to the LSN of the base image, so that when the WAL receiver
+// is started, it starts streaming from that LSN.
+//
+
+use bytes::{Buf, BytesMut};
+use log::*;
+use regex::Regex;
+use std::env;
+use std::fmt;
+
+use s3::bucket::Bucket;
+use s3::creds::Credentials;
+use s3::region::Region;
+use s3::S3Error;
+
+use tokio::runtime;
+
+use futures::future;
+
+use crate::{controlfile, page_cache, pg_constants, PageServerConf};
+
+struct Storage {
+    region: Region,
+    credentials: Credentials,
+    bucket: String,
+}
+
+pub fn restore_main(conf: &PageServerConf) {
+    // Create a new thread pool
+    let runtime = runtime::Runtime::new().unwrap();
+
+    runtime.block_on(async {
+        let result = restore_chunk(conf).await;
+
+        match result {
+            Ok(_) => {
+                return;
+            }
+            Err(err) => {
+                error!("S3 error: {}", err);
+                return;
+            }
+        }
+    });
+}
+
+//
+// Restores one chunk from S3.
+//
+// 1. Fetch the last base image >= given LSN
+// 2. Fetch all WAL
+//
+// Load it all into the page cache.
+//
+async fn restore_chunk(conf: &PageServerConf) -> Result<(), S3Error> {
+    let backend = Storage {
+        region: Region::Custom {
+            region: env::var("S3_REGION").unwrap().into(),
+            endpoint: env::var("S3_ENDPOINT").unwrap().into(),
+        },
+        credentials: Credentials::new(
+            Some(&env::var("S3_ACCESSKEY").unwrap()),
+            Some(&env::var("S3_SECRET").unwrap()),
+            None,
+            None,
+            None,
+        )
+        .unwrap(),
+        bucket: "zenith-testbucket".to_string(),
+    };
+
+    info!("Restoring from S3...");
+
+    // Create Bucket in REGION for BUCKET
+    let bucket = Bucket::new_with_path_style(&backend.bucket, backend.region, backend.credentials)?;
+
+    // List out contents of directory
+    let results: Vec<s3::serde_types::ListBucketResult> = bucket
+        .list("relationdata/".to_string(), Some("".to_string()))
+        .await?;
+
+    //Before uploading other files, slurp pg_control to set systemid
+
+    let control_results: Vec<s3::serde_types::ListBucketResult> = bucket
+        .list(
+            "relationdata/global/pg_control".to_string(),
+            Some("".to_string()),
+        )
+        .await?;
+    let object = &(&control_results[0]).contents[0];
+    let (data, _) = bucket.get_object(&object.key).await.unwrap();
+    let bytes = BytesMut::from(data.as_slice()).freeze();
+    let c = controlfile::decode_pg_control(bytes);
+
+    let pcache = page_cache::get_pagecache(conf.clone(), c.system_identifier);
+    pcache.set_controldata(c.clone());
+    trace!("uploaded controlfile {:?}", pcache.get_controldata());
+
+    let sys_id: u64 = c.system_identifier;
+    let mut oldest_lsn = 0;
+    let mut slurp_futures: Vec<_> = Vec::new();
+
+    for result in results {
+        for object in result.contents {
+            // Download every relation file, slurping them into memory
+
+            let key = object.key;
+            let relpath = key.strip_prefix("relationdata/").unwrap();
+
+            let parsed = parse_rel_file_path(&relpath);
+
+            match parsed {
+                Ok(p) => {
+                    if oldest_lsn == 0 || p.lsn < oldest_lsn {
+                        oldest_lsn = p.lsn;
+                    }
+                    let b = bucket.clone();
+                    let f = slurp_base_file(conf, sys_id, b, key.to_string(), p);
+
+                    slurp_futures.push(f);
+                }
+                Err(e) => {
+                    warn!("unrecognized file: {} ({})", relpath, e);
+                }
+            };
+        }
+    }
+
+    if oldest_lsn == 0 {
+        panic!("no base backup found");
+    }
+
+    //Now add nonrelation files
+    let nonrelresults: Vec<s3::serde_types::ListBucketResult> = bucket
+        .list("nonreldata/".to_string(), Some("".to_string()))
+        .await?;
+    for result in nonrelresults {
+        for object in result.contents {
+            // Download needed non relation files, slurping them into memory
+
+            let key = object.key;
+            let relpath = key.strip_prefix("nonreldata/").unwrap();
+            trace!("list nonrelfiles {}", relpath);
+
+            let parsed = parse_nonrel_file_path(&relpath);
+
+            match parsed {
+                Ok(p) => {
+                    let b = bucket.clone();
+                    let f = slurp_base_file(conf, sys_id, b, key.to_string(), p);
+
+                    slurp_futures.push(f);
+                }
+                Err(e) => {
+                    warn!("unrecognized file: {} ({})", relpath, e);
+                }
+            };
+        }
+    }
+
+    pcache.init_valid_lsn(oldest_lsn);
+
+    info!("{} files to restore...", slurp_futures.len());
+
+    future::join_all(slurp_futures).await;
+    info!(
+        "restored! {:?} to {:?}",
+        pcache.first_valid_lsn, pcache.last_valid_lsn
+    );
+
+    Ok(())
+}
+
+#[derive(Debug)]
+struct FilePathError {
+    msg: String,
+}
+
+impl FilePathError {
+    fn new(msg: &str) -> FilePathError {
+        FilePathError {
+            msg: msg.to_string(),
+        }
+    }
+}
+
+impl From<core::num::ParseIntError> for FilePathError {
+    fn from(e: core::num::ParseIntError) -> Self {
+        return FilePathError {
+            msg: format!("invalid filename: {}", e),
+        };
+    }
+}
+
+impl fmt::Display for FilePathError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "invalid filename")
+    }
+}
+
+fn forkname_to_forknum(forkname: Option<&str>) -> Result<u32, FilePathError> {
+    match forkname {
+        // "main" is not in filenames, it's implicit if the fork name is not present
+        None => Ok(0),
+        Some("fsm") => Ok(1),
+        Some("vm") => Ok(2),
+        Some("init") => Ok(3),
+        Some(_) => Err(FilePathError::new("invalid forkname")),
+    }
+}
+
+#[derive(Debug)]
+struct ParsedBaseImageFileName {
+    pub spcnode: u32,
+    pub dbnode: u32,
+    pub relnode: u32,
+    pub forknum: u32,
+    pub segno: u32,
+
+    pub lsn: u64,
+}
+
+fn parse_lsn_from_filename(fname: &str) -> Result<u64, FilePathError> {
+    let (_, lsn_str) = fname.split_at(fname.len() - 16);
+
+    let (lsnhi, lsnlo) = lsn_str.split_at(8);
+    let lsn_hi = u64::from_str_radix(lsnhi, 16)?;
+    let lsn_lo = u64::from_str_radix(lsnlo, 16)?;
+    let lsn = lsn_hi << 32 | lsn_lo;
+
+    return Ok(lsn);
+}
+
+// formats:
+// <oid>
+// <oid>_<fork name>
+// <oid>.<segment number>
+// <oid>_<fork name>.<segment number>
+
+fn parse_filename(fname: &str) -> Result<(u32, u32, u32, u64), FilePathError> {
+    let re = Regex::new(r"^(?P<relnode>\d+)(_(?P<forkname>[a-z]+))?(\.(?P<segno>\d+))?_(?P<lsnhi>[[:xdigit:]]{8})(?P<lsnlo>[[:xdigit:]]{8})$").unwrap();
+
+    let caps = re
+        .captures(fname)
+        .ok_or_else(|| FilePathError::new("invalid relation data file name"))?;
+
+    let relnode_str = caps.name("relnode").unwrap().as_str();
+    let relnode = u32::from_str_radix(relnode_str, 10)?;
+
+    let forkname_match = caps.name("forkname");
+    let forkname = if forkname_match.is_none() {
+        None
+    } else {
+        Some(forkname_match.unwrap().as_str())
+    };
+    let forknum = forkname_to_forknum(forkname)?;
+
+    let segno_match = caps.name("segno");
+    let segno = if segno_match.is_none() {
+        0
+    } else {
+        u32::from_str_radix(segno_match.unwrap().as_str(), 10)?
+    };
+
+    let lsn_hi = u64::from_str_radix(caps.name("lsnhi").unwrap().as_str(), 16)?;
+    let lsn_lo = u64::from_str_radix(caps.name("lsnlo").unwrap().as_str(), 16)?;
+    let lsn = lsn_hi << 32 | lsn_lo;
+
+    return Ok((relnode, forknum, segno, lsn));
+}
+
+fn parse_nonrel_file_path(path: &str) -> Result<ParsedBaseImageFileName, FilePathError> {
+    //TODO parse segno from xact filenames too
+    if let Some(fname) = path.strip_prefix("pg_xact/") {
+        let lsn = parse_lsn_from_filename(fname.clone())?;
+
+        return Ok(ParsedBaseImageFileName {
+            spcnode: 0,
+            dbnode: 0,
+            relnode: 0,
+            forknum: pg_constants::PG_XACT_FORKNUM,
+            segno: 0,
+            lsn,
+        });
+    } else if let Some(fname) = path.strip_prefix("pg_multixact/offsets") {
+        let lsn = parse_lsn_from_filename(fname.clone())?;
+
+        return Ok(ParsedBaseImageFileName {
+            spcnode: 0,
+            dbnode: 0,
+            relnode: 0,
+            forknum: pg_constants::PG_MXACT_OFFSETS_FORKNUM,
+            segno: 0,
+            lsn,
+        });
+    } else if let Some(fname) = path.strip_prefix("pg_multixact/members") {
+        let lsn = parse_lsn_from_filename(fname.clone())?;
+
+        return Ok(ParsedBaseImageFileName {
+            spcnode: 0,
+            dbnode: 0,
+            relnode: 0,
+            forknum: pg_constants::PG_MXACT_MEMBERS_FORKNUM,
+            segno: 0,
+            lsn,
+        });
+    } else {
+        return Err(FilePathError::new("invalid non relation data file name"));
+    }
+}
+
+fn parse_rel_file_path(path: &str) -> Result<ParsedBaseImageFileName, FilePathError> {
+    /*
+     * Relation data files can be in one of the following directories:
+     *
+     * global/
+     *		shared relations
+     *
+     * base/<db oid>/
+     *		regular relations, default tablespace
+     *
+     * pg_tblspc/<tblspc oid>/<tblspc version>/
+     *		within a non-default tablespace (the name of the directory
+     *		depends on version)
+     *
+     * And the relation data files themselves have a filename like:
+     *
+     * <oid>.<segment number>
+     */
+    if let Some(fname) = path.strip_prefix("global/") {
+        if fname.contains("pg_control") {
+            let lsn = parse_lsn_from_filename(fname.clone())?;
+
+            return Ok(ParsedBaseImageFileName {
+                spcnode: pg_constants::GLOBALTABLESPACE_OID,
+                dbnode: 0,
+                relnode: 0,
+                forknum: pg_constants::PG_CONTROLFILE_FORKNUM,
+                segno: 0,
+                lsn,
+            });
+        }
+
+        if fname.contains("pg_filenode") {
+            let lsn = parse_lsn_from_filename(fname.clone())?;
+
+            return Ok(ParsedBaseImageFileName {
+                spcnode: pg_constants::GLOBALTABLESPACE_OID,
+                dbnode: 0,
+                relnode: 0,
+                forknum: pg_constants::PG_FILENODEMAP_FORKNUM,
+                segno: 0,
+                lsn,
+            });
+        }
+
+        let (relnode, forknum, segno, lsn) = parse_filename(fname)?;
+
+        return Ok(ParsedBaseImageFileName {
+            spcnode: pg_constants::GLOBALTABLESPACE_OID,
+            dbnode: 0,
+            relnode,
+            forknum,
+            segno,
+            lsn,
+        });
+    } else if let Some(dbpath) = path.strip_prefix("base/") {
+        let mut s = dbpath.split("/");
+        let dbnode_str = s
+            .next()
+            .ok_or_else(|| FilePathError::new("invalid relation data file name"))?;
+        let dbnode = u32::from_str_radix(dbnode_str, 10)?;
+        let fname = s
+            .next()
+            .ok_or_else(|| FilePathError::new("invalid relation data file name"))?;
+        if s.next().is_some() {
+            return Err(FilePathError::new("invalid relation data file name"));
+        };
+
+        if fname.contains("pg_filenode") {
+            let lsn = parse_lsn_from_filename(fname.clone())?;
+
+            return Ok(ParsedBaseImageFileName {
+                spcnode: pg_constants::DEFAULTTABLESPACE_OID,
+                dbnode,
+                relnode: 0,
+                forknum: pg_constants::PG_FILENODEMAP_FORKNUM,
+                segno: 0,
+                lsn,
+            });
+        }
+
+        let (relnode, forknum, segno, lsn) = parse_filename(fname)?;
+
+        return Ok(ParsedBaseImageFileName {
+            spcnode: pg_constants::DEFAULTTABLESPACE_OID,
+            dbnode,
+            relnode,
+            forknum,
+            segno,
+            lsn,
+        });
+    } else if let Some(_) = path.strip_prefix("pg_tblspc/") {
+        // TODO
+        return Err(FilePathError::new("tablespaces not supported"));
+    } else {
+        return Err(FilePathError::new("invalid relation data file name"));
+    }
+}
+
+//
+// Load a base file from S3, and insert it into the page cache
+//
+async fn slurp_base_file(
+    conf: &PageServerConf,
+    sys_id: u64,
+    bucket: Bucket,
+    s3path: String,
+    parsed: ParsedBaseImageFileName,
+) {
+    // FIXME: rust-s3 opens a new connection for each request. Should reuse
+    // the reqwest::Client object. But that requires changes to rust-s3 itself.
+    let (data, code) = bucket.get_object(s3path.clone()).await.unwrap();
+
+    trace!("got response: {} on {}", code, &s3path);
+    assert_eq!(200, code);
+
+    let mut bytes = BytesMut::from(data.as_slice()).freeze();
+
+    let pcache = page_cache::get_pagecache(conf.clone(), sys_id);
+
+    // pg_filenode.map has non-standard size - 512 bytes
+    if parsed.forknum == pg_constants::PG_FILENODEMAP_FORKNUM {
+        let b = bytes.clone();
+        controlfile::decode_filemapping(b);
+        while bytes.remaining() >= 512 {
+            let tag = page_cache::BufferTag {
+                spcnode: parsed.spcnode,
+                dbnode: parsed.dbnode,
+                relnode: parsed.relnode,
+                forknum: parsed.forknum as u8,
+                blknum: 0,
+            };
+
+            pcache.put_page_image(tag, parsed.lsn, bytes.copy_to_bytes(512));
+        }
+
+        let tag = page_cache::RelTag {
+            spcnode: parsed.spcnode,
+            dbnode: parsed.dbnode,
+            relnode: parsed.relnode,
+            forknum: parsed.forknum as u8,
+        };
+
+        pcache.relsize_inc(&tag, Some(0));
+    } else {
+        // FIXME: use constants (BLCKSZ)
+        let mut blknum: u32 = parsed.segno * (1024 * 1024 * 1024 / 8192);
+        let reltag = page_cache::RelTag {
+            spcnode: parsed.spcnode,
+            dbnode: parsed.dbnode,
+            relnode: parsed.relnode,
+            forknum: parsed.forknum as u8,
+        };
+
+        while bytes.remaining() >= 8192 {
+            let tag = page_cache::BufferTag {
+                spcnode: parsed.spcnode,
+                dbnode: parsed.dbnode,
+                relnode: parsed.relnode,
+                forknum: parsed.forknum as u8,
+                blknum: blknum,
+            };
+
+            pcache.put_page_image(tag, parsed.lsn, bytes.copy_to_bytes(8192));
+            pcache.relsize_inc(&reltag, Some(blknum));
+
+            blknum += 1;
+        }
+    }
+}

pageserver/src/rocksdb_storage.rs

@@ -1,475 +0,0 @@
-//!
-//! An implementation of the ObjectStore interface, backed by RocksDB
-//!
-use crate::object_key::*;
-use crate::object_store::ObjectStore;
-use crate::relish::*;
-use crate::PageServerConf;
-use anyhow::{bail, Result};
-use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
-use std::sync::{Arc, Mutex};
-use zenith_utils::bin_ser::BeSer;
-use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::ZTenantId;
-use zenith_utils::zid::ZTimelineId;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-struct StorageKey {
-    obj_key: ObjectKey,
-    lsn: Lsn,
-}
-
-impl StorageKey {
-    /// The first key for a given timeline
-    fn timeline_start(timeline: ZTimelineId) -> Self {
-        Self {
-            obj_key: ObjectKey {
-                timeline,
-                tag: ObjectTag::TimelineMetadataTag,
-            },
-            lsn: Lsn(0),
-        }
-    }
-}
-
-///
-/// RocksDB very inefficiently delete random record. Instead of it we have to use merge
-/// filter, which allows to throw away records at LSM merge phase.
-/// Unfortunately, it is hard (if ever possible)  to determine whether version can be removed
-/// at merge time. Version ca be removed if:
-/// 1. It is above PITR horizon (we need to get current LSN and gc_horizon from config)
-/// 2. Page is reconstructed at horizon (all WAL records above horizon are applied and can be removed)
-///
-/// So we have GC process which reconstructs pages at horizon and mark deteriorated WAL record
-/// for deletion. To mark object for deletion we can either set some flag in object itself.
-/// But it is complicated with new object value format, because RocksDB storage knows nothing about
-/// this format. Also updating whole record just to set one bit seems to be inefficient in any case.
-/// This is why we keep keys of marked for deletion versions in HashSet in memory.
-/// When LSM merge filter found key in this map, it removes it from the set preventing memory overflow.
-///
-struct GarbageCollector {
-    garbage: Mutex<HashSet<Vec<u8>>>,
-}
-
-impl GarbageCollector {
-    fn new() -> GarbageCollector {
-        GarbageCollector {
-            garbage: Mutex::new(HashSet::new()),
-        }
-    }
-
-    /// Called by GC to mark version as delete
-    fn mark_for_deletion(&self, key: &[u8]) {
-        let mut garbage = self.garbage.lock().unwrap();
-        garbage.insert(key.to_vec());
-    }
-
-    /// Called by LSM merge filter. If it finds key in the set, then
-    /// it doesn't merge it and removes from this set.
-    fn was_deleted(&self, key: &[u8]) -> bool {
-        let key = key.to_vec();
-        let mut garbage = self.garbage.lock().unwrap();
-        garbage.remove(&key)
-    }
-}
-
-pub struct RocksObjectStore {
-    _conf: &'static PageServerConf,
-
-    // RocksDB handle
-    db: rocksdb::DB,
-    gc: Arc<GarbageCollector>,
-}
-
-impl ObjectStore for RocksObjectStore {
-    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>> {
-        let val = self.db.get(StorageKey::ser(&StorageKey {
-            obj_key: key.clone(),
-            lsn,
-        })?)?;
-        if let Some(val) = val {
-            Ok(val)
-        } else {
-            bail!("could not find page {:?}", key);
-        }
-    }
-
-    fn get_next_key(&self, key: &ObjectKey) -> Result<Option<ObjectKey>> {
-        let mut iter = self.db.raw_iterator();
-        let search_key = StorageKey {
-            obj_key: key.clone(),
-            lsn: Lsn(0),
-        };
-        iter.seek(search_key.ser()?);
-        if !iter.valid() {
-            Ok(None)
-        } else {
-            let key = StorageKey::des(iter.key().unwrap())?;
-            Ok(Some(key.obj_key.clone()))
-        }
-    }
-
-    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()> {
-        self.db.put(
-            StorageKey::ser(&StorageKey {
-                obj_key: key.clone(),
-                lsn,
-            })?,
-            value,
-        )?;
-        Ok(())
-    }
-
-    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()> {
-        self.gc.mark_for_deletion(&StorageKey::ser(&StorageKey {
-            obj_key: key.clone(),
-            lsn,
-        })?);
-        Ok(())
-    }
-
-    /// Iterate through page versions of given page, starting from the given LSN.
-    /// The versions are walked in descending LSN order.
-    fn object_versions<'a>(
-        &'a self,
-        key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>> {
-        let iter = RocksObjectVersionIter::new(&self.db, key, lsn)?;
-        Ok(Box::new(iter))
-    }
-
-    /// Iterate through all timeline objects
-    fn list_objects<'a>(
-        &'a self,
-        timeline: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = ObjectTag> + 'a>> {
-        let iter = RocksObjectIter::new(&self.db, timeline, lsn)?;
-        Ok(Box::new(iter))
-    }
-
-    /// Get a list of all distinct relations in given tablespace and database.
-    ///
-    /// TODO: This implementation is very inefficient, it scans
-    /// through all entries in the given database. In practice, this
-    /// is used for CREATE DATABASE, and usually the template database is small.
-    /// But if it's not, this will be slow.
-    fn list_rels(
-        &self,
-        timelineid: ZTimelineId,
-        spcnode: u32,
-        dbnode: u32,
-        lsn: Lsn,
-    ) -> Result<HashSet<RelTag>> {
-        // FIXME: This scans everything. Very slow
-
-        let mut rels: HashSet<RelTag> = HashSet::new();
-
-        let mut search_rel_tag = RelTag {
-            spcnode,
-            dbnode,
-            relnode: 0,
-            forknum: 0u8,
-        };
-        let mut iter = self.db.raw_iterator();
-        loop {
-            let search_key = StorageKey {
-                obj_key: ObjectKey {
-                    timeline: timelineid,
-                    tag: ObjectTag::RelationMetadata(RelishTag::Relation(search_rel_tag)),
-                },
-                lsn: Lsn(0),
-            };
-            iter.seek(search_key.ser()?);
-            if !iter.valid() {
-                break;
-            }
-            let key = StorageKey::des(iter.key().unwrap())?;
-
-            if let ObjectTag::RelationMetadata(RelishTag::Relation(rel_tag)) = key.obj_key.tag {
-                if spcnode != 0 && rel_tag.spcnode != spcnode
-                    || dbnode != 0 && rel_tag.dbnode != dbnode
-                {
-                    break;
-                }
-                if key.lsn <= lsn {
-                    // visible in this snapshot
-                    rels.insert(rel_tag);
-                }
-                search_rel_tag = rel_tag;
-                // skip to next relation
-                // FIXME: What if relnode is u32::MAX ?
-                search_rel_tag.relnode += 1;
-            } else {
-                // no more relation metadata entries
-                break;
-            }
-        }
-
-        Ok(rels)
-    }
-
-    /// Get a list of all distinct NON-relations in timeline
-    /// that are visible at given lsn.
-    ///
-    /// TODO: This implementation is very inefficient, it scans
-    /// through all non-rel page versions in the system. In practice, this
-    /// is used when initializing a new compute node, and the non-rel files
-    /// are never very large nor change very frequently, so this will do for now.
-    fn list_nonrels(&self, timelineid: ZTimelineId, lsn: Lsn) -> Result<HashSet<RelishTag>> {
-        let mut rels: HashSet<RelishTag> = HashSet::new();
-
-        let search_key = StorageKey {
-            obj_key: ObjectKey {
-                timeline: timelineid,
-                tag: ObjectTag::Buffer(FIRST_NONREL_RELISH_TAG, 0),
-            },
-            lsn: Lsn(0),
-        };
-
-        let mut iter = self.db.raw_iterator();
-        iter.seek(search_key.ser()?);
-        while iter.valid() {
-            let key = StorageKey::des(iter.key().unwrap())?;
-
-            if key.obj_key.timeline != timelineid {
-                // reached end of this timeline in the store
-                break;
-            }
-
-            if let ObjectTag::Buffer(rel_tag, _blknum) = key.obj_key.tag {
-                if key.lsn <= lsn {
-                    // visible in this snapshot
-                    rels.insert(rel_tag);
-                }
-            }
-            // TODO: we could skip to next relation here like we do in list_rels(),
-            // but hopefully there are not that many SLRU segments or other non-rel
-            // entries for it to matter.
-            iter.next();
-        }
-
-        Ok(rels)
-    }
-
-    /// Iterate through versions of all objects in a timeline.
-    ///
-    /// Returns objects in increasing key-version order.
-    /// Returns all versions up to and including the specified LSN.
-    fn objects<'a>(
-        &'a self,
-        timeline: ZTimelineId,
-        lsn: Lsn,
-    ) -> Result<Box<dyn Iterator<Item = Result<(ObjectTag, Lsn, Vec<u8>)>> + 'a>> {
-        let start_key = StorageKey::timeline_start(timeline);
-        let start_key_bytes = StorageKey::ser(&start_key)?;
-        let iter = self.db.iterator(rocksdb::IteratorMode::From(
-            &start_key_bytes,
-            rocksdb::Direction::Forward,
-        ));
-
-        Ok(Box::new(RocksObjects {
-            iter,
-            timeline,
-            lsn,
-        }))
-    }
-
-    fn compact(&self) {
-        self.db.compact_range::<&[u8], &[u8]>(None, None);
-    }
-}
-
-impl RocksObjectStore {
-    /// Open a RocksDB database.
-    pub fn open(conf: &'static PageServerConf, tenantid: &ZTenantId) -> Result<RocksObjectStore> {
-        let opts = Self::get_rocksdb_opts();
-        let obj_store = Self::new(conf, opts, tenantid)?;
-        Ok(obj_store)
-    }
-
-    /// Create a new, empty RocksDB database.
-    pub fn create(conf: &'static PageServerConf, tenantid: &ZTenantId) -> Result<RocksObjectStore> {
-        let path = conf.tenant_path(&tenantid).join("rocksdb-storage");
-        std::fs::create_dir(&path)?;
-
-        let mut opts = Self::get_rocksdb_opts();
-        opts.create_if_missing(true);
-        opts.set_error_if_exists(true);
-        let obj_store = Self::new(conf, opts, tenantid)?;
-        Ok(obj_store)
-    }
-
-    fn new(
-        conf: &'static PageServerConf,
-        mut opts: rocksdb::Options,
-        tenantid: &ZTenantId,
-    ) -> Result<RocksObjectStore> {
-        let path = conf.tenant_path(&tenantid).join("rocksdb-storage");
-        let gc = Arc::new(GarbageCollector::new());
-        let gc_ref = gc.clone();
-        opts.set_compaction_filter("ttl", move |_level: u32, key: &[u8], _val: &[u8]| {
-            if gc_ref.was_deleted(key) {
-                rocksdb::compaction_filter::Decision::Remove
-            } else {
-                rocksdb::compaction_filter::Decision::Keep
-            }
-        });
-        let db = rocksdb::DB::open(&opts, &path)?;
-        let obj_store = RocksObjectStore {
-            _conf: conf,
-            db,
-            gc,
-        };
-        Ok(obj_store)
-    }
-
-    /// common options used by `open` and `create`
-    fn get_rocksdb_opts() -> rocksdb::Options {
-        let mut opts = rocksdb::Options::default();
-        opts.set_use_fsync(true);
-        opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
-        opts
-    }
-}
-
-///
-/// Iterator for `object_versions`. Returns all page versions of a given block, in
-/// reverse LSN order.
-///
-struct RocksObjectVersionIter<'a> {
-    obj_key: ObjectKey,
-    dbiter: rocksdb::DBRawIterator<'a>,
-    first_call: bool,
-}
-impl<'a> RocksObjectVersionIter<'a> {
-    fn new(
-        db: &'a rocksdb::DB,
-        obj_key: &ObjectKey,
-        lsn: Lsn,
-    ) -> Result<RocksObjectVersionIter<'a>> {
-        let key = StorageKey {
-            obj_key: obj_key.clone(),
-            lsn,
-        };
-        let mut dbiter = db.raw_iterator();
-        dbiter.seek_for_prev(StorageKey::ser(&key)?); // locate last entry
-        Ok(RocksObjectVersionIter {
-            first_call: true,
-            obj_key: obj_key.clone(),
-            dbiter,
-        })
-    }
-}
-impl<'a> Iterator for RocksObjectVersionIter<'a> {
-    type Item = (Lsn, Vec<u8>);
-
-    fn next(&mut self) -> std::option::Option<Self::Item> {
-        if self.first_call {
-            self.first_call = false;
-        } else {
-            self.dbiter.prev(); // walk backwards
-        }
-
-        if !self.dbiter.valid() {
-            return None;
-        }
-        let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
-        if key.obj_key.tag != self.obj_key.tag {
-            return None;
-        }
-        let val = self.dbiter.value().unwrap();
-        let result = val.to_vec();
-
-        Some((key.lsn, result))
-    }
-}
-
-struct RocksObjects<'r> {
-    iter: rocksdb::DBIterator<'r>,
-    timeline: ZTimelineId,
-    lsn: Lsn,
-}
-
-impl<'r> Iterator for RocksObjects<'r> {
-    // TODO consider returning Box<[u8]>
-    type Item = Result<(ObjectTag, Lsn, Vec<u8>)>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.next_result().transpose()
-    }
-}
-
-impl<'r> RocksObjects<'r> {
-    fn next_result(&mut self) -> Result<Option<(ObjectTag, Lsn, Vec<u8>)>> {
-        for (key_bytes, v) in &mut self.iter {
-            let key = StorageKey::des(&key_bytes)?;
-
-            if key.obj_key.timeline != self.timeline {
-                return Ok(None);
-            }
-
-            if key.lsn > self.lsn {
-                // TODO can speed up by seeking iterator
-                continue;
-            }
-
-            return Ok(Some((key.obj_key.tag, key.lsn, v.to_vec())));
-        }
-
-        Ok(None)
-    }
-}
-
-///
-/// Iterator for `list_objects`. Returns all objects preceeding specified LSN
-///
-struct RocksObjectIter<'a> {
-    timeline: ZTimelineId,
-    key: StorageKey,
-    lsn: Lsn,
-    dbiter: rocksdb::DBRawIterator<'a>,
-}
-impl<'a> RocksObjectIter<'a> {
-    fn new(db: &'a rocksdb::DB, timeline: ZTimelineId, lsn: Lsn) -> Result<RocksObjectIter<'a>> {
-        let key = StorageKey {
-            obj_key: ObjectKey {
-                timeline,
-                tag: ObjectTag::FirstTag,
-            },
-            lsn: Lsn(0),
-        };
-        let dbiter = db.raw_iterator();
-        Ok(RocksObjectIter {
-            key,
-            timeline,
-            lsn,
-            dbiter,
-        })
-    }
-}
-impl<'a> Iterator for RocksObjectIter<'a> {
-    type Item = ObjectTag;
-
-    fn next(&mut self) -> std::option::Option<Self::Item> {
-        loop {
-            self.dbiter.seek(StorageKey::ser(&self.key).unwrap());
-            if !self.dbiter.valid() {
-                return None;
-            }
-            let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
-            if key.obj_key.timeline != self.timeline {
-                // End of this timeline
-                return None;
-            }
-            self.key = key.clone();
-            self.key.lsn = Lsn(u64::MAX); // next seek should skip all versions
-            if key.lsn <= self.lsn {
-                // visible in this snapshot
-                return Some(key.obj_key.tag);
-            }
-        }
-    }
-}

pageserver/src/tui.rs

@@ -0,0 +1,300 @@
+use crate::tui_event::{Event, Events};
+use crate::tui_logger::TuiLogger;
+use crate::tui_logger::TuiLoggerWidget;
+
+use lazy_static::lazy_static;
+use std::sync::Arc;
+use std::{error::Error, io};
+use termion::{event::Key, input::MouseTerminal, raw::IntoRawMode, screen::AlternateScreen};
+use tui::backend::TermionBackend;
+use tui::buffer::Buffer;
+use tui::layout::{Constraint, Direction, Layout, Rect};
+use tui::style::{Color, Modifier, Style};
+use tui::text::{Span, Spans, Text};
+use tui::widgets::{Block, BorderType, Borders, Paragraph, Widget};
+use tui::Terminal;
+
+use slog;
+use slog::Drain;
+
+lazy_static! {
+    pub static ref PAGESERVICE_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
+    pub static ref WALRECEIVER_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
+    pub static ref WALREDO_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
+    pub static ref CATCHALL_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
+}
+
+pub fn init_logging() -> slog_scope::GlobalLoggerGuard {
+    let pageservice_drain =
+        slog::Filter::new(PAGESERVICE_DRAIN.as_ref(), |record: &slog::Record| {
+            if record.level().is_at_least(slog::Level::Debug)
+                && record.module().starts_with("pageserver::page_service")
+            {
+                return true;
+            }
+            return false;
+        })
+        .fuse();
+
+    let walredo_drain = slog::Filter::new(WALREDO_DRAIN.as_ref(), |record: &slog::Record| {
+        if record.level().is_at_least(slog::Level::Debug)
+            && record.module().starts_with("pageserver::walredo")
+        {
+            return true;
+        }
+        return false;
+    })
+    .fuse();
+
+    let walreceiver_drain =
+        slog::Filter::new(WALRECEIVER_DRAIN.as_ref(), |record: &slog::Record| {
+            if record.level().is_at_least(slog::Level::Debug)
+                && record.module().starts_with("pageserver::walreceiver")
+            {
+                return true;
+            }
+            return false;
+        })
+        .fuse();
+
+    let catchall_drain = slog::Filter::new(CATCHALL_DRAIN.as_ref(), |record: &slog::Record| {
+        if record.level().is_at_least(slog::Level::Info) {
+            return true;
+        }
+        if record.level().is_at_least(slog::Level::Debug)
+            && record.module().starts_with("pageserver")
+        {
+            return true;
+        }
+        return false;
+    })
+    .fuse();
+
+    let drain = pageservice_drain;
+    let drain = slog::Duplicate::new(drain, walreceiver_drain).fuse();
+    let drain = slog::Duplicate::new(drain, walredo_drain).fuse();
+    let drain = slog::Duplicate::new(drain, catchall_drain).fuse();
+    let drain = slog_async::Async::new(drain).chan_size(1000).build().fuse();
+    let drain = slog::Filter::new(drain, |record: &slog::Record| {
+        if record.level().is_at_least(slog::Level::Info) {
+            return true;
+        }
+        if record.level().is_at_least(slog::Level::Debug)
+            && record.module().starts_with("pageserver")
+        {
+            return true;
+        }
+
+        return false;
+    })
+    .fuse();
+    let logger = slog::Logger::root(drain, slog::o!());
+    return slog_scope::set_global_logger(logger);
+}
+
+pub fn ui_main<'b>() -> Result<(), Box<dyn Error>> {
+    // Terminal initialization
+    let stdout = io::stdout().into_raw_mode()?;
+    let stdout = MouseTerminal::from(stdout);
+    let stdout = AlternateScreen::from(stdout);
+    let backend = TermionBackend::new(stdout);
+    let mut terminal = Terminal::new(backend)?;
+
+    // Setup event handlers
+    let events = Events::new();
+
+    loop {
+        terminal.draw(|f| {
+            let size = f.size();
+
+            // +----------------+----------------+
+            // |                |                |
+            // |  top_top_left  | top_top_right  |
+            // |                |                |
+            // +----------------+----------------|
+            // |                |                |
+            // |  top_bot_left  | top_left_right |
+            // |                |                |
+            // +----------------+----------------+
+            // |                                 |
+            // |             bottom              |
+            // |                                 |
+            // +---------------------------------+
+            let chunks = Layout::default()
+                .direction(Direction::Vertical)
+                .constraints([Constraint::Percentage(70), Constraint::Percentage(30)].as_ref())
+                .split(size);
+            let top_chunk = chunks[0];
+            let bottom_chunk = chunks[1];
+
+            let top_chunks = Layout::default()
+                .direction(Direction::Horizontal)
+                .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
+                .split(top_chunk);
+            let top_left_chunk = top_chunks[0];
+            let top_right_chunk = top_chunks[1];
+
+            let c = Layout::default()
+                .direction(Direction::Vertical)
+                .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
+                .split(top_left_chunk);
+            let top_top_left_chunk = c[0];
+            let top_bot_left_chunk = c[1];
+
+            let c = Layout::default()
+                .direction(Direction::Vertical)
+                .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
+                .split(top_right_chunk);
+            let top_top_right_chunk = c[0];
+            let top_bot_right_chunk = c[1];
+
+            f.render_widget(
+                LogWidget::new(PAGESERVICE_DRAIN.as_ref(), "Page Service"),
+                top_top_left_chunk,
+            );
+
+            f.render_widget(
+                LogWidget::new(WALREDO_DRAIN.as_ref(), "WAL Redo"),
+                top_bot_left_chunk,
+            );
+
+            f.render_widget(
+                LogWidget::new(WALRECEIVER_DRAIN.as_ref(), "WAL Receiver"),
+                top_top_right_chunk,
+            );
+
+            f.render_widget(MetricsWidget {}, top_bot_right_chunk);
+
+            f.render_widget(
+                LogWidget::new(CATCHALL_DRAIN.as_ref(), "All Log").show_module(true),
+                bottom_chunk,
+            );
+        })?;
+
+        // If ther user presses 'q', quit.
+        if let Event::Input(key) = events.next()? {
+            match key {
+                Key::Char('q') => {
+                    break;
+                }
+                _ => (),
+            }
+        }
+    }
+
+    terminal.show_cursor().unwrap();
+    terminal.clear().unwrap();
+
+    Ok(())
+}
+
+struct LogWidget<'a> {
+    logger: &'a TuiLogger,
+    title: &'a str,
+    show_module: bool,
+}
+
+impl<'a> LogWidget<'a> {
+    fn new(logger: &'a TuiLogger, title: &'a str) -> LogWidget<'a> {
+        LogWidget {
+            logger,
+            title,
+            show_module: false,
+        }
+    }
+
+    fn show_module(mut self, b: bool) -> LogWidget<'a> {
+        self.show_module = b;
+        self
+    }
+}
+
+impl<'a> Widget for LogWidget<'a> {
+    fn render(self, area: Rect, buf: &mut Buffer) {
+        let w = TuiLoggerWidget::default(self.logger)
+            .block(
+                Block::default()
+                    .borders(Borders::ALL)
+                    .title(self.title)
+                    .border_type(BorderType::Rounded),
+            )
+            .show_module(true)
+            .style_error(Style::default().fg(Color::Red))
+            .style_warn(Style::default().fg(Color::Yellow))
+            .style_info(Style::default().fg(Color::Green));
+        w.render(area, buf);
+    }
+}
+
+// Render a widget to show some metrics
+struct MetricsWidget {}
+
+fn get_metric_u64<'a>(title: &'a str, value: u64) -> Spans<'a> {
+    Spans::from(vec![
+        Span::styled(format!("{:<20}", title), Style::default()),
+        Span::raw(": "),
+        Span::styled(
+            value.to_string(),
+            Style::default().add_modifier(Modifier::BOLD),
+        ),
+    ])
+}
+
+fn get_metric_str<'a>(title: &'a str, value: &'a str) -> Spans<'a> {
+    Spans::from(vec![
+        Span::styled(format!("{:<20}", title), Style::default()),
+        Span::raw(": "),
+        Span::styled(value, Style::default().add_modifier(Modifier::BOLD)),
+    ])
+}
+
+// FIXME: We really should define a datatype for LSNs, with Display trait and
+// helper functions. There's one in tokio-postgres, but I don't think we want
+// to rely on that.
+fn format_lsn(lsn: u64) -> String {
+    return format!("{:X}/{:X}", lsn >> 32, lsn & 0xffff_ffff);
+}
+
+impl tui::widgets::Widget for MetricsWidget {
+    fn render(self, area: Rect, buf: &mut Buffer) {
+        let block = Block::default()
+            .borders(Borders::ALL)
+            .title("Page Cache Metrics")
+            .border_type(BorderType::Rounded);
+        let inner_area = block.inner(area);
+
+        block.render(area, buf);
+
+        let mut lines: Vec<Spans> = Vec::new();
+
+        let page_cache_stats = crate::page_cache::get_stats();
+        let lsnrange = format!(
+            "{} - {}",
+            format_lsn(page_cache_stats.first_valid_lsn),
+            format_lsn(page_cache_stats.last_valid_lsn)
+        );
+        let last_valid_recordlsn_str = format_lsn(page_cache_stats.last_record_lsn);
+        lines.push(get_metric_str("Valid LSN range", &lsnrange));
+        lines.push(get_metric_str("Last record LSN", &last_valid_recordlsn_str));
+        lines.push(get_metric_u64(
+            "# of cache entries",
+            page_cache_stats.num_entries,
+        ));
+        lines.push(get_metric_u64(
+            "# of page images",
+            page_cache_stats.num_page_images,
+        ));
+        lines.push(get_metric_u64(
+            "# of WAL records",
+            page_cache_stats.num_wal_records,
+        ));
+        lines.push(get_metric_u64(
+            "# of GetPage@LSN calls",
+            page_cache_stats.num_getpage_requests,
+        ));
+
+        let text = Text::from(lines);
+
+        Paragraph::new(text).render(inner_area, buf);
+    }
+}

pageserver/src/tui_event.rs

@@ -0,0 +1,97 @@
+use std::io;
+use std::sync::mpsc;
+use std::sync::{
+    atomic::{AtomicBool, Ordering},
+    Arc,
+};
+use std::thread;
+use std::time::Duration;
+
+use termion::event::Key;
+use termion::input::TermRead;
+
+#[allow(dead_code)]
+pub enum Event<I> {
+    Input(I),
+    Tick,
+}
+
+/// A small event handler that wrap termion input and tick events. Each event
+/// type is handled in its own thread and returned to a common `Receiver`
+#[allow(dead_code)]
+pub struct Events {
+    rx: mpsc::Receiver<Event<Key>>,
+    input_handle: thread::JoinHandle<()>,
+    ignore_exit_key: Arc<AtomicBool>,
+    tick_handle: thread::JoinHandle<()>,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct Config {
+    pub exit_key: Key,
+    pub tick_rate: Duration,
+}
+
+impl Default for Config {
+    fn default() -> Config {
+        Config {
+            exit_key: Key::Char('q'),
+            tick_rate: Duration::from_millis(250),
+        }
+    }
+}
+
+impl Events {
+    pub fn new() -> Events {
+        Events::with_config(Config::default())
+    }
+
+    pub fn with_config(config: Config) -> Events {
+        let (tx, rx) = mpsc::channel();
+        let ignore_exit_key = Arc::new(AtomicBool::new(false));
+        let input_handle = {
+            let tx = tx.clone();
+            let ignore_exit_key = ignore_exit_key.clone();
+            thread::spawn(move || {
+                let stdin = io::stdin();
+                for evt in stdin.keys() {
+                    if let Ok(key) = evt {
+                        if let Err(err) = tx.send(Event::Input(key)) {
+                            eprintln!("{}", err);
+                            return;
+                        }
+                        if !ignore_exit_key.load(Ordering::Relaxed) && key == config.exit_key {
+                            return;
+                        }
+                    }
+                }
+            })
+        };
+        let tick_handle = {
+            thread::spawn(move || loop {
+                if tx.send(Event::Tick).is_err() {
+                    break;
+                }
+                thread::sleep(config.tick_rate);
+            })
+        };
+        Events {
+            rx,
+            ignore_exit_key,
+            input_handle,
+            tick_handle,
+        }
+    }
+
+    pub fn next(&self) -> Result<Event<Key>, mpsc::RecvError> {
+        self.rx.recv()
+    }
+
+    pub fn disable_exit_key(&mut self) {
+        self.ignore_exit_key.store(true, Ordering::Relaxed);
+    }
+
+    pub fn enable_exit_key(&mut self) {
+        self.ignore_exit_key.store(false, Ordering::Relaxed);
+    }
+}

pageserver/src/tui_logger.rs

@@ -0,0 +1,200 @@
+//
+// A TUI Widget that displays log entries
+//
+// This is heavily inspired by gin66's tui_logger crate at https://github.com/gin66/tui-logger,
+// but I wrote this based on the 'slog' module, which simplified things a lot. tui-logger also
+// implemented the slog Drain trait, but it had a model of one global buffer for the records.
+// With this implementation, each TuiLogger is a separate ring buffer and separate slog Drain.
+// Also, I didn't do any of the "hot log" stuff that gin66's implementation had, you can use an
+// AsyncDrain to buffer and handle overflow if desired.
+//
+use chrono::offset::Local;
+use chrono::DateTime;
+use slog;
+use slog::{Drain, Level, OwnedKVList, Record};
+use slog_async::AsyncRecord;
+use std::collections::VecDeque;
+use std::sync::Mutex;
+use std::time::SystemTime;
+use tui::buffer::Buffer;
+use tui::layout::Rect;
+use tui::style::{Modifier, Style};
+use tui::text::{Span, Spans};
+use tui::widgets::{Block, Paragraph, Widget, Wrap};
+
+// Size of the log ring buffer, in # of records
+static BUFFER_SIZE: usize = 1000;
+
+pub struct TuiLogger {
+    events: Mutex<VecDeque<(SystemTime, AsyncRecord)>>,
+}
+
+impl<'a> Default for TuiLogger {
+    fn default() -> TuiLogger {
+        TuiLogger {
+            events: Mutex::new(VecDeque::with_capacity(BUFFER_SIZE)),
+        }
+    }
+}
+
+impl Drain for TuiLogger {
+    type Ok = ();
+    type Err = slog::Error;
+
+    fn log(&self, record: &Record, values: &OwnedKVList) -> Result<Self::Ok, Self::Err> {
+        let mut events = self.events.lock().unwrap();
+
+        let now = SystemTime::now();
+        let asyncrec = AsyncRecord::from(record, values);
+        events.push_front((now, asyncrec));
+
+        if events.len() > BUFFER_SIZE {
+            events.pop_back();
+        }
+
+        return Ok(());
+    }
+}
+
+// TuiLoggerWidget renders a TuiLogger ring buffer
+pub struct TuiLoggerWidget<'b> {
+    block: Option<Block<'b>>,
+    /// Base style of the widget
+    style: Style,
+    /// Level based style
+    style_error: Option<Style>,
+    style_warn: Option<Style>,
+    style_debug: Option<Style>,
+    style_trace: Option<Style>,
+    style_info: Option<Style>,
+    show_module: bool,
+    logger: &'b TuiLogger,
+}
+impl<'b> TuiLoggerWidget<'b> {
+    pub fn default(logger: &'b TuiLogger) -> TuiLoggerWidget<'b> {
+        TuiLoggerWidget {
+            block: None,
+            style: Default::default(),
+            style_error: None,
+            style_warn: None,
+            style_debug: None,
+            style_trace: None,
+            style_info: None,
+            show_module: true,
+            logger: logger,
+        }
+    }
+}
+impl<'b> TuiLoggerWidget<'b> {
+    pub fn block(mut self, block: Block<'b>) -> TuiLoggerWidget<'b> {
+        self.block = Some(block);
+        self
+    }
+    #[allow(unused)]
+    pub fn style(mut self, style: Style) -> TuiLoggerWidget<'b> {
+        self.style = style;
+        self
+    }
+    pub fn style_error(mut self, style: Style) -> TuiLoggerWidget<'b> {
+        self.style_error = Some(style);
+        self
+    }
+    pub fn style_warn(mut self, style: Style) -> TuiLoggerWidget<'b> {
+        self.style_warn = Some(style);
+        self
+    }
+    pub fn style_info(mut self, style: Style) -> TuiLoggerWidget<'b> {
+        self.style_info = Some(style);
+        self
+    }
+    #[allow(unused)]
+    pub fn style_trace(mut self, style: Style) -> TuiLoggerWidget<'b> {
+        self.style_trace = Some(style);
+        self
+    }
+    #[allow(unused)]
+    pub fn style_debug(mut self, style: Style) -> TuiLoggerWidget<'b> {
+        self.style_debug = Some(style);
+        self
+    }
+
+    pub fn show_module(mut self, b: bool) -> TuiLoggerWidget<'b> {
+        self.show_module = b;
+        self
+    }
+}
+impl<'b> Widget for TuiLoggerWidget<'b> {
+    fn render(mut self, area: Rect, buf: &mut Buffer) {
+        buf.set_style(area, self.style);
+        let list_area = match self.block.take() {
+            Some(b) => {
+                let inner_area = b.inner(area);
+                b.render(area, buf);
+                inner_area
+            }
+            None => area,
+        };
+        if list_area.width == 0 || list_area.height == 0 {
+            return;
+        }
+
+        let la_height = list_area.height as usize;
+
+        //
+        // Iterate through the records in the buffer. The records are
+        // pushed to the front, so the newest records come first.
+        //
+        let mut lines: Vec<Spans> = Vec::new();
+
+        let style_msg = Style::default().add_modifier(Modifier::BOLD);
+        {
+            let events = self.logger.events.lock().unwrap();
+
+            for evt in events.iter() {
+                let (timestamp, rec) = evt;
+
+                rec.as_record_values(|rec, _kwlist| {
+                    let mut line: Vec<Span> = Vec::new();
+
+                    let datetime: DateTime<Local> = timestamp.clone().into();
+                    let ts = format!("{}", datetime.format("%H:%M:%S%.3f "));
+                    line.push(Span::raw(ts));
+
+                    let (lvl_style, txt, with_loc) = match rec.level() {
+                        Level::Critical => (self.style_error, "CRIT ", true),
+                        Level::Error => (self.style_error, "ERROR", true),
+                        Level::Warning => (self.style_warn, "WARN ", true),
+                        Level::Info => (self.style_info, "INFO ", false),
+                        Level::Debug => (self.style_debug, "DEBUG", true),
+                        Level::Trace => (self.style_trace, "TRACE", true),
+                    };
+                    line.push(Span::styled(txt, lvl_style.unwrap_or(Style::default())));
+
+                    if self.show_module {
+                        line.push(Span::raw(" "));
+                        line.push(Span::raw(rec.module()));
+                    }
+                    if with_loc {
+                        let loc = format!(" {}:{}", rec.file(), rec.line());
+                        line.push(Span::raw(loc));
+                    }
+                    let msg = format!(" {}", rec.msg());
+                    line.push(Span::styled(msg, style_msg));
+
+                    lines.push(Spans::from(line));
+                });
+                if lines.len() == la_height {
+                    break;
+                }
+            }
+        }
+
+        lines.reverse();
+
+        let text = tui::text::Text::from(lines);
+
+        Paragraph::new(text)
+            .wrap(Wrap { trim: true })
+            .render(list_area, buf);
+    }
+}

pageserver/src/walreceiver.rs

@@ -1,413 +1,194 @@
-//!
-//! WAL receiver connects to the WAL safekeeper service,
-//! streams WAL, decodes records and saves them in page cache.
-//!
-//! We keep one WAL receiver active per timeline.
+//
+// WAL receiver
+//
+// The WAL receiver connects to the WAL safekeeper service, and streams WAL.
+// For each WAL record, it decodes the record to figure out which data blocks
+// the record affects, and adds the records to the page cache.
+//
+use log::*;
+
+use tokio::runtime;
+use tokio::time::{sleep, Duration};
+use tokio_stream::StreamExt;
 
 use crate::page_cache;
-use crate::relish::*;
-use crate::restore_local_repo;
-use crate::waldecoder::*;
-use crate::{PageServerConf, RepositoryFormat};
-use anyhow::{Error, Result};
-use lazy_static::lazy_static;
-use log::*;
-use postgres::fallible_iterator::FallibleIterator;
-use postgres::replication::ReplicationIter;
-use postgres::{Client, NoTls, SimpleQueryMessage, SimpleQueryRow};
-use postgres_ffi::xlog_utils::*;
-use postgres_ffi::*;
+use crate::page_cache::BufferTag;
+use crate::waldecoder::WalStreamDecoder;
+use crate::PageServerConf;
+
 use postgres_protocol::message::backend::ReplicationMessage;
-use postgres_types::PgLsn;
-use std::cmp::{max, min};
-use std::collections::HashMap;
-use std::fs;
-use std::str::FromStr;
-use std::sync::Mutex;
-use std::thread;
-use std::thread::sleep;
-use std::time::{Duration, SystemTime};
-use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::ZTenantId;
-use zenith_utils::zid::ZTimelineId;
-
-//
-// We keep one WAL Receiver active per timeline.
-//
-struct WalReceiverEntry {
-    wal_producer_connstr: String,
-}
-
-lazy_static! {
-    static ref WAL_RECEIVERS: Mutex<HashMap<ZTimelineId, WalReceiverEntry>> =
-        Mutex::new(HashMap::new());
-}
-
-// Launch a new WAL receiver, or tell one that's running about change in connection string
-pub fn launch_wal_receiver(
-    conf: &'static PageServerConf,
-    timelineid: ZTimelineId,
-    wal_producer_connstr: &str,
-    tenantid: ZTenantId,
-) {
-    let mut receivers = WAL_RECEIVERS.lock().unwrap();
-
-    match receivers.get_mut(&timelineid) {
-        Some(receiver) => {
-            receiver.wal_producer_connstr = wal_producer_connstr.into();
-        }
-        None => {
-            let receiver = WalReceiverEntry {
-                wal_producer_connstr: wal_producer_connstr.into(),
-            };
-            receivers.insert(timelineid, receiver);
-
-            // Also launch a new thread to handle this connection
-            let _walreceiver_thread = thread::Builder::new()
-                .name("WAL receiver thread".into())
-                .spawn(move || {
-                    thread_main(conf, timelineid, &tenantid);
-                })
-                .unwrap();
-        }
-    };
-}
-
-// Look up current WAL producer connection string in the hash table
-fn get_wal_producer_connstr(timelineid: ZTimelineId) -> String {
-    let receivers = WAL_RECEIVERS.lock().unwrap();
-
-    receivers
-        .get(&timelineid)
-        .unwrap()
-        .wal_producer_connstr
-        .clone()
-}
+use tokio_postgres::{connect_replication, Error, NoTls, ReplicationMode};
 
 //
 // This is the entry point for the WAL receiver thread.
 //
-fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId, tenantid: &ZTenantId) {
-    info!(
-        "WAL receiver thread started for timeline : '{}'",
-        timelineid
-    );
+pub fn thread_main(conf: PageServerConf, wal_producer_connstr: &String) {
+    info!("WAL receiver thread started: '{}'", wal_producer_connstr);
 
-    //
-    // Make a connection to the WAL safekeeper, or directly to the primary PostgreSQL server,
-    // and start streaming WAL from it. If the connection is lost, keep retrying.
-    //
-    loop {
-        // Look up the current WAL producer address
-        let wal_producer_connstr = get_wal_producer_connstr(timelineid);
+    let runtime = runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .unwrap();
 
-        let res = walreceiver_main(conf, timelineid, &wal_producer_connstr, tenantid);
+    runtime.block_on(async {
+        loop {
+            let _res = walreceiver_main(conf.clone(), wal_producer_connstr).await;
 
-        if let Err(e) = res {
+            // TODO: print/log the error
             info!(
-                "WAL streaming connection failed ({}), retrying in 1 second",
-                e
+                "WAL streaming connection failed, retrying in 1 second...: {:?}",
+                _res
             );
-            sleep(Duration::from_secs(1));
+            sleep(Duration::from_secs(1)).await;
         }
-    }
+    });
 }
 
-fn walreceiver_main(
-    conf: &PageServerConf,
-    timelineid: ZTimelineId,
-    wal_producer_connstr: &str,
-    tenantid: &ZTenantId,
+async fn walreceiver_main(
+    conf: PageServerConf,
+    wal_producer_connstr: &String,
 ) -> Result<(), Error> {
     // Connect to the database in replication mode.
-    info!("connecting to {:?}", wal_producer_connstr);
-    let connect_cfg = format!(
-        "{} application_name=pageserver replication=true",
-        wal_producer_connstr
-    );
+    debug!("connecting to {}...", wal_producer_connstr);
+    let (mut rclient, connection) = connect_replication(
+        wal_producer_connstr.as_str(),
+        NoTls,
+        ReplicationMode::Physical,
+    )
+    .await?;
+    debug!("connected!");
 
-    let mut rclient = Client::connect(&connect_cfg, NoTls)?;
-    info!("connected!");
+    // The connection object performs the actual communication with the database,
+    // so spawn it off to run on its own.
+    tokio::spawn(async move {
+        if let Err(e) = connection.await {
+            error!("connection error: {}", e);
+        }
+    });
 
-    // Immediately increment the gauge, then create a job to decrement it on thread exit.
-    // One of the pros of `defer!` is that this will *most probably*
-    // get called, even in presence of panics.
-    let gauge = crate::LIVE_CONNECTIONS_COUNT.with_label_values(&["wal_receiver"]);
-    gauge.inc();
-    scopeguard::defer! {
-        gauge.dec();
-    }
-
-    let identify = identify_system(&mut rclient)?;
-    info!("{:?}", identify);
-    let end_of_wal = Lsn::from(u64::from(identify.xlogpos));
+    let identify_system = rclient.identify_system().await?;
+    let end_of_wal = u64::from(identify_system.xlogpos());
     let mut caught_up = false;
 
-    let repository = page_cache::get_repository_for_tenant(tenantid)?;
-    let timeline = repository.get_timeline(timelineid).unwrap();
+    let sysid: u64 = identify_system.systemid().parse().unwrap();
+    let pcache = page_cache::get_pagecache(conf, sysid);
 
     //
     // Start streaming the WAL, from where we left off previously.
     //
-    // If we had previously received WAL up to some point in the middle of a WAL record, we
-    // better start from the end of last full WAL record, not in the middle of one. Hence,
-    // use 'last_record_lsn' rather than 'last_valid_lsn' here.
-    let mut last_rec_lsn = timeline.get_last_record_lsn();
-    let mut startpoint = last_rec_lsn;
-
-    if startpoint == Lsn(0) {
-        error!("No previous WAL position");
+    let mut startpoint = pcache.get_last_valid_lsn();
+    if startpoint == 0 {
+        // If we start here with identify_system.xlogpos() we will have race condition with
+        // postgres start: insert into postgres may request page that was modified with lsn
+        // smaller than identify_system.xlogpos().
+        //
+        // Current procedure for starting postgres will anyway be changed to something
+        // different like having 'initdb' method on a pageserver (or importing some shared
+        // empty database snapshot), so for now I just put start of first segment which
+        // seems to be a valid record.
+        pcache.init_valid_lsn(0x_1_000_000_u64);
+        startpoint = u64::from(0x_1_000_000_u64);
+    } else {
+        // There might be some padding after the last full record, skip it.
+        //
+        // FIXME: It probably would be better to always start streaming from the beginning
+        // of the page, or the segment, so that we could check the page/segment headers
+        // too. Just for the sake of paranoia.
+        if startpoint % 8 != 0 {
+            startpoint += 8 - (startpoint % 8);
+        }
     }
-
-    // There might be some padding after the last full record, skip it.
-    //
-    // FIXME: It probably would be better to always start streaming from the beginning
-    // of the page, or the segment, so that we could check the page/segment headers
-    // too. Just for the sake of paranoia.
-    startpoint += startpoint.calc_padding(8u32);
-
     debug!(
-        "last_record_lsn {} starting replication from {} for timeline {}, server is at {}...",
-        last_rec_lsn, startpoint, timelineid, end_of_wal
+        "starting replication from {:X}/{:X}, server is at {:X}/{:X}...",
+        (startpoint >> 32),
+        (startpoint & 0xffffffff),
+        (end_of_wal >> 32),
+        (end_of_wal & 0xffffffff)
     );
+    let startpoint = tokio_postgres::types::Lsn::from(startpoint);
+    let mut physical_stream = rclient
+        .start_physical_replication(None, startpoint, None)
+        .await?;
+    let mut waldecoder = WalStreamDecoder::new(u64::from(startpoint));
 
-    let query = format!("START_REPLICATION PHYSICAL {}", startpoint);
-
-    let copy_stream = rclient.copy_both_simple(&query)?;
-    let mut physical_stream = ReplicationIter::new(copy_stream);
-
-    let mut waldecoder = WalStreamDecoder::new(startpoint, true);
-
-    let checkpoint_bytes = timeline.get_page_at_lsn_nowait(RelishTag::Checkpoint, 0, startpoint)?;
-    let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
-    trace!("CheckPoint.nextXid = {}", checkpoint.nextXid.value);
-
-    while let Some(replication_message) = physical_stream.next()? {
-        let status_update = match replication_message {
+    while let Some(replication_message) = physical_stream.next().await {
+        match replication_message? {
             ReplicationMessage::XLogData(xlog_data) => {
                 // Pass the WAL data to the decoder, and see if we can decode
                 // more records as a result.
                 let data = xlog_data.data();
-                let startlsn = Lsn::from(xlog_data.wal_start());
+                let startlsn = xlog_data.wal_start();
                 let endlsn = startlsn + data.len() as u64;
-                let prev_last_rec_lsn = last_rec_lsn;
 
-                trace!("received XLogData between {} and {}", startlsn, endlsn);
+                trace!(
+                    "received XLogData between {:X}/{:X} and {:X}/{:X}",
+                    (startlsn >> 32),
+                    (startlsn & 0xffffffff),
+                    (endlsn >> 32),
+                    (endlsn & 0xffffffff)
+                );
+
                 waldecoder.feed_bytes(data);
 
                 loop {
-                    match waldecoder.poll_decode() {
-                        Ok(Some((lsn, recdata))) => {
-                            // Save old checkpoint value to compare with it after decoding WAL record
-                            let old_checkpoint_bytes = checkpoint.encode();
-                            let decoded = decode_wal_record(recdata.clone());
-                            restore_local_repo::save_decoded_record(
-                                &mut checkpoint,
-                                &*timeline,
-                                &decoded,
-                                recdata,
-                                lsn,
-                            )?;
-                            last_rec_lsn = lsn;
+                    if let Some((lsn, recdata)) = waldecoder.poll_decode() {
+                        let decoded =
+                            crate::waldecoder::decode_wal_record(startlsn, recdata.clone());
 
-                            let new_checkpoint_bytes = checkpoint.encode();
-                            // Check if checkpoint data was updated by save_decoded_record
-                            if new_checkpoint_bytes != old_checkpoint_bytes {
-                                timeline.put_page_image(
-                                    RelishTag::Checkpoint,
-                                    0,
-                                    lsn,
-                                    new_checkpoint_bytes,
-                                    false,
-                                )?;
-                            }
-                        }
-                        Ok(None) => {
-                            trace!(
-                                "End of replication stream {}..{} at {}",
-                                startlsn,
-                                endlsn,
-                                last_rec_lsn
-                            );
-                            break;
-                        }
-                        Err(e) => {
-                            info!("Decode error {}", e);
-                            return Err(e.into());
+                        // Put the WAL record to the page cache. We make a separate copy of
+                        // it for every block it modifies. (The actual WAL record is kept in
+                        // a Bytes, which uses a reference counter for the underlying buffer,
+                        // so having multiple copies of it doesn't cost that much)
+                        for blk in decoded.blocks.iter() {
+                            let tag = BufferTag {
+                                spcnode: blk.rnode_spcnode,
+                                dbnode: blk.rnode_dbnode,
+                                relnode: blk.rnode_relnode,
+                                forknum: blk.forknum as u8,
+                                blknum: blk.blkno,
+                            };
+
+                            let rec = page_cache::WALRecord {
+                                lsn: lsn,
+                                will_init: blk.will_init || blk.apply_image,
+                                rec: recdata.clone(),
+                            };
+
+                            pcache.put_wal_record(tag, rec);
                         }
+
+                        // Now that this record has been handled, let the page cache know that
+                        // it is up-to-date to this LSN
+                        pcache.advance_last_valid_lsn(lsn);
+                    } else {
+                        break;
                     }
                 }
+
                 // Update the last_valid LSN value in the page cache one more time. We updated
                 // it in the loop above, between each WAL record, but we might have received
                 // a partial record after the last completed record. Our page cache's value
                 // better reflect that, because GetPage@LSN requests might also point in the
                 // middle of a record, if the request LSN was taken from the server's current
                 // flush ptr.
-                timeline.advance_last_valid_lsn(endlsn);
-
-                // Somewhat arbitrarily, if we have at least 10 complete wal segments (16 MB each),
-                // "checkpoint" the repository to flush all the changes from WAL we've processed
-                // so far to disk. After this, we don't need the original WAL anymore, and it
-                // can be removed. This is probably too aggressive for production, but it's useful
-                // to expose bugs now.
-                //
-                // TODO: We don't actually dare to remove the WAL. It's useful for debugging,
-                // and we might it for logical decoding other things in the future. Although
-                // we should also be able to fetch it back from the WAL safekeepers or S3 if
-                // needed.
-                if prev_last_rec_lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE)
-                    != last_rec_lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE)
-                {
-                    info!("switched segment {} to {}", prev_last_rec_lsn, last_rec_lsn);
-                    let (oldest_segno, newest_segno) = find_wal_file_range(
-                        conf,
-                        &timelineid,
-                        pg_constants::WAL_SEGMENT_SIZE,
-                        last_rec_lsn,
-                        tenantid,
-                    )?;
-
-                    if newest_segno - oldest_segno >= 10 {
-                        // FIXME: The layered repository performs checkpointing in a separate thread, so this
-                        // isn't needed anymore. Remove 'checkpoint' from the Timeline trait altogether?
-                        if conf.repository_format == RepositoryFormat::RocksDb {
-                            timeline.checkpoint()?;
-                        }
-
-                        // TODO: This is where we could remove WAL older than last_rec_lsn.
-                        //remove_wal_files(timelineid, pg_constants::WAL_SEGMENT_SIZE, last_rec_lsn)?;
-                    }
-                }
+                pcache.advance_last_valid_lsn(endlsn);
 
                 if !caught_up && endlsn >= end_of_wal {
-                    info!("caught up at LSN {}", endlsn);
+                    info!(
+                        "caught up at LSN {:X}/{:X}",
+                        (endlsn >> 32),
+                        (endlsn & 0xffffffff)
+                    );
                     caught_up = true;
                 }
-
-                Some(endlsn)
             }
 
-            ReplicationMessage::PrimaryKeepAlive(keepalive) => {
-                let wal_end = keepalive.wal_end();
-                let timestamp = keepalive.timestamp();
-                let reply_requested = keepalive.reply() != 0;
-
-                trace!(
-                    "received PrimaryKeepAlive(wal_end: {}, timestamp: {:?} reply: {})",
-                    wal_end,
-                    timestamp,
-                    reply_requested,
-                );
-
-                if reply_requested {
-                    Some(timeline.get_last_valid_lsn())
-                } else {
-                    None
-                }
+            ReplicationMessage::PrimaryKeepAlive(_keepalive) => {
+                trace!("received PrimaryKeepAlive");
+                // FIXME: Reply, or the connection will time out
             }
-
-            _ => None,
-        };
-
-        if let Some(last_lsn) = status_update {
-            // TODO: More thought should go into what values are sent here.
-            let last_lsn = PgLsn::from(u64::from(last_lsn));
-            let write_lsn = last_lsn;
-            let flush_lsn = last_lsn;
-            let apply_lsn = PgLsn::from(0);
-            let ts = SystemTime::now();
-            const NO_REPLY: u8 = 0;
-
-            physical_stream.standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY)?;
+            _ => (),
         }
     }
-    Ok(())
-}
-
-fn find_wal_file_range(
-    conf: &PageServerConf,
-    timeline: &ZTimelineId,
-    wal_seg_size: usize,
-    written_upto: Lsn,
-    tenant: &ZTenantId,
-) -> Result<(u64, u64)> {
-    let written_upto_segno = written_upto.segment_number(wal_seg_size);
-
-    let mut oldest_segno = written_upto_segno;
-    let mut newest_segno = written_upto_segno;
-    // Scan the wal directory, and count how many WAL filed we could remove
-    let wal_dir = conf.wal_dir_path(timeline, tenant);
-    for entry in fs::read_dir(wal_dir)? {
-        let entry = entry?;
-        let path = entry.path();
-
-        if path.is_dir() {
-            continue;
-        }
-
-        let filename = path.file_name().unwrap().to_str().unwrap();
-
-        if IsXLogFileName(filename) {
-            let (segno, _tli) = XLogFromFileName(filename, wal_seg_size);
-
-            if segno > written_upto_segno {
-                // that's strange.
-                warn!("there is a WAL file from future at {}", path.display());
-                continue;
-            }
-
-            oldest_segno = min(oldest_segno, segno);
-            newest_segno = max(newest_segno, segno);
-        }
-    }
-    // FIXME: would be good to assert that there are no gaps in the WAL files
-
-    Ok((oldest_segno, newest_segno))
-}
-
-/// Data returned from the postgres `IDENTIFY_SYSTEM` command
-///
-/// See the [postgres docs] for more details.
-///
-/// [postgres docs]: https://www.postgresql.org/docs/current/protocol-replication.html
-#[derive(Debug)]
-pub struct IdentifySystem {
-    systemid: u64,
-    timeline: u32,
-    xlogpos: PgLsn,
-    dbname: Option<String>,
-}
-
-/// There was a problem parsing the response to
-/// a postgres IDENTIFY_SYSTEM command.
-#[derive(Debug, thiserror::Error)]
-#[error("IDENTIFY_SYSTEM parse error")]
-pub struct IdentifyError;
-
-/// Run the postgres `IDENTIFY_SYSTEM` command
-pub fn identify_system(client: &mut Client) -> Result<IdentifySystem, Error> {
-    let query_str = "IDENTIFY_SYSTEM";
-    let response = client.simple_query(query_str)?;
-
-    // get(N) from row, then parse it as some destination type.
-    fn get_parse<T>(row: &SimpleQueryRow, idx: usize) -> Result<T, IdentifyError>
-    where
-        T: FromStr,
-    {
-        let val = row.get(idx).ok_or(IdentifyError)?;
-        val.parse::<T>().or(Err(IdentifyError))
-    }
-
-    // extract the row contents into an IdentifySystem struct.
-    // written as a closure so I can use ? for Option here.
-    if let Some(SimpleQueryMessage::Row(first_row)) = response.get(0) {
-        Ok(IdentifySystem {
-            systemid: get_parse(first_row, 0)?,
-            timeline: get_parse(first_row, 1)?,
-            xlogpos: get_parse(first_row, 2)?,
-            dbname: get_parse(first_row, 3).ok(),
-        })
-    } else {
-        Err(IdentifyError.into())
-    }
+    return Ok(());
 }

pageserver/src/walredo.rs

@@ -1,496 +1,162 @@
-//!
-//! WAL redo. This service runs PostgreSQL in a special wal_redo mode
-//! to apply given WAL records over an old page image and return new
-//! page image.
-//!
-//! We rely on Postgres to perform WAL redo for us. We launch a
-//! postgres process in special "wal redo" mode that's similar to
-//! single-user mode. We then pass the previous page image, if any,
-//! and all the WAL records we want to apply, to the postgres
-//! process. Then we get the page image back. Communication with the
-//! postgres process happens via stdin/stdout
-//!
-//! See src/backend/tcop/zenith_wal_redo.c for the other side of
-//! this communication.
-//!
-//! The Postgres process is assumed to be secure against malicious WAL
-//! records. It achieves it by dropping privileges before replaying
-//! any WAL records, so that even if an attacker hijacks the Postgres
-//! process, he cannot escape out of it.
-//!
-use byteorder::{ByteOrder, LittleEndian};
-use bytes::{Buf, BufMut, Bytes, BytesMut};
-use lazy_static::lazy_static;
+//
+// WAL redo
+//
+// We rely on Postgres to perform WAL redo for us. We launch a
+// postgres process in special "wal redo" mode that's similar to
+// single-user mode. We then pass the the previous page image, if any,
+// and all the WAL records we want to apply, to the postgress
+// process. Then we get the page image back. Communication with the
+// postgres process happens via stdin/stdout
+//
+// See src/backend/tcop/zenith_wal_redo.c for the other side of
+// this communication.
+//
+// TODO: Even though the postgres code runs in a separate process,
+// it's not a secure sandbox.
+//
 use log::*;
-use serde::{Deserialize, Serialize};
+use std::assert;
 use std::cell::RefCell;
 use std::fs;
-use std::fs::OpenOptions;
-use std::io::prelude::*;
 use std::io::Error;
-use std::path::PathBuf;
-use std::process::Stdio;
-use std::sync::Mutex;
+use std::sync::Arc;
 use std::time::Duration;
 use std::time::Instant;
+use std::{path::PathBuf, process::Stdio};
 use tokio::io::AsyncBufReadExt;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
-use tokio::process::{ChildStdin, ChildStdout, Command};
+use tokio::process::{Child, ChildStdin, ChildStdout, Command};
+use tokio::runtime::Runtime;
 use tokio::time::timeout;
-use zenith_metrics::{register_histogram, register_int_counter, Histogram, IntCounter};
-use zenith_utils::bin_ser::BeSer;
-use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::ZTenantId;
 
-use crate::relish::*;
-use crate::repository::WALRecord;
-use crate::waldecoder::XlXactParsedRecord;
-use crate::waldecoder::{MultiXactId, XlMultiXactCreate};
-use crate::PageServerConf;
-use postgres_ffi::nonrelfile_utils::transaction_id_set_status;
-use postgres_ffi::pg_constants;
-use postgres_ffi::XLogRecord;
+use bytes::{BufMut, Bytes, BytesMut};
 
-///
-/// `RelTag` + block number (`blknum`) gives us a unique id of the page in the cluster.
-///
-/// In Postgres `BufferTag` structure is used for exactly the same purpose.
-/// [See more related comments here](https://github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/buf_internals.h#L91).
-///
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Serialize, Deserialize)]
-pub struct BufferTag {
-    pub rel: RelTag,
-    pub blknum: u32,
-}
-
-///
-/// WAL Redo Manager is responsible for replaying WAL records.
-///
-/// Callers use the WAL redo manager through this abstract interface,
-/// which makes it easy to mock it in tests.
-pub trait WalRedoManager: Send + Sync {
-    /// Apply some WAL records.
-    ///
-    /// The caller passes an old page image, and WAL records that should be
-    /// applied over it. The return value is a new page image, after applying
-    /// the reords.
-    fn request_redo(
-        &self,
-        rel: RelishTag,
-        blknum: u32,
-        lsn: Lsn,
-        base_img: Option<Bytes>,
-        records: Vec<WALRecord>,
-    ) -> Result<Bytes, WalRedoError>;
-}
-
-///
-/// A dummy WAL Redo Manager implementation that doesn't allow replaying
-/// anything. Currently used during bootstrapping (zenith init), to create
-/// a Repository object without launching the real WAL redo process.
-///
-pub struct DummyRedoManager {}
-impl crate::walredo::WalRedoManager for DummyRedoManager {
-    fn request_redo(
-        &self,
-        _rel: RelishTag,
-        _blknum: u32,
-        _lsn: Lsn,
-        _base_img: Option<Bytes>,
-        _records: Vec<WALRecord>,
-    ) -> Result<Bytes, WalRedoError> {
-        Err(WalRedoError::InvalidState)
-    }
-}
+use crate::page_cache;
+use crate::page_cache::CacheEntry;
+use crate::page_cache::WALRecord;
+use crate::{page_cache::BufferTag, PageServerConf};
 
 static TIMEOUT: Duration = Duration::from_secs(20);
 
-// Metrics collected on WAL redo operations
 //
-// We collect the time spent in actual WAL redo ('redo'), and time waiting
-// for access to the postgres process ('wait') since there is only one for
-// each tenant.
-lazy_static! {
-    static ref WAL_REDO_TIME: Histogram =
-        register_histogram!("pageserver_wal_redo_time", "Time spent on WAL redo")
-            .expect("failed to define a metric");
-    static ref WAL_REDO_WAIT_TIME: Histogram = register_histogram!(
-        "pageserver_wal_redo_wait_time",
-        "Time spent waiting for access to the WAL redo process"
-    )
-    .expect("failed to define a metric");
-    static ref WAL_REDO_RECORD_COUNTER: IntCounter = register_int_counter!(
-        "pageserver_wal_records_replayed",
-        "Number of WAL records replayed"
-    )
-    .unwrap();
-}
+// Main entry point for the WAL applicator thread.
+//
+pub fn wal_redo_main(conf: PageServerConf, sys_id: u64) {
+    info!("WAL redo thread started {}", sys_id);
 
-///
-/// This is the real implementation that uses a Postgres process to
-/// perform WAL replay. Only one thread can use the processs at a time,
-/// that is controlled by the Mutex. In the future, we might want to
-/// launch a pool of processes to allow concurrent replay of multiple
-/// records.
-///
-pub struct PostgresRedoManager {
-    tenantid: ZTenantId,
-    conf: &'static PageServerConf,
+    // We block on waiting for requests on the walredo request channel, but
+    // use async I/O to communicate with the child process. Initialize the
+    // runtime for the async part.
+    let runtime = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .unwrap();
 
-    runtime: tokio::runtime::Runtime,
-    process: Mutex<Option<PostgresRedoProcess>>,
-}
+    let pcache = page_cache::get_pagecache(conf.clone(), sys_id);
 
-#[derive(Debug)]
-struct WalRedoRequest {
-    rel: RelishTag,
-    blknum: u32,
-    lsn: Lsn,
+    // Loop forever, handling requests as they come.
+    let walredo_channel_receiver = &pcache.walredo_receiver;
+    loop {
+        let mut process: WalRedoProcess;
+        let datadir = conf.data_dir.join(format!("wal-redo/{}", sys_id));
 
-    base_img: Option<Bytes>,
-    records: Vec<WALRecord>,
-}
+        info!("launching WAL redo postgres process {}", sys_id);
+        {
+            let _guard = runtime.enter();
+            process = WalRedoProcess::launch(&datadir, &runtime).unwrap();
+        }
 
-/// An error happened in WAL redo
-#[derive(Debug, thiserror::Error)]
-pub enum WalRedoError {
-    #[error(transparent)]
-    IoError(#[from] std::io::Error),
+        // Pretty arbitrarily, reuse the same Postgres process for 100 requests.
+        // After that, kill it and start a new one. This is mostly to avoid
+        // using up all shared buffers in Postgres's shared buffer cache; we don't
+        // want to write any pages to disk in the WAL redo process.
+        for _i in 1..100 {
+            let request = walredo_channel_receiver.recv().unwrap();
 
-    #[error("cannot perform WAL redo now")]
-    InvalidState,
-}
-
-///
-/// Public interface of WAL redo manager
-///
-impl WalRedoManager for PostgresRedoManager {
-    ///
-    /// Request the WAL redo manager to apply some WAL records
-    ///
-    /// The WAL redo is handled by a separate thread, so this just sends a request
-    /// to the thread and waits for response.
-    ///
-    fn request_redo(
-        &self,
-        rel: RelishTag,
-        blknum: u32,
-        lsn: Lsn,
-        base_img: Option<Bytes>,
-        records: Vec<WALRecord>,
-    ) -> Result<Bytes, WalRedoError> {
-        let start_time;
-        let lock_time;
-        let end_time;
-
-        let request = WalRedoRequest {
-            rel,
-            blknum,
-            lsn,
-            base_img,
-            records,
-        };
-
-        start_time = Instant::now();
-        let result = {
-            let mut process_guard = self.process.lock().unwrap();
-            lock_time = Instant::now();
-
-            // launch the WAL redo process on first use
-            if process_guard.is_none() {
-                let p = self
-                    .runtime
-                    .block_on(PostgresRedoProcess::launch(self.conf, &self.tenantid))?;
-                *process_guard = Some(p);
+            let result = handle_apply_request(&pcache, &process, &runtime, request);
+            if result.is_err() {
+                // On error, kill the process.
+                break;
             }
-            let process = (*process_guard).as_ref().unwrap();
+        }
 
-            self.runtime
-                .block_on(self.handle_apply_request(&process, &request))
-        };
-        end_time = Instant::now();
-
-        WAL_REDO_WAIT_TIME.observe(lock_time.duration_since(start_time).as_secs_f64());
-        WAL_REDO_TIME.observe(end_time.duration_since(lock_time).as_secs_f64());
-
-        result
+        info!("killing WAL redo postgres process");
+        let _ = runtime.block_on(process.stdin.get_mut().shutdown());
+        let mut child = process.child;
+        drop(process.stdin);
+        let _ = runtime.block_on(child.wait());
     }
 }
 
-fn mx_offset_to_flags_offset(xid: MultiXactId) -> usize {
-    ((xid / pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP as u32) as u16
-        % pg_constants::MULTIXACT_MEMBERGROUPS_PER_PAGE
-        * pg_constants::MULTIXACT_MEMBERGROUP_SIZE) as usize
-}
+fn handle_apply_request(
+    pcache: &page_cache::PageCache,
+    process: &WalRedoProcess,
+    runtime: &Runtime,
+    entry_rc: Arc<CacheEntry>,
+) -> Result<(), Error> {
+    let tag = entry_rc.key.tag;
+    let lsn = entry_rc.key.lsn;
+    let (base_img, records) = pcache.collect_records_for_apply(entry_rc.as_ref());
 
-fn mx_offset_to_flags_bitshift(xid: MultiXactId) -> u16 {
-    (xid as u16) % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP
-        * pg_constants::MXACT_MEMBER_BITS_PER_XACT
-}
+    let mut entry = entry_rc.content.lock().unwrap();
+    entry.apply_pending = false;
 
-/* Location (byte offset within page) of TransactionId of given member */
-fn mx_offset_to_member_offset(xid: MultiXactId) -> usize {
-    mx_offset_to_flags_offset(xid)
-        + (pg_constants::MULTIXACT_FLAGBYTES_PER_GROUP
-            + (xid as u16 % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP) * 4) as usize
-}
+    let nrecords = records.len();
 
-impl PostgresRedoManager {
-    ///
-    /// Create a new PostgresRedoManager.
-    ///
-    pub fn new(conf: &'static PageServerConf, tenantid: ZTenantId) -> PostgresRedoManager {
-        // We block on waiting for requests on the walredo request channel, but
-        // use async I/O to communicate with the child process. Initialize the
-        // runtime for the async part.
-        let runtime = tokio::runtime::Builder::new_current_thread()
-            .enable_all()
-            .build()
-            .unwrap();
+    let start = Instant::now();
+    let apply_result = process.apply_wal_records(runtime, tag, base_img, records);
+    let duration = start.elapsed();
 
-        // The actual process is launched lazily, on first request.
-        PostgresRedoManager {
-            runtime,
-            tenantid,
-            conf,
-            process: Mutex::new(None),
-        }
+    let result;
+
+    debug!(
+        "applied {} WAL records in {} ms to reconstruct page image at LSN {:X}/{:X}",
+        nrecords,
+        duration.as_millis(),
+        lsn >> 32,
+        lsn & 0xffff_ffff
+    );
+
+    if let Err(e) = apply_result {
+        error!("could not apply WAL records: {}", e);
+        result = Err(e);
+    } else {
+        entry.page_image = Some(apply_result.unwrap());
+        pcache
+            .num_page_images
+            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+        result = Ok(());
     }
 
-    ///
-    /// Process one request for WAL redo.
-    ///
-    async fn handle_apply_request(
-        &self,
-        process: &PostgresRedoProcess,
-        request: &WalRedoRequest,
-    ) -> Result<Bytes, WalRedoError> {
-        let rel = request.rel;
-        let blknum = request.blknum;
-        let lsn = request.lsn;
-        let base_img = request.base_img.clone();
-        let records = &request.records;
+    // Wake up the requester, whether the operation succeeded or not.
+    entry_rc.walredo_condvar.notify_all();
 
-        let nrecords = records.len();
-
-        let start = Instant::now();
-
-        let apply_result: Result<Bytes, Error>;
-        if let RelishTag::Relation(rel) = rel {
-            // Relational WAL records are applied using wal-redo-postgres
-            let buf_tag = BufferTag { rel, blknum };
-            apply_result = process.apply_wal_records(buf_tag, base_img, records).await;
-        } else {
-            // Non-relational WAL records are handled here, with custom code that has the
-            // same effects as the corresponding Postgres WAL redo function.
-            const ZERO_PAGE: [u8; 8192] = [0u8; 8192];
-            let mut page = BytesMut::new();
-            if let Some(fpi) = base_img {
-                // If full-page image is provided, then use it...
-                page.extend_from_slice(&fpi[..]);
-            } else {
-                // otherwise initialize page with zeros
-                page.extend_from_slice(&ZERO_PAGE);
-            }
-            // Apply all collected WAL records
-            for record in records {
-                let mut buf = record.rec.clone();
-
-                WAL_REDO_RECORD_COUNTER.inc();
-
-                // 1. Parse XLogRecord struct
-                // FIXME: refactor to avoid code duplication.
-                let xlogrec = XLogRecord::from_bytes(&mut buf);
-
-                //move to main data
-                // TODO probably, we should store some records in our special format
-                // to avoid this weird parsing on replay
-                let skip = (record.main_data_offset - pg_constants::SIZEOF_XLOGRECORD) as usize;
-                if buf.remaining() > skip {
-                    buf.advance(skip);
-                }
-
-                if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
-                    // Transaction manager stuff
-                    let rec_segno = match rel {
-                        RelishTag::Slru { slru, segno } => {
-                            if slru != SlruKind::Clog {
-                                panic!("Not valid XACT relish tag {:?}", rel);
-                            }
-                            segno
-                        }
-                        _ => panic!("Not valid XACT relish tag {:?}", rel),
-                    };
-                    let parsed_xact =
-                        XlXactParsedRecord::decode(&mut buf, xlogrec.xl_xid, xlogrec.xl_info);
-                    if parsed_xact.info == pg_constants::XLOG_XACT_COMMIT
-                        || parsed_xact.info == pg_constants::XLOG_XACT_COMMIT_PREPARED
-                    {
-                        transaction_id_set_status(
-                            parsed_xact.xid,
-                            pg_constants::TRANSACTION_STATUS_COMMITTED,
-                            &mut page,
-                        );
-                        for subxact in &parsed_xact.subxacts {
-                            let pageno = *subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
-                            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-                            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-                            // only update xids on the requested page
-                            if rec_segno == segno && blknum == rpageno {
-                                transaction_id_set_status(
-                                    *subxact,
-                                    pg_constants::TRANSACTION_STATUS_SUB_COMMITTED,
-                                    &mut page,
-                                );
-                            }
-                        }
-                    } else if parsed_xact.info == pg_constants::XLOG_XACT_ABORT
-                        || parsed_xact.info == pg_constants::XLOG_XACT_ABORT_PREPARED
-                    {
-                        transaction_id_set_status(
-                            parsed_xact.xid,
-                            pg_constants::TRANSACTION_STATUS_ABORTED,
-                            &mut page,
-                        );
-                        for subxact in &parsed_xact.subxacts {
-                            let pageno = *subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
-                            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-                            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-                            // only update xids on the requested page
-                            if rec_segno == segno && blknum == rpageno {
-                                transaction_id_set_status(
-                                    *subxact,
-                                    pg_constants::TRANSACTION_STATUS_ABORTED,
-                                    &mut page,
-                                );
-                            }
-                        }
-                    }
-                } else if xlogrec.xl_rmid == pg_constants::RM_MULTIXACT_ID {
-                    // Multixact operations
-                    let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
-                    if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
-                        let xlrec = XlMultiXactCreate::decode(&mut buf);
-                        if let RelishTag::Slru {
-                            slru,
-                            segno: rec_segno,
-                        } = rel
-                        {
-                            if slru == SlruKind::MultiXactMembers {
-                                for i in 0..xlrec.nmembers {
-                                    let pageno =
-                                        i / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-                                    let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-                                    let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-                                    if segno == rec_segno && rpageno == blknum {
-                                        // update only target block
-                                        let offset = xlrec.moff + i;
-                                        let memberoff = mx_offset_to_member_offset(offset);
-                                        let flagsoff = mx_offset_to_flags_offset(offset);
-                                        let bshift = mx_offset_to_flags_bitshift(offset);
-                                        let mut flagsval =
-                                            LittleEndian::read_u32(&page[flagsoff..flagsoff + 4]);
-                                        flagsval &= !(((1
-                                            << pg_constants::MXACT_MEMBER_BITS_PER_XACT)
-                                            - 1)
-                                            << bshift);
-                                        flagsval |= xlrec.members[i as usize].status << bshift;
-                                        LittleEndian::write_u32(
-                                            &mut page[flagsoff..flagsoff + 4],
-                                            flagsval,
-                                        );
-                                        LittleEndian::write_u32(
-                                            &mut page[memberoff..memberoff + 4],
-                                            xlrec.members[i as usize].xid,
-                                        );
-                                    }
-                                }
-                            } else {
-                                // Multixact offsets SLRU
-                                let offs = (xlrec.mid
-                                    % pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32
-                                    * 4) as usize;
-                                LittleEndian::write_u32(&mut page[offs..offs + 4], xlrec.moff);
-                            }
-                        } else {
-                            panic!();
-                        }
-                    } else {
-                        panic!();
-                    }
-                } else if xlogrec.xl_rmid == pg_constants::RM_RELMAP_ID {
-                    // Relation map file has size 512 bytes
-                    page.clear();
-                    page.extend_from_slice(&buf[12..]); // skip xl_relmap_update
-                    assert!(page.len() == 512); // size of pg_filenode.map
-                }
-            }
-
-            apply_result = Ok::<Bytes, Error>(page.freeze());
-        }
-
-        let duration = start.elapsed();
-
-        let result: Result<Bytes, WalRedoError>;
-
-        debug!(
-            "applied {} WAL records in {} ms to reconstruct page image at LSN {}",
-            nrecords,
-            duration.as_millis(),
-            lsn
-        );
-
-        if let Err(e) = apply_result {
-            error!("could not apply WAL records: {}", e);
-            result = Err(WalRedoError::IoError(e));
-        } else {
-            let img = apply_result.unwrap();
-
-            result = Ok(img);
-        }
-
-        // The caller is responsible for sending the response
-        result
-    }
+    return result;
 }
 
-///
-/// Handle to the Postgres WAL redo process
-///
-struct PostgresRedoProcess {
+struct WalRedoProcess {
+    child: Child,
     stdin: RefCell<ChildStdin>,
     stdout: RefCell<ChildStdout>,
 }
 
-impl PostgresRedoProcess {
+impl WalRedoProcess {
     //
     // Start postgres binary in special WAL redo mode.
     //
-    async fn launch(
-        conf: &PageServerConf,
-        tenantid: &ZTenantId,
-    ) -> Result<PostgresRedoProcess, Error> {
-        // FIXME: We need a dummy Postgres cluster to run the process in. Currently, we
-        // just create one with constant name. That fails if you try to launch more than
-        // one WAL redo manager concurrently.
-        let datadir = conf.tenant_path(&tenantid).join("wal-redo-datadir");
-
-        // Create empty data directory for wal-redo postgres, deleting old one first.
-        if datadir.exists() {
-            info!("directory {:?} exists, removing", &datadir);
-            if let Err(e) = fs::remove_dir_all(&datadir) {
-                error!("could not remove old wal-redo-datadir: {:?}", e);
-            }
-        }
-        info!("running initdb in {:?}", datadir.display());
-        let initdb = Command::new(conf.pg_bin_dir().join("initdb"))
-            .args(&["-D", datadir.to_str().unwrap()])
-            .arg("-N")
-            .env_clear()
-            .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
-            .output()
-            .await
+    // Tests who run pageserver binary are setting proper PG_BIN_DIR
+    // and PG_LIB_DIR so that WalRedo would start right postgres. We may later
+    // switch to setting same things in pageserver config file.
+    fn launch(datadir: &PathBuf, runtime: &Runtime) -> Result<WalRedoProcess, Error> {
+        // Create empty data directory for wal-redo postgres deleting old one.
+        fs::remove_dir_all(datadir.to_str().unwrap()).ok();
+        let initdb = runtime
+            .block_on(
+                Command::new("initdb")
+                    .args(&["-D", datadir.to_str().unwrap()])
+                    .arg("-N")
+                    .output(),
+            )
             .expect("failed to execute initdb");
 
         if !initdb.status.success() {
@@ -499,32 +165,21 @@ impl PostgresRedoProcess {
                 std::str::from_utf8(&initdb.stdout).unwrap(),
                 std::str::from_utf8(&initdb.stderr).unwrap()
             );
-        } else {
-            // Limit shared cache for wal-redo-postres
-            let mut config = OpenOptions::new()
-                .append(true)
-                .open(PathBuf::from(&datadir).join("postgresql.conf"))?;
-            config.write_all(b"shared_buffers=128kB\n")?;
-            config.write_all(b"fsync=off\n")?;
-            config.write_all(b"shared_preload_libraries=zenith\n")?;
-            config.write_all(b"zenith.wal_redo=on\n")?;
         }
+
         // Start postgres itself
-        let mut child = Command::new(conf.pg_bin_dir().join("postgres"))
+        let mut child = Command::new("postgres")
             .arg("--wal-redo")
             .stdin(Stdio::piped())
             .stderr(Stdio::piped())
             .stdout(Stdio::piped())
-            .env_clear()
-            .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
-            .env("PGDATA", &datadir)
+            .env("PGDATA", datadir.to_str().unwrap())
             .spawn()
             .expect("postgres --wal-redo command failed to start");
 
         info!(
-            "launched WAL redo postgres process on {:?}",
-            datadir.display()
+            "launched WAL redo postgres process on {}",
+            datadir.to_str().unwrap()
         );
 
         let stdin = child.stdin.take().expect("failed to open child's stdin");
@@ -545,14 +200,15 @@ impl PostgresRedoProcess {
                 if res.unwrap() == 0 {
                     break;
                 }
-                error!("wal-redo-postgres: {}", line.trim());
+                debug!("wal-redo-postgres: {}", line.trim());
                 line.clear();
             }
             Ok::<(), Error>(())
         };
         tokio::spawn(f_stderr);
 
-        Ok(PostgresRedoProcess {
+        Ok(WalRedoProcess {
+            child: child,
             stdin: RefCell::new(stdin),
             stdout: RefCell::new(stdout),
         })
@@ -562,157 +218,146 @@ impl PostgresRedoProcess {
     // Apply given WAL records ('records') over an old page image. Returns
     // new page image.
     //
-    async fn apply_wal_records(
+    fn apply_wal_records(
         &self,
+        runtime: &Runtime,
         tag: BufferTag,
         base_img: Option<Bytes>,
-        records: &[WALRecord],
-    ) -> Result<Bytes, std::io::Error> {
+        records: Vec<WALRecord>,
+    ) -> Result<Bytes, Error> {
         let mut stdin = self.stdin.borrow_mut();
         let mut stdout = self.stdout.borrow_mut();
-
-        // We do three things simultaneously: send the old base image and WAL records to
-        // the child process's stdin, read the result from child's stdout, and forward any logging
-        // information that the child writes to its stderr to the page server's log.
-        //
-        // 'f_stdin' handles writing the base image and WAL records to the child process.
-        // 'f_stdout' below reads the result back. And 'f_stderr', which was spawned into the
-        // tokio runtime in the 'launch' function already, forwards the logging.
-        let f_stdin = async {
-            // Send base image, if any. (If the record initializes the page, previous page
-            // version is not needed.)
-            timeout(
-                TIMEOUT,
-                stdin.write_all(&build_begin_redo_for_block_msg(tag)),
-            )
-            .await??;
-            if base_img.is_some() {
+        return runtime.block_on(async {
+            //
+            // This async block sends all the commands to the process.
+            //
+            // For reasons I don't understand, this needs to be a "move" block;
+            // otherwise the stdin pipe doesn't get closed, despite the shutdown()
+            // call.
+            //
+            let f_stdin = async {
+                // Send base image, if any. (If the record initializes the page, previous page
+                // version is not needed.)
                 timeout(
                     TIMEOUT,
-                    stdin.write_all(&build_push_page_msg(tag, base_img.unwrap())),
+                    stdin.write_all(&build_begin_redo_for_block_msg(tag)),
                 )
                 .await??;
-            }
+                if base_img.is_some() {
+                    timeout(
+                        TIMEOUT,
+                        stdin.write_all(&build_push_page_msg(tag, base_img.unwrap())),
+                    )
+                    .await??;
+                }
 
-            // Send WAL records.
-            for rec in records.iter() {
-                let r = rec.clone();
+                // Send WAL records.
+                for rec in records.iter() {
+                    let r = rec.clone();
 
-                WAL_REDO_RECORD_COUNTER.inc();
+                    stdin
+                        .write_all(&build_apply_record_msg(r.lsn, r.rec))
+                        .await?;
 
-                stdin
-                    .write_all(&build_apply_record_msg(r.lsn, r.rec))
-                    .await?;
+                    //debug!("sent WAL record to wal redo postgres process ({:X}/{:X}",
+                    //       r.lsn >> 32, r.lsn & 0xffff_ffff);
+                }
+                //debug!("sent {} WAL records to wal redo postgres process ({:X}/{:X}",
+                //       records.len(), lsn >> 32, lsn & 0xffff_ffff);
 
-                //debug!("sent WAL record to wal redo postgres process ({:X}/{:X}",
-                //       r.lsn >> 32, r.lsn & 0xffff_ffff);
-            }
-            //debug!("sent {} WAL records to wal redo postgres process ({:X}/{:X}",
-            //       records.len(), lsn >> 32, lsn & 0xffff_ffff);
+                // Send GetPage command to get the result back
+                timeout(TIMEOUT, stdin.write_all(&build_get_page_msg(tag))).await??;
+                timeout(TIMEOUT, stdin.flush()).await??;
+                //debug!("sent GetPage for {}", tag.blknum);
+                Ok::<(), Error>(())
+            };
 
-            // Send GetPage command to get the result back
-            timeout(TIMEOUT, stdin.write_all(&build_get_page_msg(tag))).await??;
-            timeout(TIMEOUT, stdin.flush()).await??;
-            //debug!("sent GetPage for {}", tag.blknum);
-            Ok::<(), Error>(())
-        };
+            // Read back new page image
+            let f_stdout = async {
+                let mut buf = [0u8; 8192];
 
-        // Read back new page image
-        let f_stdout = async {
-            let mut buf = [0u8; 8192];
+                timeout(TIMEOUT, stdout.read_exact(&mut buf)).await??;
+                //debug!("got response for {}", tag.blknum);
+                Ok::<[u8; 8192], Error>(buf)
+            };
 
-            timeout(TIMEOUT, stdout.read_exact(&mut buf)).await??;
-            //debug!("got response for {}", tag.blknum);
-            Ok::<[u8; 8192], Error>(buf)
-        };
+            // Kill the process. This closes its stdin, which should signal the process
+            // to terminate. TODO: SIGKILL if needed
+            //child.wait();
 
-        let res = tokio::try_join!(f_stdout, f_stdin)?;
+            let res = futures::try_join!(f_stdout, f_stdin)?;
 
-        let buf = res.0;
+            let buf = res.0;
 
-        Ok::<Bytes, Error>(Bytes::from(std::vec::Vec::from(buf)))
+            Ok::<Bytes, Error>(Bytes::from(std::vec::Vec::from(buf)))
+        });
     }
 }
 
-// Functions for constructing messages to send to the postgres WAL redo
-// process. See vendor/postgres/src/backend/tcop/zenith_wal_redo.c for
-// explanation of the protocol.
-
 fn build_begin_redo_for_block_msg(tag: BufferTag) -> Bytes {
-    let len = 4 + 1 + 4 * 4;
+    let len = 4 + 5 * 4;
     let mut buf = BytesMut::with_capacity(1 + len);
 
-    buf.put_u8(b'B');
+    buf.put_u8('B' as u8);
     buf.put_u32(len as u32);
+    buf.put_u32(tag.spcnode);
+    buf.put_u32(tag.dbnode);
+    buf.put_u32(tag.relnode);
+    buf.put_u32(tag.forknum as u32);
+    buf.put_u32(tag.blknum);
 
-    // FIXME: this is a temporary hack that should go away when we refactor
-    // the postgres protocol serialization + handlers.
-    //
-    // BytesMut is a dynamic growable buffer, used a lot in tokio code but
-    // not in the std library. To write to a BytesMut from a serde serializer,
-    // we need to either:
-    // - pre-allocate the required buffer space. This is annoying because we
-    //   shouldn't care what the exact serialized size is-- that's the
-    //   serializer's job.
-    // - Or, we need to create a temporary "writer" (which implements the
-    //   `Write` trait). It's a bit awkward, because the writer consumes the
-    //   underlying BytesMut, and we need to extract it later with
-    //   `into_inner`.
-    let mut writer = buf.writer();
-    tag.ser_into(&mut writer)
-        .expect("serialize BufferTag should always succeed");
-    let buf = writer.into_inner();
+    assert!(buf.len() == 1 + len);
 
-    debug_assert!(buf.len() == 1 + len);
-
-    buf.freeze()
+    return buf.freeze();
 }
 
 fn build_push_page_msg(tag: BufferTag, base_img: Bytes) -> Bytes {
     assert!(base_img.len() == 8192);
 
-    let len = 4 + 1 + 4 * 4 + base_img.len();
+    let len = 4 + 5 * 4 + base_img.len();
     let mut buf = BytesMut::with_capacity(1 + len);
 
-    buf.put_u8(b'P');
+    buf.put_u8('P' as u8);
     buf.put_u32(len as u32);
-    let mut writer = buf.writer();
-    tag.ser_into(&mut writer)
-        .expect("serialize BufferTag should always succeed");
-    let mut buf = writer.into_inner();
+    buf.put_u32(tag.spcnode);
+    buf.put_u32(tag.dbnode);
+    buf.put_u32(tag.relnode);
+    buf.put_u32(tag.forknum as u32);
+    buf.put_u32(tag.blknum);
     buf.put(base_img);
 
-    debug_assert!(buf.len() == 1 + len);
+    assert!(buf.len() == 1 + len);
 
-    buf.freeze()
+    return buf.freeze();
 }
 
-fn build_apply_record_msg(endlsn: Lsn, rec: Bytes) -> Bytes {
+fn build_apply_record_msg(endlsn: u64, rec: Bytes) -> Bytes {
     let len = 4 + 8 + rec.len();
     let mut buf = BytesMut::with_capacity(1 + len);
 
-    buf.put_u8(b'A');
+    buf.put_u8('A' as u8);
     buf.put_u32(len as u32);
-    buf.put_u64(endlsn.0);
+    buf.put_u64(endlsn);
     buf.put(rec);
 
-    debug_assert!(buf.len() == 1 + len);
+    assert!(buf.len() == 1 + len);
 
-    buf.freeze()
+    return buf.freeze();
 }
 
 fn build_get_page_msg(tag: BufferTag) -> Bytes {
-    let len = 4 + 1 + 4 * 4;
+    let len = 4 + 5 * 4;
     let mut buf = BytesMut::with_capacity(1 + len);
 
-    buf.put_u8(b'G');
+    buf.put_u8('G' as u8);
     buf.put_u32(len as u32);
-    let mut writer = buf.writer();
-    tag.ser_into(&mut writer)
-        .expect("serialize BufferTag should always succeed");
-    let buf = writer.into_inner();
+    buf.put_u32(tag.spcnode);
+    buf.put_u32(tag.dbnode);
+    buf.put_u32(tag.relnode);
+    buf.put_u32(tag.forknum as u32);
+    buf.put_u32(tag.blknum);
 
-    debug_assert!(buf.len() == 1 + len);
+    assert!(buf.len() == 1 + len);
 
-    buf.freeze()
+    return buf.freeze();
 }

pgbuild.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+#
+#   Purpose of this script is to build and install postgres in a local directory
+# so that zenith intergation tests would find pg binaries and support files.
+#
+# ./pgbuild.sh would do following:
+#
+#   1) run out-of-source build of postgres in REPO_ROOT/tmp_install/build directory (I'm reusing
+#  tmp_install path here since it is already present in .gitignore)
+#
+#   2) installs postgres to REPO_ROOT/tmp_install/
+#
+REPO_ROOT=$(dirname "$0")
+REPO_ROOT="`( cd \"$REPO_ROOT\" && pwd )`"
+
+# configure
+echo "Configuring postgres build"
+mkdir -p $REPO_ROOT/tmp_install/build
+cd $REPO_ROOT/tmp_install/build
+../../vendor/postgres/configure CFLAGS='-O0' --enable-debug --enable-cassert \
+    --enable-depend --with-libxml --prefix=/ > configure.log
+
+# compile
+echo "Compiling postgres"
+make -j8 -s
+export DESTDIR=$REPO_ROOT/tmp_install
+
+echo "Installing postgres to $DESTDIR"
+make install -s

postgres_ffi/Cargo.toml

@@ -1,26 +0,0 @@
-[package]
-name = "postgres_ffi"
-version = "0.1.0"
-authors = ["Heikki Linnakangas <heikki@zenith.tech>"]
-edition = "2018"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-chrono = "0.4.19"
-rand = "0.8.3"
-regex = "1.4.5"
-bytes = "1.0.1"
-byteorder = "1.4.3"
-anyhow = "1.0"
-crc32c = "0.6.0"
-hex = "0.4.3"
-lazy_static = "1.4"
-log = "0.4.14"
-memoffset = "0.6.2"
-thiserror = "1.0"
-workspace_hack = { path = "../workspace_hack" }
-zenith_utils = { path = "../zenith_utils" }
-
-[build-dependencies]
-bindgen = "0.57"

postgres_ffi/README

@@ -1,25 +0,0 @@
-This module contains utilities for working with PostgreSQL file
-formats. It's a collection of structs that are auto-generated from the
-PostgreSQL header files using bindgen, and Rust functions to read and
-manipulate them.
-
-There are also a bunch of constants in `pg_constants.rs` that are copied
-from various PostgreSQL headers, rather than auto-generated. They mostly
-should be auto-generated too, but that's a TODO.
-
-The PostgreSQL on-disk file format is not portable across different
-CPU architectures and operating systems. It is also subject to change
-in each major PostgreSQL version. Currently, this module is based on
-PostgreSQL v14, but in the future we will probably need a separate
-copy for each PostgreSQL version.
-
-To interact with the C structs, there is some unsafe code in this
-module. Do not copy-paste that to the rest of the codebase! Keep the
-amount of unsafe code to a minimum, and limited to this module only,
-and only where it's truly needed.
-
-TODO: Currently, there is also some code that deals with WAL records
-in pageserver/src/waldecoder.rs.  That should be moved into this
-module. The rest of the codebase should not have intimate knowledge of
-PostgreSQL file formats or WAL layout, that knowledge should be
-encapsulated in this module.

postgres_ffi/build.rs

@@ -1,57 +0,0 @@
-extern crate bindgen;
-
-use std::env;
-use std::path::PathBuf;
-
-fn main() {
-    // Tell cargo to invalidate the built crate whenever the wrapper changes
-    println!("cargo:rerun-if-changed=pg_control_ffi.h");
-
-    // The bindgen::Builder is the main entry point
-    // to bindgen, and lets you build up options for
-    // the resulting bindings.
-    let bindings = bindgen::Builder::default()
-        //
-        // All the needed PostgreSQL headers are included from 'pg_control_ffi.h'
-        //
-        .header("pg_control_ffi.h")
-        //
-        // Tell cargo to invalidate the built crate whenever any of the
-        // included header files changed.
-        //
-        .parse_callbacks(Box::new(bindgen::CargoCallbacks))
-        //
-        // These are the types and constants that we want to generate bindings for
-        //
-        .whitelist_type("ControlFileData")
-        .whitelist_type("CheckPoint")
-        .whitelist_type("FullTransactionId")
-        .whitelist_type("XLogRecord")
-        .whitelist_type("XLogPageHeaderData")
-        .whitelist_type("XLogLongPageHeaderData")
-        .whitelist_var("XLOG_PAGE_MAGIC")
-        .whitelist_var("PG_CONTROL_FILE_SIZE")
-        .whitelist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
-        .whitelist_type("DBState")
-        //
-        // Path the server include dir. It is in tmp_install/include/server, if you did
-        // "configure --prefix=<path to tmp_install>". But if you used "configure --prefix=/",
-        // and used DESTDIR to move it into tmp_install, then it's in
-        // tmp_install/include/postgres/server
-        // 'pg_config --includedir-server' would perhaps be the more proper way to find it,
-        // but this will do for now.
-        //
-        .clang_arg("-I../tmp_install/include/server")
-        .clang_arg("-I../tmp_install/include/postgresql/server")
-        //
-        // Finish the builder and generate the bindings.
-        //
-        .generate()
-        .expect("Unable to generate bindings");
-
-    // Write the bindings to the $OUT_DIR/bindings.rs file.
-    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
-    bindings
-        .write_to_file(out_path.join("bindings.rs"))
-        .expect("Couldn't write bindings!");
-}

postgres_ffi/pg_control_ffi.h

@@ -1,10 +0,0 @@
-/*
- * This header file is the input to bindgen. It includes all the
- * PostgreSQL headers that we need to auto-generate Rust structs
- * from. If you need to expose a new struct to Rust code, add the
- * header here, and whitelist the struct in the build.rs file.
- */
-#include "c.h"
-#include "catalog/pg_control.h"
-#include "access/xlog_internal.h"
-

postgres_ffi/samples/pg_hba.conf

@@ -1,98 +0,0 @@
-# PostgreSQL Client Authentication Configuration File
-# ===================================================
-#
-# Refer to the "Client Authentication" section in the PostgreSQL
-# documentation for a complete description of this file.  A short
-# synopsis follows.
-#
-# This file controls: which hosts are allowed to connect, how clients
-# are authenticated, which PostgreSQL user names they can use, which
-# databases they can access.  Records take one of these forms:
-#
-# local         DATABASE  USER  METHOD  [OPTIONS]
-# host          DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
-# hostssl       DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
-# hostnossl     DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
-# hostgssenc    DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
-# hostnogssenc  DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
-#
-# (The uppercase items must be replaced by actual values.)
-#
-# The first field is the connection type:
-# - "local" is a Unix-domain socket
-# - "host" is a TCP/IP socket (encrypted or not)
-# - "hostssl" is a TCP/IP socket that is SSL-encrypted
-# - "hostnossl" is a TCP/IP socket that is not SSL-encrypted
-# - "hostgssenc" is a TCP/IP socket that is GSSAPI-encrypted
-# - "hostnogssenc" is a TCP/IP socket that is not GSSAPI-encrypted
-#
-# DATABASE can be "all", "sameuser", "samerole", "replication", a
-# database name, or a comma-separated list thereof. The "all"
-# keyword does not match "replication". Access to replication
-# must be enabled in a separate record (see example below).
-#
-# USER can be "all", a user name, a group name prefixed with "+", or a
-# comma-separated list thereof.  In both the DATABASE and USER fields
-# you can also write a file name prefixed with "@" to include names
-# from a separate file.
-#
-# ADDRESS specifies the set of hosts the record matches.  It can be a
-# host name, or it is made up of an IP address and a CIDR mask that is
-# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that
-# specifies the number of significant bits in the mask.  A host name
-# that starts with a dot (.) matches a suffix of the actual host name.
-# Alternatively, you can write an IP address and netmask in separate
-# columns to specify the set of hosts.  Instead of a CIDR-address, you
-# can write "samehost" to match any of the server's own IP addresses,
-# or "samenet" to match any address in any subnet that the server is
-# directly connected to.
-#
-# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256",
-# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert".
-# Note that "password" sends passwords in clear text; "md5" or
-# "scram-sha-256" are preferred since they send encrypted passwords.
-#
-# OPTIONS are a set of options for the authentication in the format
-# NAME=VALUE.  The available options depend on the different
-# authentication methods -- refer to the "Client Authentication"
-# section in the documentation for a list of which options are
-# available for which authentication methods.
-#
-# Database and user names containing spaces, commas, quotes and other
-# special characters must be quoted.  Quoting one of the keywords
-# "all", "sameuser", "samerole" or "replication" makes the name lose
-# its special character, and just match a database or username with
-# that name.
-#
-# This file is read on server startup and when the server receives a
-# SIGHUP signal.  If you edit the file on a running system, you have to
-# SIGHUP the server for the changes to take effect, run "pg_ctl reload",
-# or execute "SELECT pg_reload_conf()".
-#
-# Put your actual configuration here
-# ----------------------------------
-#
-# If you want to allow non-local connections, you need to add more
-# "host" records.  In that case you will also need to make PostgreSQL
-# listen on a non-local interface via the listen_addresses
-# configuration parameter, or via the -i or -h command line switches.
-
-# CAUTION: Configuring the system for local "trust" authentication
-# allows any local user to connect as any PostgreSQL user, including
-# the database superuser.  If you do not trust all your local users,
-# use another authentication method.
-
-
-# TYPE  DATABASE        USER            ADDRESS                 METHOD
-
-# "local" is for Unix domain socket connections only
-local   all             all                                     trust
-# IPv4 local connections:
-host    all             all             127.0.0.1/32            trust
-# IPv6 local connections:
-host    all             all             ::1/128                 trust
-# Allow replication connections from localhost, by a user with the
-# replication privilege.
-local   replication     all                                     trust
-host    replication     all             127.0.0.1/32            trust
-host    replication     all             ::1/128                 trust