Add in-memory storage engine

.circleci/config.yml

@@ -7,29 +7,13 @@ executors:
   zenith-build-executor:
     resource_class: xlarge
     docker:
-      - image: cimg/rust:1.52.1
+      - image: cimg/rust:1.51.0
 
 jobs:
-  check-codestyle:
-    executor: zenith-build-executor
-    steps:
-      - checkout
-
-      - run:
-          name: rustfmt
-          when: always
-          command: |
-            cargo fmt --all -- --check
 
   # A job to build postgres
   build-postgres:
     executor: zenith-build-executor
-    parameters:
-      build_type:
-        type: enum
-        enum: ["debug", "release"]
-    environment:
-      BUILD_TYPE: << parameters.build_type >>
     steps:
         # Checkout the git repo (circleci doesn't have a flag to enable submodules here)
       - checkout
@@ -45,7 +29,7 @@ jobs:
           name: Restore postgres cache
           keys:
             # Restore ONLY if the rev key matches exactly
-            - v04-postgres-cache-<< parameters.build_type >>-{{ checksum "/tmp/cache-key-postgres" }}
+            - v03-postgres-cache-{{ checksum "/tmp/cache-key-postgres" }}
 
         # FIXME We could cache our own docker container, instead of installing packages every time.
       - run:
@@ -65,12 +49,12 @@ jobs:
             if [ ! -e tmp_install/bin/postgres ]; then
               # "depth 1" saves some time by not cloning the whole repo
               git submodule update --init --depth 1
-              make postgres -j8
+              make postgres
             fi
 
       - save_cache:
           name: Save postgres cache
-          key: v04-postgres-cache-<< parameters.build_type >>-{{ checksum "/tmp/cache-key-postgres" }}
+          key: v03-postgres-cache-{{ checksum "/tmp/cache-key-postgres" }}
           paths:
             - tmp_install
 
@@ -102,7 +86,7 @@ jobs:
           name: Restore postgres cache
           keys:
             # Restore ONLY if the rev key matches exactly
-            - v04-postgres-cache-<< parameters.build_type >>-{{ checksum "/tmp/cache-key-postgres" }}
+            - v03-postgres-cache-{{ checksum "/tmp/cache-key-postgres" }}
 
       - restore_cache:
           name: Restore rust cache
@@ -134,14 +118,6 @@ jobs:
             - ~/.cargo/git
             - target
 
-        # Run style checks
-        # has to run separately from cargo fmt section
-        # since needs to run with dependencies
-      - run:
-          name: clippy
-          command: |
-            ./run_clippy.sh
-
         # Run rust unit tests
       - run: cargo test
 
@@ -207,9 +183,6 @@ jobs:
       needs_postgres_source:
         type: boolean
         default: false
-      run_in_parallel:
-        type: boolean
-        default: true
     steps:
       - attach_workspace:
           at: /tmp/zenith
@@ -238,24 +211,16 @@ jobs:
               echo "test_selection must be set"
               exit 1
             fi
-            if << parameters.run_in_parallel >>; then
-              EXTRA_PARAMS="-n4 $EXTRA_PARAMS"
-            fi;
-            ./netstat-script.sh &
-            NS_PID=$!
             # Run the tests.
             #
             # The junit.xml file allows CircleCI to display more fine-grained test information
             # in its "Tests" tab in the results page.
+            # -s prevents pytest from capturing output, which helps to see
+            # what's going on if the test hangs
             # --verbose prints name of each test (helpful when there are
             # multiple tests in one file)
             # -rA prints summary in the end
-            # -n4 uses four processes to run tests via pytest-xdist
-            # -s is not used to prevent pytest from capturing output, because tests are running
-            # in parallel and logs are mixed between different tests
-            pipenv run pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short --verbose -rA $TEST_SELECTION $EXTRA_PARAMS
-            kill $NS_PID
-            awk '/===/ {if (count) print count; print; count=0; next} {count++} END {print count}' $TEST_OUTPUT/netstat.stdout > $TEST_OUTPUT/netstat_stats.stdout
+            pipenv run pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short -s --verbose -rA $TEST_SELECTION $EXTRA_PARAMS
       - run:
           # CircleCI artifacts are preserved one file at a time, so skipping
           # this step isn't a good idea. If you want to extract the
@@ -264,7 +229,7 @@ jobs:
           when: always
           command: |
             du -sh /tmp/test_output/*
-            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "safekeeper.log" ! -name "regression.diffs" ! -name "junit.xml" ! -name "*.filediff" ! -name "*.stdout" ! -name "*.stderr" -delete
+            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "wal_acceptor.log" -delete
             du -sh /tmp/test_output/*
       - store_artifacts:
           path: /tmp/test_output
@@ -272,85 +237,19 @@ jobs:
       - store_test_results:
           path: /tmp/test_output
 
-  # Build zenithdb/zenith:latest image and push it to Docker hub
-  docker-image:
-    docker:
-      - image: cimg/base:2021.04
-    steps:
-      - checkout
-      - setup_remote_docker:
-          docker_layer_caching: true
-      - run:
-          name: Init postgres submodule
-          command: git submodule update --init --depth 1
-      - run:
-          name: Build and push Docker image
-          command: |
-            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
-            docker build -t zenithdb/zenith:latest . && docker push zenithdb/zenith:latest
-
-  # Trigger a new remote CI job
-  remote-ci-trigger:
-    docker:
-      - image: cimg/base:2021.04
-    parameters:
-      remote_repo:
-        type: string
-    environment:
-      REMOTE_REPO: << parameters.remote_repo >>
-    steps:
-      - run:
-          name: Set PR's status to pending
-          command: |
-            LOCAL_REPO=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
-
-            curl -f -X POST \
-            https://api.github.com/repos/$LOCAL_REPO/statuses/$CIRCLE_SHA1 \
-            -H "Accept: application/vnd.github.v3+json" \
-            --user "$CI_ACCESS_TOKEN" \
-            --data \
-              "{
-                \"state\": \"pending\",
-                \"context\": \"zenith-remote-ci\",
-                \"description\": \"[$REMOTE_REPO] Remote CI job is about to start\"
-              }"
-      - run:
-          name: Request a remote CI test
-          command: |
-            LOCAL_REPO=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
-
-            curl -f -X POST \
-            https://api.github.com/repos/$REMOTE_REPO/actions/workflows/testing.yml/dispatches \
-            -H "Accept: application/vnd.github.v3+json" \
-            --user "$CI_ACCESS_TOKEN" \
-            --data \
-              "{
-                \"ref\": \"main\",
-                \"inputs\": {
-                  \"ci_job_name\": \"zenith-remote-ci\",
-                  \"commit_hash\": \"$CIRCLE_SHA1\",
-                  \"remote_repo\": \"$LOCAL_REPO\"
-                }
-              }"
-
 workflows:
   build_and_test:
     jobs:
-      - check-codestyle
-      - build-postgres:
-          name: build-postgres-<< matrix.build_type >>
-          matrix:
-            parameters:
-              build_type: ["debug", "release"]
+      - build-postgres
       - build-zenith:
           name: build-zenith-<< matrix.build_type >>
           matrix:
             parameters:
               build_type: ["debug", "release"]
           requires:
-            - build-postgres-<< matrix.build_type >>
+            - build-postgres
       - run-pytest:
-          name: pg_regress-tests-<< matrix.build_type >>
+          name: pg_regress tests << matrix.build_type >>
           matrix:
             parameters:
               build_type: ["debug", "release"]
@@ -359,39 +258,10 @@ workflows:
           requires:
             - build-zenith-<< matrix.build_type >>
       - run-pytest:
-          name: other-tests-<< matrix.build_type >>
+          name: other tests << matrix.build_type >>
           matrix:
             parameters:
               build_type: ["debug", "release"]
           test_selection: batch_others
           requires:
             - build-zenith-<< matrix.build_type >>
-      - run-pytest:
-          name: benchmarks
-          build_type: release
-          test_selection: performance
-          run_in_parallel: false
-          requires:
-            - build-zenith-release
-      - docker-image:
-          # Context gives an ability to login
-          context: Docker Hub
-          # Build image only for commits to main
-          filters:
-            branches:
-              only:
-                - main
-          requires:
-            - pg_regress-tests-release
-            - other-tests-release
-      - remote-ci-trigger:
-          # Context passes credentials for gh api
-          context: CI_ACCESS_TOKEN
-          remote_repo: "zenithdb/console"
-          requires:
-            # XXX: Successful build doesn't mean everything is OK, but
-            # the job to be triggered takes so much time to complete (~22 min)
-            # that it's better not to wait for the commented-out steps
-            - build-zenith-debug
-            # - pg_regress-tests-release
-            # - other-tests-release

.dockerignore

@@ -2,17 +2,12 @@
 **/__pycache__
 **/.pytest_cache
 
-.git
-target
-tmp_check
-tmp_install
-tmp_check_cli
-test_output
-.vscode
-.zenith
-integration_tests/.zenith
-.mypy_cache
-
-Dockerfile
-.dockerignore
-
+/target
+/tmp_check
+/tmp_install
+/tmp_check_cli
+/test_output
+/.vscode
+/.zenith
+/integration_tests/.zenith
+/Dockerfile

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "vendor/postgres"]
 	path = vendor/postgres
-	url = https://github.com/zenithdb/postgres
+	url = https://github.com/libzenith/postgres
 	branch = main

Cargo.toml

@@ -1,14 +1,13 @@
 [workspace]
 members = [
-    "control_plane",
     "pageserver",
-    "postgres_ffi",
-    "proxy",
     "walkeeper",
-    "workspace_hack",
     "zenith",
-    "zenith_metrics",
+    "control_plane",
+    "postgres_ffi",
     "zenith_utils",
+    "workspace_hack",
+    "proxy"
 ]
 
 [profile.release]

Dockerfile

@@ -1,61 +1,94 @@
 #
 # Docker image for console integration testing.
 #
+# We may also reuse it in CI to unify installation process and as a general binaries building
+# tool for production servers.
+#
+# Dynamic linking is used for librocksdb and libstdc++ bacause librocksdb-sys calls
+# bindgen with "dynamic" feature flag. This also prevents usage of dockerhub alpine-rust
+# images which are statically linked and have guards against any dlopen. I would rather
+# prefer all static binaries so we may change the way librocksdb-sys builds or wait until
+# we will have our own storage and drop rockdb dependency.
+#
+# Cargo-chef is used to separate dependencies building from main binaries building. This
+# way `docker build` will download and install dependencies only of there are changes to
+# out Cargo.toml files.
+#
+
 
 #
-# Build Postgres separately --- this layer will be rebuilt only if one of
-# mentioned paths will get any changes.
+# build postgres separately -- this layer will be rebuilt only if one of
+# mentioned paths will get any changes
 #
-FROM zenithdb/build:buster AS pg-build
-WORKDIR /zenith
+FROM alpine:3.13 as pg-build
+RUN apk add --update clang llvm compiler-rt compiler-rt-static lld musl-dev binutils \
+                     make bison flex readline-dev zlib-dev perl linux-headers
+WORKDIR zenith
 COPY ./vendor/postgres vendor/postgres
 COPY ./Makefile Makefile
-ENV BUILD_TYPE release
-RUN make -j $(getconf _NPROCESSORS_ONLN) -s postgres
-RUN rm -rf postgres_install/build
+# Build using clang and lld
+RUN CC='clang' LD='lld' CFLAGS='-fuse-ld=lld --rtlib=compiler-rt' make postgres -j4
+
+#
+# Calculate cargo dependencies.
+# This will always run, but only generate recipe.json with list of dependencies without
+# installing them.
+#
+FROM alpine:20210212 as cargo-deps-inspect
+RUN apk add --update rust cargo
+RUN cargo install cargo-chef
+WORKDIR zenith
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+#
+# Build cargo dependencies.
+# This temp cantainner would be build only if recipe.json was changed.
+#
+FROM alpine:20210212 as deps-build
+RUN apk add --update rust cargo openssl-dev clang build-base
+# rust-rocksdb can be built against system-wide rocksdb -- that saves about
+# 10 minutes during build. Rocksdb apk package is in testing now, but use it
+# anyway. In case of any troubles we can download and build rocksdb here manually
+# (to cache it as a docker layer).
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
+WORKDIR zenith
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=cargo-deps-inspect /root/.cargo/bin/cargo-chef /root/.cargo/bin/
+COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
+RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json
 
 #
 # Build zenith binaries
 #
-# TODO: build cargo deps as separate layer. We used cargo-chef before but that was
-# net time waste in a lot of cases. Copying Cargo.lock with empty lib.rs should do the work.
-#
-FROM zenithdb/build:buster AS build
-WORKDIR /zenith
-COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
-
+FROM alpine:20210212 as build
+RUN apk add --update rust cargo openssl-dev clang build-base
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
+WORKDIR zenith
 COPY . .
+# Copy cached dependencies
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=deps-build /zenith/target target
+COPY --from=deps-build /root/.cargo /root/.cargo
 RUN cargo build --release
 
 #
 # Copy binaries to resulting image.
+# build-base hare to provide libstdc++ (it will also bring gcc, but leave it this way until we figure
+# out how to statically link rocksdb or avoid it at all).
 #
-FROM debian:buster-slim
-WORKDIR /data
-
-RUN apt-get update && apt-get -yq install libreadline-dev libseccomp-dev openssl ca-certificates && \
-    mkdir zenith_install
-
+FROM alpine:3.13
+RUN apk add --update openssl build-base
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb
 COPY --from=build /zenith/target/release/pageserver /usr/local/bin
-COPY --from=build /zenith/target/release/safekeeper /usr/local/bin
-# TODO: temporary alias for compatibility, see https://github.com/zenithdb/zenith/pull/740
-RUN ln -s /usr/local/bin/safekeeper /usr/local/bin/wal_acceptor
+COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin
 COPY --from=build /zenith/target/release/proxy /usr/local/bin
-COPY --from=pg-build /zenith/tmp_install postgres_install
+COPY --from=pg-build /zenith/tmp_install /usr/local
 COPY docker-entrypoint.sh /docker-entrypoint.sh
 
-# Remove build artifacts (~ 500 MB)
-RUN rm -rf postgres_install/build && \
-    # 'Install' Postgres binaries locally
-    cp -r postgres_install/* /usr/local/ && \
-    # Prepare an archive of Postgres binaries (should be around 11 MB)
-    # and keep it inside container for an ease of deploy pipeline.
-    cd postgres_install && tar -czf /data/postgres_install.tar.gz . && cd .. && \
-    rm -rf postgres_install
-
-RUN useradd -d /data zenith && chown -R zenith:zenith /data
-
+RUN addgroup zenith && adduser -h /data -D -G zenith zenith
 VOLUME ["/data"]
+WORKDIR /data
 USER zenith
 EXPOSE 6400
 ENTRYPOINT ["/docker-entrypoint.sh"]

Dockerfile.alpine

@@ -1,97 +0,0 @@
-#
-# Docker image for console integration testing.
-#
-# We may also reuse it in CI to unify installation process and as a general binaries building
-# tool for production servers.
-#
-# Dynamic linking is used for librocksdb and libstdc++ bacause librocksdb-sys calls
-# bindgen with "dynamic" feature flag. This also prevents usage of dockerhub alpine-rust
-# images which are statically linked and have guards against any dlopen. I would rather
-# prefer all static binaries so we may change the way librocksdb-sys builds or wait until
-# we will have our own storage and drop rockdb dependency.
-#
-# Cargo-chef is used to separate dependencies building from main binaries building. This
-# way `docker build` will download and install dependencies only of there are changes to
-# out Cargo.toml files.
-#
-
-
-#
-# build postgres separately -- this layer will be rebuilt only if one of
-# mentioned paths will get any changes
-#
-FROM alpine:3.13 as pg-build
-RUN apk add --update clang llvm compiler-rt compiler-rt-static lld musl-dev binutils \
-                     make bison flex readline-dev zlib-dev perl linux-headers libseccomp-dev
-WORKDIR zenith
-COPY ./vendor/postgres vendor/postgres
-COPY ./Makefile Makefile
-# Build using clang and lld
-RUN CC='clang' LD='lld' CFLAGS='-fuse-ld=lld --rtlib=compiler-rt' make postgres -j4
-
-#
-# Calculate cargo dependencies.
-# This will always run, but only generate recipe.json with list of dependencies without
-# installing them.
-#
-FROM alpine:20210212 as cargo-deps-inspect
-RUN apk add --update rust cargo
-RUN cargo install cargo-chef
-WORKDIR zenith
-COPY . .
-RUN cargo chef prepare --recipe-path recipe.json
-
-#
-# Build cargo dependencies.
-# This temp cantainner would be build only if recipe.json was changed.
-#
-FROM alpine:20210212 as deps-build
-RUN apk add --update rust cargo openssl-dev clang build-base
-# rust-rocksdb can be built against system-wide rocksdb -- that saves about
-# 10 minutes during build. Rocksdb apk package is in testing now, but use it
-# anyway. In case of any troubles we can download and build rocksdb here manually
-# (to cache it as a docker layer).
-RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
-WORKDIR zenith
-COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
-COPY --from=cargo-deps-inspect /root/.cargo/bin/cargo-chef /root/.cargo/bin/
-COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
-RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json
-
-#
-# Build zenith binaries
-#
-FROM alpine:20210212 as build
-RUN apk add --update rust cargo openssl-dev clang build-base
-RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
-WORKDIR zenith
-COPY . .
-# Copy cached dependencies
-COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
-COPY --from=deps-build /zenith/target target
-COPY --from=deps-build /root/.cargo /root/.cargo
-RUN cargo build --release
-
-#
-# Copy binaries to resulting image.
-# build-base hare to provide libstdc++ (it will also bring gcc, but leave it this way until we figure
-# out how to statically link rocksdb or avoid it at all).
-#
-FROM alpine:3.13
-RUN apk add --update openssl build-base libseccomp-dev
-RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb
-COPY --from=build /zenith/target/release/pageserver /usr/local/bin
-COPY --from=build /zenith/target/release/safekeeper /usr/local/bin
-# TODO: temporary alias for compatibility, see https://github.com/zenithdb/zenith/pull/740
-RUN ln -s /usr/local/bin/safekeeper /usr/local/bin/wal_acceptor
-COPY --from=build /zenith/target/release/proxy /usr/local/bin
-COPY --from=pg-build /zenith/tmp_install /usr/local
-COPY docker-entrypoint.sh /docker-entrypoint.sh
-
-RUN addgroup zenith && adduser -h /data -D -G zenith zenith
-VOLUME ["/data"]
-WORKDIR /data
-USER zenith
-EXPOSE 6400
-ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD ["pageserver"]

Dockerfile.build

@@ -1,15 +0,0 @@
-#
-# Image with all the required dependencies to build https://github.com/zenithdb/zenith
-# and Postgres from https://github.com/zenithdb/postgres
-# Also includes some rust development and build tools.
-#
-FROM rust:slim-buster
-WORKDIR /zenith
-
-# Install postgres and zenith build dependencies
-# clang is for rocksdb
-RUN apt-get update && apt-get -yq install automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \
-                                          libseccomp-dev pkg-config libssl-dev clang
-
-# Install rust tools
-RUN rustup component add clippy && cargo install cargo-audit

Makefile

@@ -6,99 +6,63 @@ else
 	SECCOMP =
 endif
 
-#
-# We differentiate between release / debug build types using the BUILD_TYPE
-# environment variable.
-#
-BUILD_TYPE ?= debug
-ifeq ($(BUILD_TYPE),release)
-	PG_CONFIGURE_OPTS = --enable-debug
-	PG_CFLAGS = -O2 -g3 $(CFLAGS)
-	# Unfortunately, `--profile=...` is a nightly feature
-	CARGO_BUILD_FLAGS += --release
-else ifeq ($(BUILD_TYPE),debug)
-	PG_CONFIGURE_OPTS = --enable-debug --enable-cassert --enable-depend
-	PG_CFLAGS = -O0 -g3 $(CFLAGS)
-else
-$(error Bad build type `$(BUILD_TYPE)', see Makefile for options)
-endif
-
-# Choose whether we should be silent or verbose
-CARGO_BUILD_FLAGS += --$(if $(filter s,$(MAKEFLAGS)),quiet,verbose)
-# Fix for a corner case when make doesn't pass a jobserver
-CARGO_BUILD_FLAGS += $(filter -j1,$(MAKEFLAGS))
-
-# This option has a side effect of passing make jobserver to cargo.
-# However, we shouldn't do this if `make -n` (--dry-run) has been asked.
-CARGO_CMD_PREFIX += $(if $(filter n,$(MAKEFLAGS)),,+)
-# Force cargo not to print progress bar
-CARGO_CMD_PREFIX += CARGO_TERM_PROGRESS_WHEN=never CI=1
-
 #
 # Top level Makefile to build Zenith and PostgreSQL
 #
-.PHONY: all
 all: zenith postgres
 
+# We don't want to run 'cargo build' in parallel with the postgres build,
+# because interleaving cargo build output with postgres build output looks
+# confusing. Also, 'cargo build' is parallel on its own, so it would be too
+# much parallelism. (Recursive invocation of postgres target still gets any
+# '-j' flag from the command line, so 'make -j' is still useful.)
+.NOTPARALLEL:
+
 ### Zenith Rust bits
 #
 # The 'postgres_ffi' depends on the Postgres headers.
-.PHONY: zenith
 zenith: postgres-headers
-	+@echo "Compiling Zenith"
-	$(CARGO_CMD_PREFIX) cargo build $(CARGO_BUILD_FLAGS)
+	cargo build
 
 ### PostgreSQL parts
 tmp_install/build/config.status:
 	+@echo "Configuring postgres build"
 	mkdir -p tmp_install/build
 	(cd tmp_install/build && \
-	../../vendor/postgres/configure CFLAGS='$(PG_CFLAGS)' \
-		$(PG_CONFIGURE_OPTS) \
+	../../vendor/postgres/configure CFLAGS='-O0 -g3 $(CFLAGS)' \
+		--enable-cassert \
+		--enable-debug \
+		--enable-depend \
 		$(SECCOMP) \
 		--prefix=$(abspath tmp_install) > configure.log)
 
 # nicer alias for running 'configure'
-.PHONY: postgres-configure
 postgres-configure: tmp_install/build/config.status
 
 # Install the PostgreSQL header files into tmp_install/include
-.PHONY: postgres-headers
 postgres-headers: postgres-configure
 	+@echo "Installing PostgreSQL headers"
 	$(MAKE) -C tmp_install/build/src/include MAKELEVEL=0 install
 
+
 # Compile and install PostgreSQL and contrib/zenith
-.PHONY: postgres
-postgres: postgres-configure \
-		  postgres-headers # to prevent `make install` conflicts with zenith's `postgres-headers`
+postgres: postgres-configure
 	+@echo "Compiling PostgreSQL"
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 install
 	+@echo "Compiling contrib/zenith"
 	$(MAKE) -C tmp_install/build/contrib/zenith install
-	+@echo "Compiling contrib/zenith_test_utils"
-	$(MAKE) -C tmp_install/build/contrib/zenith_test_utils install
 
-.PHONY: postgres-clean
 postgres-clean:
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 clean
 
 # This doesn't remove the effects of 'configure'.
-.PHONY: clean
 clean:
-	cd tmp_install/build && $(MAKE) clean
-	$(CARGO_CMD_PREFIX) cargo clean
+	cd tmp_install/build && ${MAKE} clean
+	cargo clean
 
 # This removes everything
-.PHONY: distclean
 distclean:
 	rm -rf tmp_install
-	$(CARGO_CMD_PREFIX) cargo clean
+	cargo clean
 
-.PHONY: fmt
-fmt:
-	./pre-commit.py --fix-inplace
-
-.PHONY: setup-pre-commit-hook
-setup-pre-commit-hook:
-	ln -s -f ../../pre-commit.py .git/hooks/pre-commit
+.PHONY: postgres-configure postgres postgres-headers zenith

README.md

@@ -2,22 +2,6 @@
 
 Zenith substitutes PostgreSQL storage layer and redistributes data across a cluster of nodes
 
-## Architecture overview
-
-A Zenith installation consists of Compute nodes and Storage engine.
-
-Compute nodes are stateless PostgreSQL nodes, backed by zenith storage.
-
-Zenith storage engine consists of two major components:
-- Pageserver. Scalable storage backend for compute nodes.
-- WAL service. The service that receives WAL from compute node and ensures that it is stored durably.
-
-Pageserver consists of:
-- Repository - Zenith storage implementation.
-- WAL receiver - service that receives WAL from WAL service and stores it in the repository.
-- Page service - service that communicates with compute nodes and responds with pages from the repository.
-- WAL redo - service that builds pages from base images and WAL records on Page service request.
-
 ## Running local installation
 
 1. Install build dependencies and other useful packages
@@ -25,19 +9,22 @@ Pageserver consists of:
 On Ubuntu or Debian this set of packages should be sufficient to build the code:
 ```text
 apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libseccomp-dev \
-libssl-dev clang pkg-config libpq-dev
+libssl-dev clang
 ```
 
-[Rust] 1.52 or later is also required.
+[Rust] 1.48 or later is also required.
 
 To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively.
 
 To run the integration tests (not required to use the code), install
-Python (3.6 or higher), and install python3 packages with `pipenv` using `pipenv install` in the project directory.
+Python (3.6 or higher), and install python3 packages with `pip` (called `pip3` on some systems):
+```
+pip install pytest psycopg2
+```
 
 2. Build zenith and patched postgres
 ```sh
-git clone --recursive https://github.com/zenithdb/zenith.git
+git clone --recursive https://github.com/libzenith/zenith.git
 cd zenith
 make -j5
 ```
@@ -47,7 +34,8 @@ make -j5
 # Create repository in .zenith with proper paths to binaries and data
 # Later that would be responsibility of a package install script
 > ./target/debug/zenith init
-pageserver init succeeded
+<...>
+new zenith repository was created in .zenith
 
 # start pageserver
 > ./target/debug/zenith start
@@ -67,7 +55,7 @@ main	127.0.0.1:55432	0/1609610	running
 
 4. Now it is possible to connect to postgres and run some queries:
 ```text
-> psql -p55432 -h 127.0.0.1 -U zenith_admin postgres
+> psql -p55432 -h 127.0.0.1 postgres
 postgres=# CREATE TABLE t(key int primary key, value text);
 CREATE TABLE
 postgres=# insert into t values(1,1);
@@ -97,9 +85,9 @@ waiting for server to start.... done
 
 # this new postgres instance will have all the data from 'main' postgres,
 # but all modifications would not affect data in original postgres
-> psql -p55433 -h 127.0.0.1 -U zenith_admin postgres
+> psql -p55433 -h 127.0.0.1 postgres
 postgres=# select * from t;
- key | value
+ key | value 
 -----+-------
    1 | 1
 (1 row)
@@ -108,17 +96,10 @@ postgres=# insert into t values(2,2);
 INSERT 0 1
 ```
 
-6. If you want to run tests afterwards (see below), you have to stop pageserver and all postgres instances you have just started:
-```sh
-> ./target/debug/zenith pg stop migration_check
-> ./target/debug/zenith pg stop main
-> ./target/debug/zenith stop
-```
-
 ## Running tests
 
 ```sh
-git clone --recursive https://github.com/zenithdb/zenith.git
+git clone --recursive https://github.com/libzenith/zenith.git
 make # builds also postgres and installs it to ./tmp_install
 cd test_runner
 pytest
@@ -126,25 +107,61 @@ pytest
 
 ## Documentation
 
-Now we use README files to cover design ideas and overall architecture for each module and `rustdoc` style documentation comments. See also [/docs/](/docs/) a top-level overview of all available markdown documentation.
+Now we use README files to cover design ideas and overall architecture for each module.
+And rustdoc style documentation comments.
 
-- [/docs/sourcetree.md](/docs/sourcetree.md) contains overview of source tree layout.
+To view your documentation in a browser, try running `cargo doc --no-deps --open`
 
-To view your `rustdoc` documentation in a browser, try running `cargo doc --no-deps --open`
+## Source tree layout
 
-### Postgres-specific terms
+`/control_plane`:
 
-Due to Zenith's very close relation with PostgreSQL internals, there are numerous specific terms used.
-Same applies to certain spelling: i.e. we use MB to denote 1024 * 1024 bytes, while MiB would be technically more correct, it's inconsistent with what PostgreSQL code and its documentation use.
+Local control plane.
+Functions to start, configure and stop pageserver and postgres instances running as a local processes.
+Intended to be used in integration tests and in CLI tools for local installations.
 
-To get more familiar with this aspect, refer to:
+`/zenith`
 
-- [Zenith glossary](/docs/glossary.md)
-- [PostgreSQL glossary](https://www.postgresql.org/docs/13/glossary.html)
-- Other PostgreSQL documentation and sources (Zenith fork sources can be found [here](https://github.com/zenithdb/postgres))
+Main entry point for the 'zenith' CLI utility.
+TODO: Doesn't it belong to control_plane?
 
-## Join the development
+`/postgres_ffi`:
 
-- Read `CONTRIBUTING.md` to learn about project code style and practices.
-- To get familiar with a source tree layout, use [/docs/sourcetree.md](/docs/sourcetree.md).
-- To learn more about PostgreSQL internals, check http://www.interdb.jp/pg/index.html
+Utility functions for interacting with PostgreSQL file formats.
+Misc constants, copied from PostgreSQL headers.
+
+`/zenith_utils`:
+
+Helpers that are shared between other crates in this repository.
+
+`/walkeeper`:
+
+WAL safekeeper (also known as WAL acceptor). Written in Rust.
+
+`/pageserver`:
+
+Page Server. Written in Rust.
+
+Depends on the modified 'postgres' binary for WAL redo.
+
+`/vendor/postgres`:
+
+PostgreSQL source tree, with the modifications needed for Zenith.
+
+`/vendor/postgres/contrib/zenith`:
+
+PostgreSQL extension that implements storage manager API and network communications with remote page server.
+
+`/test_runner`:
+
+Integration tests, written in Python using the `pytest` framework.
+
+`test_runner/zenith_regress`:
+
+Quick way to add new SQL regression test to integration tests set.
+
+`/integration_tests`:
+
+Another pack of integration tests. Written in Rust.
+
+[Rust]: https://www.rust-lang.org/learn/get-started

control_plane/Cargo.toml

@@ -16,12 +16,9 @@ toml = "0.5"
 lazy_static = "1.4"
 regex = "1"
 anyhow = "1.0"
-thiserror = "1"
 bytes = "1.0.1"
 nix = "0.20"
 url = "2.2.2"
-hex = { version = "0.4.3", features = ["serde"] }
-reqwest = { version = "0.11", features = ["blocking", "json"] }
 
 pageserver = { path = "../pageserver" }
 walkeeper = { path = "../walkeeper" }

control_plane/src/compute.rs

@@ -1,24 +1,24 @@
-use std::collections::BTreeMap;
-use std::fs::{self, File};
 use std::io::Write;
 use std::net::SocketAddr;
 use std::net::TcpStream;
 use std::os::unix::fs::PermissionsExt;
-use std::path::PathBuf;
-use std::process::{Command, Stdio};
-use std::str::FromStr;
+use std::process::Command;
 use std::sync::Arc;
 use std::time::Duration;
+use std::{collections::BTreeMap, path::PathBuf};
+use std::{
+    fs::{self, OpenOptions},
+    io::Read,
+};
 
 use anyhow::{Context, Result};
+use lazy_static::lazy_static;
+use regex::Regex;
 use zenith_utils::connstring::connection_host_port;
-use zenith_utils::lsn::Lsn;
-use zenith_utils::postgres_backend::AuthType;
-use zenith_utils::zid::ZTenantId;
-use zenith_utils::zid::ZTimelineId;
 
 use crate::local_env::LocalEnv;
-use crate::postgresql_conf::PostgresConf;
+use pageserver::ZTimelineId;
+
 use crate::storage::PageServerNode;
 
 //
@@ -27,36 +27,27 @@ use crate::storage::PageServerNode;
 pub struct ComputeControlPlane {
     base_port: u16,
     pageserver: Arc<PageServerNode>,
-    pub nodes: BTreeMap<(ZTenantId, String), Arc<PostgresNode>>,
+    pub nodes: BTreeMap<String, Arc<PostgresNode>>,
     env: LocalEnv,
 }
 
 impl ComputeControlPlane {
     // Load current nodes with ports from data directories on disk
-    // Directory structure has the following layout:
-    // pgdatadirs
-    // |- tenants
-    // |  |- <tenant_id>
-    // |  |   |- <branch name>
     pub fn load(env: LocalEnv) -> Result<ComputeControlPlane> {
         // TODO: since pageserver do not have config file yet we believe here that
         // it is running on default port. Change that when pageserver will have config.
         let pageserver = Arc::new(PageServerNode::from_env(&env));
 
-        let mut nodes = BTreeMap::default();
         let pgdatadirspath = &env.pg_data_dirs_path();
-
-        for tenant_dir in fs::read_dir(&pgdatadirspath)
+        let nodes: Result<BTreeMap<_, _>> = fs::read_dir(&pgdatadirspath)
             .with_context(|| format!("failed to list {}", pgdatadirspath.display()))?
-        {
-            let tenant_dir = tenant_dir?;
-            for timeline_dir in fs::read_dir(tenant_dir.path())
-                .with_context(|| format!("failed to list {}", tenant_dir.path().display()))?
-            {
-                let node = PostgresNode::from_dir_entry(timeline_dir?, &env, &pageserver)?;
-                nodes.insert((node.tenantid, node.name.clone()), Arc::new(node));
-            }
-        }
+            .into_iter()
+            .map(|f| {
+                PostgresNode::from_dir_entry(f?, &env, &pageserver)
+                    .map(|node| (node.name.clone(), Arc::new(node)))
+            })
+            .collect();
+        let nodes = nodes?;
 
         Ok(ComputeControlPlane {
             base_port: 55431,
@@ -84,34 +75,47 @@ impl ComputeControlPlane {
         }
     }
 
-    pub fn new_node(
+    /// Connect to a page server, get base backup, and untar it to initialize a
+    /// new data directory
+    pub fn new_from_page_server(
         &mut self,
-        tenantid: ZTenantId,
-        branch_name: &str,
-        port: Option<u16>,
+        is_test: bool,
+        timelineid: ZTimelineId,
+        name: &str,
     ) -> Result<Arc<PostgresNode>> {
-        let timeline_id = self
-            .pageserver
-            .branch_get_by_name(&tenantid, branch_name)?
-            .timeline_id;
-
-        let port = port.unwrap_or_else(|| self.get_port());
         let node = Arc::new(PostgresNode {
-            name: branch_name.to_owned(),
-            address: SocketAddr::new("127.0.0.1".parse().unwrap(), port),
+            name: name.to_owned(),
+            address: SocketAddr::new("127.0.0.1".parse().unwrap(), self.get_port()),
             env: self.env.clone(),
             pageserver: Arc::clone(&self.pageserver),
-            is_test: false,
-            timelineid: timeline_id,
-            tenantid,
-            uses_wal_proposer: false,
+            is_test,
+            timelineid,
         });
 
-        node.create_pgdata()?;
-        node.setup_pg_conf(self.env.auth_type)?;
+        node.init_from_page_server()?;
+        self.nodes.insert(node.name.clone(), Arc::clone(&node));
 
-        self.nodes
-            .insert((tenantid, node.name.clone()), Arc::clone(&node));
+        Ok(node)
+    }
+
+    pub fn new_node(&mut self, branch_name: &str) -> Result<Arc<PostgresNode>> {
+        let timeline_id = self.pageserver.branch_get_by_name(branch_name)?.timeline_id;
+
+        let node = self.new_from_page_server(false, timeline_id, branch_name)?;
+
+        // Configure the node to stream WAL directly to the pageserver
+        node.append_conf(
+            "postgresql.conf",
+            format!(
+                concat!(
+                    "shared_preload_libraries = zenith\n",
+                    "synchronous_standby_names = 'pageserver'\n", // TODO: add a new function arg?
+                    "zenith.callmemaybe_connstring = '{}'\n",     // FIXME escaping
+                ),
+                node.connstr()
+            )
+            .as_str(),
+        )?;
 
         Ok(node)
     }
@@ -119,7 +123,6 @@ impl ComputeControlPlane {
 
 ///////////////////////////////////////////////////////////////////////////////
 
-#[derive(Debug)]
 pub struct PostgresNode {
     pub address: SocketAddr,
     name: String,
@@ -127,8 +130,6 @@ pub struct PostgresNode {
     pageserver: Arc<PageServerNode>,
     is_test: bool,
     pub timelineid: ZTimelineId,
-    pub tenantid: ZTenantId,
-    uses_wal_proposer: bool,
 }
 
 impl PostgresNode {
@@ -144,25 +145,56 @@ impl PostgresNode {
             );
         }
 
+        lazy_static! {
+            static ref CONF_PORT_RE: Regex = Regex::new(r"(?m)^\s*port\s*=\s*(\d+)\s*$").unwrap();
+            static ref CONF_TIMELINE_RE: Regex =
+                Regex::new(r"(?m)^\s*zenith.zenith_timeline\s*=\s*'(\w+)'\s*$").unwrap();
+        }
+
         // parse data directory name
         let fname = entry.file_name();
         let name = fname.to_str().unwrap().to_string();
 
-        // Read config file into memory
+        // find out tcp port in config file
         let cfg_path = entry.path().join("postgresql.conf");
-        let cfg_path_str = cfg_path.to_string_lossy();
-        let mut conf_file = File::open(&cfg_path)
-            .with_context(|| format!("failed to open config file in {}", cfg_path_str))?;
-        let conf = PostgresConf::read(&mut conf_file)
-            .with_context(|| format!("failed to read config file in {}", cfg_path_str))?;
+        let config = fs::read_to_string(cfg_path.clone()).with_context(|| {
+            format!(
+                "failed to read config file in {}",
+                cfg_path.to_str().unwrap()
+            )
+        })?;
 
-        // Read a few options from the config file
-        let context = format!("in config file {}", cfg_path_str);
-        let port: u16 = conf.parse_field("port", &context)?;
-        let timelineid: ZTimelineId = conf.parse_field("zenith.zenith_timeline", &context)?;
-        let tenantid: ZTenantId = conf.parse_field("zenith.zenith_tenant", &context)?;
+        // parse port
+        let err_msg = format!(
+            "failed to find port definition in config file {}",
+            cfg_path.to_str().unwrap()
+        );
+        let port: u16 = CONF_PORT_RE
+            .captures(config.as_str())
+            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 1"))?
+            .iter()
+            .last()
+            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 2"))?
+            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 3"))?
+            .as_str()
+            .parse()
+            .with_context(|| err_msg)?;
 
-        let uses_wal_proposer = conf.get("wal_acceptors").is_some();
+        // parse timeline
+        let err_msg = format!(
+            "failed to find timeline definition in config file {}",
+            cfg_path.to_str().unwrap()
+        );
+        let timelineid: ZTimelineId = CONF_TIMELINE_RE
+            .captures(config.as_str())
+            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 1"))?
+            .iter()
+            .last()
+            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 2"))?
+            .ok_or_else(|| anyhow::Error::msg(err_msg.clone() + " 3"))?
+            .as_str()
+            .parse()
+            .with_context(|| err_msg)?;
 
         // ok now
         Ok(PostgresNode {
@@ -172,170 +204,110 @@ impl PostgresNode {
             pageserver: Arc::clone(pageserver),
             is_test: false,
             timelineid,
-            tenantid,
-            uses_wal_proposer,
         })
     }
 
-    fn sync_walkeepers(&self) -> Result<Lsn> {
-        let pg_path = self.env.pg_bin_dir().join("postgres");
-        let sync_handle = Command::new(pg_path)
-            .arg("--sync-safekeepers")
-            .env_clear()
-            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-            .env("PGDATA", self.pgdata().to_str().unwrap())
-            .stdout(Stdio::piped())
-            // Comment this to avoid capturing stderr (useful if command hangs)
-            .stderr(Stdio::piped())
-            .spawn()
-            .expect("postgres --sync-safekeepers failed to start");
+    // Connect to a page server, get base backup, and untar it to initialize a
+    // new data directory
+    pub fn init_from_page_server(&self) -> Result<()> {
+        let pgdata = self.pgdata();
 
-        let sync_output = sync_handle
-            .wait_with_output()
-            .expect("postgres --sync-safekeepers failed");
-        if !sync_output.status.success() {
-            anyhow::bail!(
-                "sync-safekeepers failed: '{}'",
-                String::from_utf8_lossy(&sync_output.stderr)
-            );
-        }
-
-        let lsn = Lsn::from_str(std::str::from_utf8(&sync_output.stdout)?.trim())?;
-        println!("Walkeepers synced on {}", lsn);
-        Ok(lsn)
-    }
-
-    /// Get basebackup from the pageserver as a tar archive and extract it
-    /// to the `self.pgdata()` directory.
-    fn do_basebackup(&self, lsn: Option<Lsn>) -> Result<()> {
         println!(
             "Extracting base backup to create postgres instance: path={} port={}",
-            self.pgdata().display(),
+            pgdata.display(),
             self.address.port()
         );
 
-        let sql = if let Some(lsn) = lsn {
-            format!("basebackup {} {} {}", self.tenantid, self.timelineid, lsn)
-        } else {
-            format!("basebackup {} {}", self.tenantid, self.timelineid)
-        };
+        // initialize data directory
+        if self.is_test {
+            fs::remove_dir_all(&pgdata).ok();
+        }
 
+        let sql = format!("basebackup {}", self.timelineid);
         let mut client = self
             .pageserver
             .page_server_psql_client()
             .with_context(|| "connecting to page server failed")?;
 
-        let copyreader = client
+        fs::create_dir_all(&pgdata)
+            .with_context(|| format!("could not create data directory {}", pgdata.display()))?;
+        fs::set_permissions(pgdata.as_path(), fs::Permissions::from_mode(0o700)).with_context(
+            || {
+                format!(
+                    "could not set permissions in data directory {}",
+                    pgdata.display()
+                )
+            },
+        )?;
+
+        // FIXME: The compute node should be able to stream the WAL it needs from the WAL safekeepers or archive.
+        // But that's not implemented yet. For now, 'pg_wal' is included in the base backup tarball that
+        // we receive from the Page Server, so we don't need to create the empty 'pg_wal' directory here.
+        //fs::create_dir_all(pgdata.join("pg_wal"))?;
+
+        let mut copyreader = client
             .copy_out(sql.as_str())
             .with_context(|| "page server 'basebackup' command failed")?;
 
-        // Read the archive directly from the `CopyOutReader`
-        tar::Archive::new(copyreader)
-            .unpack(&self.pgdata())
+        // FIXME: Currently, we slurp the whole tarball into memory, and then extract it,
+        // but we really should do this:
+        //let mut ar = tar::Archive::new(copyreader);
+        let mut buf = vec![];
+        copyreader
+            .read_to_end(&mut buf)
+            .with_context(|| "reading base backup from page server failed")?;
+        let mut ar = tar::Archive::new(buf.as_slice());
+        ar.unpack(&pgdata)
             .with_context(|| "extracting page backup failed")?;
 
-        Ok(())
-    }
-
-    fn create_pgdata(&self) -> Result<()> {
-        fs::create_dir_all(&self.pgdata()).with_context(|| {
-            format!(
-                "could not create data directory {}",
-                self.pgdata().display()
-            )
-        })?;
-        fs::set_permissions(self.pgdata().as_path(), fs::Permissions::from_mode(0o700))
-            .with_context(|| {
-                format!(
-                    "could not set permissions in data directory {}",
-                    self.pgdata().display()
-                )
-            })
-    }
-
-    // Connect to a page server, get base backup, and untar it to initialize a
-    // new data directory
-    fn setup_pg_conf(&self, auth_type: AuthType) -> Result<()> {
-        let mut conf = PostgresConf::new();
-        conf.append("max_wal_senders", "10");
         // wal_log_hints is mandatory when running against pageserver (see gh issue#192)
         // TODO: is it possible to check wal_log_hints at pageserver side via XLOG_PARAMETER_CHANGE?
-        conf.append("wal_log_hints", "on");
-        conf.append("max_replication_slots", "10");
-        conf.append("hot_standby", "on");
-        conf.append("shared_buffers", "1MB");
-        conf.append("fsync", "off");
-        conf.append("max_connections", "100");
-        conf.append("wal_sender_timeout", "0");
-        conf.append("wal_level", "replica");
-        conf.append("listen_addresses", &self.address.ip().to_string());
-        conf.append("port", &self.address.port().to_string());
+        self.append_conf(
+            "postgresql.conf",
+            &format!(
+                "max_wal_senders = 10\n\
+                 wal_log_hints = on\n\
+                 max_replication_slots = 10\n\
+                 hot_standby = on\n\
+                 shared_buffers = 1MB\n\
+                 fsync = off\n\
+                 max_connections = 100\n\
+                 wal_sender_timeout = 0\n\
+                 wal_level = replica\n\
+                 listen_addresses = '{address}'\n\
+                 port = {port}\n",
+                address = self.address.ip(),
+                port = self.address.port()
+            ),
+        )?;
 
         // Never clean up old WAL. TODO: We should use a replication
         // slot or something proper, to prevent the compute node
-        // from removing WAL that hasn't been streamed to the safekeeper or
-        // page server yet. (gh issue #349)
-        conf.append("wal_keep_size", "10TB");
+        // from removing WAL that hasn't been streamed to the safekeepr or
+        // page server yet. But this will do for now.
+        self.append_conf("postgresql.conf", "wal_keep_size='10TB'\n")?;
 
-        // Configure the node to fetch pages from pageserver
-        let pageserver_connstr = {
-            let (host, port) = connection_host_port(&self.pageserver.pg_connection_config);
+        // Connect it to the page server.
 
-            // Set up authentication
-            //
-            // $ZENITH_AUTH_TOKEN will be replaced with value from environment
-            // variable during compute pg startup. It is done this way because
-            // otherwise user will be able to retrieve the value using SHOW
-            // command or pg_settings
-            let password = if let AuthType::ZenithJWT = auth_type {
-                "$ZENITH_AUTH_TOKEN"
-            } else {
-                ""
-            };
-
-            format!("host={} port={} password={}", host, port, password)
-        };
-        conf.append("shared_preload_libraries", "zenith");
-        conf.append_line("");
-        conf.append("zenith.page_server_connstring", &pageserver_connstr);
-        conf.append("zenith.zenith_tenant", &self.tenantid.to_string());
-        conf.append("zenith.zenith_timeline", &self.timelineid.to_string());
-        conf.append_line("");
-
-        // Configure the node to stream WAL directly to the pageserver
-        conf.append("synchronous_standby_names", "pageserver"); // TODO: add a new function arg?
-        conf.append("zenith.callmemaybe_connstring", &self.connstr());
-
-        let mut file = File::create(self.pgdata().join("postgresql.conf"))?;
-        file.write_all(conf.to_string().as_bytes())?;
-
-        Ok(())
-    }
-
-    fn load_basebackup(&self) -> Result<()> {
-        let lsn = if self.uses_wal_proposer {
-            // LSN 0 means that it is bootstrap and we need to download just
-            // latest data from the pageserver. That is a bit clumsy but whole bootstrap
-            // procedure evolves quite actively right now, so let's think about it again
-            // when things would be more stable (TODO).
-            let lsn = self.sync_walkeepers()?;
-            if lsn == Lsn(0) {
-                None
-            } else {
-                Some(lsn)
-            }
-        } else {
-            None
-        };
-
-        self.do_basebackup(lsn)?;
+        // Configure that node to take pages from pageserver
+        let (host, port) = connection_host_port(&self.pageserver.connection_config());
+        self.append_conf(
+            "postgresql.conf",
+            &format!(
+                "shared_preload_libraries = zenith \n\
+                 zenith.page_server_connstring = 'host={} port={}'\n\
+                 zenith.zenith_timeline='{}'\n",
+                host, port, self.timelineid
+            ),
+        )?;
 
+        fs::create_dir_all(self.pgdata().join("pg_wal"))?;
+        fs::create_dir_all(self.pgdata().join("pg_wal").join("archive_status"))?;
         Ok(())
     }
 
     pub fn pgdata(&self) -> PathBuf {
-        self.env.pg_data_dir(&self.tenantid, &self.name)
+        self.env.pg_data_dir(&self.name)
     }
 
     pub fn status(&self) -> &str {
@@ -351,98 +323,69 @@ impl PostgresNode {
         }
     }
 
-    fn pg_ctl(&self, args: &[&str], auth_token: &Option<String>) -> Result<()> {
+    pub fn append_conf(&self, config: &str, opts: &str) -> Result<()> {
+        OpenOptions::new()
+            .append(true)
+            .open(self.pgdata().join(config).to_str().unwrap())?
+            .write_all(opts.as_bytes())?;
+        Ok(())
+    }
+
+    fn pg_ctl(&self, args: &[&str]) -> Result<()> {
         let pg_ctl_path = self.env.pg_bin_dir().join("pg_ctl");
-        let mut cmd = Command::new(pg_ctl_path);
-        cmd.args(
-            [
-                &[
-                    "-D",
-                    self.pgdata().to_str().unwrap(),
-                    "-l",
-                    self.pgdata().join("pg.log").to_str().unwrap(),
-                    "-w", //wait till pg_ctl actually does what was asked
-                ],
-                args,
-            ]
-            .concat(),
-        )
-        .env_clear()
-        .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
-        .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap());
-
-        if let Some(token) = auth_token {
-            cmd.env("ZENITH_AUTH_TOKEN", token);
-        }
-        let pg_ctl = cmd.status().with_context(|| "pg_ctl failed")?;
 
+        let pg_ctl = Command::new(pg_ctl_path)
+            .args(
+                [
+                    &[
+                        "-D",
+                        self.pgdata().to_str().unwrap(),
+                        "-l",
+                        self.pgdata().join("pg.log").to_str().unwrap(),
+                        "-w", //wait till pg_ctl actually does what was asked
+                    ],
+                    args,
+                ]
+                .concat(),
+            )
+            .env_clear()
+            .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
+            .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
+            .status()
+            .with_context(|| "pg_ctl failed")?;
         if !pg_ctl.success() {
             anyhow::bail!("pg_ctl failed");
         }
         Ok(())
     }
 
-    pub fn start(&self, auth_token: &Option<String>) -> Result<()> {
-        // Bail if the node already running.
-        if self.status() == "running" {
-            anyhow::bail!("The node is already running");
-        }
-
-        // 1. We always start compute node from scratch, so
-        // if old dir exists, preserve 'postgresql.conf' and drop the directory
-        let postgresql_conf_path = self.pgdata().join("postgresql.conf");
-        let postgresql_conf = fs::read(&postgresql_conf_path).with_context(|| {
-            format!(
-                "failed to read config file in {}",
-                postgresql_conf_path.to_str().unwrap()
-            )
-        })?;
-        fs::remove_dir_all(&self.pgdata())?;
-        self.create_pgdata()?;
-
-        // 2. Bring back config files
-        fs::write(&postgresql_conf_path, postgresql_conf)?;
-
-        // 3. Load basebackup
-        self.load_basebackup()?;
-
-        // 4. Finally start the compute node postgres
+    pub fn start(&self) -> Result<()> {
         println!("Starting postgres node at '{}'", self.connstr());
-        self.pg_ctl(&["start"], auth_token)
+        self.pg_ctl(&["start"])
     }
 
-    pub fn restart(&self, auth_token: &Option<String>) -> Result<()> {
-        self.pg_ctl(&["restart"], auth_token)
+    pub fn restart(&self) -> Result<()> {
+        self.pg_ctl(&["restart"])
     }
 
     pub fn stop(&self, destroy: bool) -> Result<()> {
-        // If we are going to destroy data directory,
-        // use immediate shutdown mode, otherwise,
-        // shutdown gracefully to leave the data directory sane.
-        //
-        // Compute node always starts from scratch, so stop
-        // without destroy only used for testing and debugging.
-        //
+        self.pg_ctl(&["-m", "immediate", "stop"])?;
         if destroy {
-            self.pg_ctl(&["-m", "immediate", "stop"], &None)?;
             println!(
                 "Destroying postgres data directory '{}'",
                 self.pgdata().to_str().unwrap()
             );
             fs::remove_dir_all(&self.pgdata())?;
-        } else {
-            self.pg_ctl(&["stop"], &None)?;
         }
         Ok(())
     }
 
     pub fn connstr(&self) -> String {
         format!(
-            "host={} port={} user={} dbname={}",
+            "host={} port={} user={}",
             self.address.ip(),
             self.address.port(),
-            "zenith_admin",
-            "postgres"
+            self.whoami()
         )
     }
 
@@ -452,7 +395,9 @@ impl PostgresNode {
             .output()
             .expect("failed to execute whoami");
 
-        assert!(output.status.success(), "whoami failed");
+        if !output.status.success() {
+            panic!("whoami failed");
+        }
 
         String::from_utf8(output.stdout).unwrap().trim().to_string()
     }

control_plane/src/lib.rs

@@ -12,7 +12,6 @@ use std::path::Path;
 
 pub mod compute;
 pub mod local_env;
-pub mod postgresql_conf;
 pub mod storage;
 
 /// Read a PID file

control_plane/src/local_env.rs

@@ -4,24 +4,22 @@
 // Now it also provides init method which acts like a stub for proper installation
 // script which will use local paths.
 //
-use anyhow::{Context, Result};
+use anyhow::{anyhow, Result};
 use serde::{Deserialize, Serialize};
-use std::env;
 use std::fs;
 use std::path::PathBuf;
-use std::process::{Command, Stdio};
-use zenith_utils::auth::{encode_from_key_path, Claims, Scope};
-use zenith_utils::postgres_backend::AuthType;
-use zenith_utils::zid::ZTenantId;
+use std::{collections::BTreeMap, env};
+use url::Url;
+
+pub type Remotes = BTreeMap<String, String>;
 
 //
 // This data structures represent deserialized zenith CLI config
 //
-#[derive(Serialize, Deserialize, Clone, Debug)]
+#[derive(Serialize, Deserialize, Clone)]
 pub struct LocalEnv {
-    // Pageserver connection settings
-    pub pageserver_pg_port: u16,
-    pub pageserver_http_port: u16,
+    // Pageserver connection strings
+    pub pageserver_connstring: String,
 
     // Base directory for both pageserver and compute nodes
     pub base_data_dir: PathBuf,
@@ -32,21 +30,10 @@ pub struct LocalEnv {
     // to four separate paths and match OS-specific installation layout.
     pub pg_distrib_dir: PathBuf,
 
-    // Path to pageserver binary.
-    pub zenith_distrib_dir: PathBuf,
+    // Path to pageserver binary. Empty for remote pageserver.
+    pub zenith_distrib_dir: Option<PathBuf>,
 
-    // keeping tenant id in config to reduce copy paste when running zenith locally with single tenant
-    #[serde(with = "hex")]
-    pub tenantid: ZTenantId,
-
-    // jwt auth token used for communication with pageserver
-    pub auth_token: String,
-
-    // used to determine which auth type is used
-    pub auth_type: AuthType,
-
-    // used to issue tokens during e.g pg start
-    pub private_key_path: PathBuf,
+    pub remotes: Remotes,
 }
 
 impl LocalEnv {
@@ -59,17 +46,19 @@ impl LocalEnv {
     }
 
     pub fn pageserver_bin(&self) -> Result<PathBuf> {
-        Ok(self.zenith_distrib_dir.join("pageserver"))
+        Ok(self
+            .zenith_distrib_dir
+            .as_ref()
+            .ok_or_else(|| anyhow!("Can not manage remote pageserver"))?
+            .join("pageserver"))
     }
 
     pub fn pg_data_dirs_path(&self) -> PathBuf {
-        self.base_data_dir.join("pgdatadirs").join("tenants")
+        self.base_data_dir.join("pgdatadirs")
     }
 
-    pub fn pg_data_dir(&self, tenantid: &ZTenantId, branch_name: &str) -> PathBuf {
-        self.pg_data_dirs_path()
-            .join(tenantid.to_string())
-            .join(branch_name)
+    pub fn pg_data_dir(&self, name: &str) -> PathBuf {
+        self.pg_data_dirs_path().join(name)
     }
 
     // TODO: move pageserver files into ./pageserver
@@ -88,12 +77,7 @@ fn base_path() -> PathBuf {
 //
 // Initialize a new Zenith repository
 //
-pub fn init(
-    pageserver_pg_port: u16,
-    pageserver_http_port: u16,
-    tenantid: ZTenantId,
-    auth_type: AuthType,
-) -> Result<()> {
+pub fn init(remote_pageserver: Option<&str>) -> Result<()> {
     // check if config already exists
     let base_path = base_path();
     if base_path.exists() {
@@ -102,7 +86,6 @@ pub fn init(
             base_path.to_str().unwrap()
         );
     }
-    fs::create_dir(&base_path)?;
 
     // ok, now check that expected binaries are present
 
@@ -119,64 +102,36 @@ pub fn init(
         anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
     }
 
-    // generate keys for jwt
-    // openssl genrsa -out private_key.pem 2048
-    let private_key_path = base_path.join("auth_private_key.pem");
-    let keygen_output = Command::new("openssl")
-        .arg("genrsa")
-        .args(&["-out", private_key_path.to_str().unwrap()])
-        .arg("2048")
-        .stdout(Stdio::null())
-        .output()
-        .with_context(|| "failed to generate auth private key")?;
-    if !keygen_output.status.success() {
-        anyhow::bail!(
-            "openssl failed: '{}'",
-            String::from_utf8_lossy(&keygen_output.stderr)
-        );
-    }
+    fs::create_dir(&base_path)?;
+    fs::create_dir(base_path.join("pgdatadirs"))?;
 
-    let public_key_path = base_path.join("auth_public_key.pem");
-    // openssl rsa -in private_key.pem -pubout -outform PEM -out public_key.pem
-    let keygen_output = Command::new("openssl")
-        .arg("rsa")
-        .args(&["-in", private_key_path.to_str().unwrap()])
-        .arg("-pubout")
-        .args(&["-outform", "PEM"])
-        .args(&["-out", public_key_path.to_str().unwrap()])
-        .stdout(Stdio::null())
-        .output()
-        .with_context(|| "failed to generate auth private key")?;
-    if !keygen_output.status.success() {
-        anyhow::bail!(
-            "openssl failed: '{}'",
-            String::from_utf8_lossy(&keygen_output.stderr)
-        );
-    }
+    let conf = if let Some(addr) = remote_pageserver {
+        // check that addr is parsable
+        let _uri = Url::parse(addr).map_err(|e| anyhow!("{}: {}", addr, e))?;
 
-    let auth_token =
-        encode_from_key_path(&Claims::new(None, Scope::PageServerApi), &private_key_path)?;
+        LocalEnv {
+            pageserver_connstring: format!("postgresql://{}/", addr),
+            pg_distrib_dir,
+            zenith_distrib_dir: None,
+            base_data_dir: base_path,
+            remotes: BTreeMap::default(),
+        }
+    } else {
+        // Find zenith binaries.
+        let zenith_distrib_dir = env::current_exe()?.parent().unwrap().to_owned();
+        if !zenith_distrib_dir.join("pageserver").exists() {
+            anyhow::bail!("Can't find pageserver binary.",);
+        }
 
-    // Find zenith binaries.
-    let zenith_distrib_dir = env::current_exe()?.parent().unwrap().to_owned();
-    if !zenith_distrib_dir.join("pageserver").exists() {
-        anyhow::bail!("Can't find pageserver binary.",);
-    }
-
-    let conf = LocalEnv {
-        pageserver_pg_port,
-        pageserver_http_port,
-        pg_distrib_dir,
-        zenith_distrib_dir,
-        base_data_dir: base_path,
-        tenantid,
-        auth_token,
-        auth_type,
-        private_key_path,
+        LocalEnv {
+            pageserver_connstring: "postgresql://127.0.0.1:6400".to_string(),
+            pg_distrib_dir,
+            zenith_distrib_dir: Some(zenith_distrib_dir),
+            base_data_dir: base_path,
+            remotes: BTreeMap::default(),
+        }
     };
 
-    fs::create_dir_all(conf.pg_data_dirs_path())?;
-
     let toml = toml::to_string_pretty(&conf)?;
     fs::write(conf.base_data_dir.join("config"), toml)?;
 
@@ -200,3 +155,12 @@ pub fn load_config() -> Result<LocalEnv> {
     let config = fs::read_to_string(repopath.join("config"))?;
     toml::from_str(config.as_str()).map_err(|e| e.into())
 }
+
+// Save config. We use that to change set of remotes from CLI itself.
+pub fn save_config(conf: &LocalEnv) -> Result<()> {
+    let config_path = base_path().join("config");
+    let conf_str = toml::to_string_pretty(conf)?;
+
+    fs::write(config_path, conf_str)?;
+    Ok(())
+}

control_plane/src/postgresql_conf.rs

@@ -1,212 +0,0 @@
-///
-/// Module for parsing postgresql.conf file.
-///
-/// NOTE: This doesn't implement the full, correct postgresql.conf syntax. Just
-/// enough to extract a few settings we need in Zenith, assuming you don't do
-/// funny stuff like include-directives or funny escaping.
-use anyhow::{anyhow, bail, Context, Result};
-use lazy_static::lazy_static;
-use regex::Regex;
-use std::collections::HashMap;
-use std::fmt;
-use std::io::BufRead;
-use std::str::FromStr;
-
-/// In-memory representation of a postgresql.conf file
-#[derive(Default)]
-pub struct PostgresConf {
-    lines: Vec<String>,
-    hash: HashMap<String, String>,
-}
-
-lazy_static! {
-    static ref CONF_LINE_RE: Regex = Regex::new(r"^((?:\w|\.)+)\s*=\s*(\S+)$").unwrap();
-}
-
-impl PostgresConf {
-    pub fn new() -> PostgresConf {
-        PostgresConf::default()
-    }
-
-    /// Read file into memory
-    pub fn read(read: impl std::io::Read) -> Result<PostgresConf> {
-        let mut result = Self::new();
-
-        for line in std::io::BufReader::new(read).lines() {
-            let line = line?;
-
-            // Store each line in a vector, in original format
-            result.lines.push(line.clone());
-
-            // Also parse each line and insert key=value lines into a hash map.
-            //
-            // FIXME: This doesn't match exactly the flex/bison grammar in PostgreSQL.
-            // But it's close enough for our usage.
-            let line = line.trim();
-            if line.starts_with('#') {
-                // comment, ignore
-                continue;
-            } else if let Some(caps) = CONF_LINE_RE.captures(line) {
-                let name = caps.get(1).unwrap().as_str();
-                let raw_val = caps.get(2).unwrap().as_str();
-
-                if let Ok(val) = deescape_str(raw_val) {
-                    // Note: if there's already an entry in the hash map for
-                    // this key, this will replace it. That's the behavior what
-                    // we want; when PostgreSQL reads the file, each line
-                    // overrides any previous value for the same setting.
-                    result.hash.insert(name.to_string(), val.to_string());
-                }
-            }
-        }
-        Ok(result)
-    }
-
-    /// Return the current value of 'option'
-    pub fn get(&self, option: &str) -> Option<&str> {
-        self.hash.get(option).map(|x| x.as_ref())
-    }
-
-    /// Return the current value of a field, parsed to the right datatype.
-    ///
-    /// This calls the FromStr::parse() function on the value of the field. If
-    /// the field does not exist, or parsing fails, returns an error.
-    ///
-    pub fn parse_field<T>(&self, field_name: &str, context: &str) -> Result<T>
-    where
-        T: FromStr,
-        <T as FromStr>::Err: std::error::Error + Send + Sync + 'static,
-    {
-        self.get(field_name)
-            .ok_or_else(|| anyhow!("could not find '{}' option {}", field_name, context))?
-            .parse::<T>()
-            .with_context(|| format!("could not parse '{}' option {}", field_name, context))
-    }
-
-    ///
-    /// Note: if you call this multiple times for the same option, the config
-    /// file will a line for each call. It would be nice to have a function
-    /// to change an existing line, but that's a TODO.
-    ///
-    pub fn append(&mut self, option: &str, value: &str) {
-        self.lines
-            .push(format!("{}={}\n", option, escape_str(value)));
-        self.hash.insert(option.to_string(), value.to_string());
-    }
-
-    /// Append an arbitrary non-setting line to the config file
-    pub fn append_line(&mut self, line: &str) {
-        self.lines.push(line.to_string());
-    }
-}
-
-impl fmt::Display for PostgresConf {
-    /// Return the whole configuration file as a string
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        for line in self.lines.iter() {
-            f.write_str(line)?;
-        }
-        Ok(())
-    }
-}
-
-/// Escape a value for putting in postgresql.conf.
-fn escape_str(s: &str) -> String {
-    // If the string doesn't contain anything that needs quoting or escaping, return it
-    // as it is.
-    //
-    // The first part of the regex, before the '|', matches the INTEGER rule in the
-    // PostgreSQL flex grammar (guc-file.l). It matches plain integers like "123" and
-    // "-123", and also accepts units like "10MB". The second part of the regex matches
-    // the UNQUOTED_STRING rule, and accepts strings that contain a single word, beginning
-    // with a letter. That covers words like "off" or "posix". Everything else is quoted.
-    //
-    // This regex is a bit more conservative than the rules in guc-file.l, so we quote some
-    // strings that PostgreSQL would accept without quoting, but that's OK.
-    lazy_static! {
-        static ref UNQUOTED_RE: Regex =
-            Regex::new(r"(^[-+]?[0-9]+[a-zA-Z]*$)|(^[a-zA-Z][a-zA-Z0-9]*$)").unwrap();
-    }
-    if UNQUOTED_RE.is_match(s) {
-        s.to_string()
-    } else {
-        // Otherwise escape and quote it
-        let s = s
-            .replace('\\', "\\\\")
-            .replace('\n', "\\n")
-            .replace('\'', "''");
-
-        "\'".to_owned() + &s + "\'"
-    }
-}
-
-/// De-escape a possibly-quoted value.
-///
-/// See `DeescapeQuotedString` function in PostgreSQL sources for how PostgreSQL
-/// does this.
-fn deescape_str(s: &str) -> Result<String> {
-    // If the string has a quote at the beginning and end, strip them out.
-    if s.len() >= 2 && s.starts_with('\'') && s.ends_with('\'') {
-        let mut result = String::new();
-
-        let mut iter = s[1..(s.len() - 1)].chars().peekable();
-        while let Some(c) = iter.next() {
-            let newc = if c == '\\' {
-                match iter.next() {
-                    Some('b') => '\x08',
-                    Some('f') => '\x0c',
-                    Some('n') => '\n',
-                    Some('r') => '\r',
-                    Some('t') => '\t',
-                    Some('0'..='7') => {
-                        // TODO
-                        bail!("octal escapes not supported");
-                    }
-                    Some(n) => n,
-                    None => break,
-                }
-            } else if c == '\'' && iter.peek() == Some(&'\'') {
-                // doubled quote becomes just one quote
-                iter.next().unwrap()
-            } else {
-                c
-            };
-
-            result.push(newc);
-        }
-        Ok(result)
-    } else {
-        Ok(s.to_string())
-    }
-}
-
-#[test]
-fn test_postgresql_conf_escapes() -> Result<()> {
-    assert_eq!(escape_str("foo bar"), "'foo bar'");
-    // these don't need to be quoted
-    assert_eq!(escape_str("foo"), "foo");
-    assert_eq!(escape_str("123"), "123");
-    assert_eq!(escape_str("+123"), "+123");
-    assert_eq!(escape_str("-10"), "-10");
-    assert_eq!(escape_str("1foo"), "1foo");
-    assert_eq!(escape_str("foo1"), "foo1");
-    assert_eq!(escape_str("10MB"), "10MB");
-    assert_eq!(escape_str("-10kB"), "-10kB");
-
-    // these need quoting and/or escaping
-    assert_eq!(escape_str("foo bar"), "'foo bar'");
-    assert_eq!(escape_str("fo'o"), "'fo''o'");
-    assert_eq!(escape_str("fo\no"), "'fo\\no'");
-    assert_eq!(escape_str("fo\\o"), "'fo\\\\o'");
-    assert_eq!(escape_str("10 cats"), "'10 cats'");
-
-    // Test de-escaping
-    assert_eq!(deescape_str(&escape_str("foo"))?, "foo");
-    assert_eq!(deescape_str(&escape_str("fo'o\nba\\r"))?, "fo'o\nba\\r");
-    assert_eq!(deescape_str("'\\b\\f\\n\\r\\t'")?, "\x08\x0c\n\r\t");
-
-    // octal-escapes are currently not supported
-    assert!(deescape_str("'foo\\7\\07\\007'").is_err());
-
-    Ok(())
-}

control_plane/src/storage.rs

@@ -1,127 +1,63 @@
-use std::io::Write;
+use std::collections::HashMap;
 use std::net::TcpStream;
 use std::path::PathBuf;
 use std::process::Command;
+use std::thread;
 use std::time::Duration;
-use std::{io, result, thread};
 
-use anyhow::{anyhow, bail};
+use anyhow::{anyhow, bail, Result};
 use nix::sys::signal::{kill, Signal};
 use nix::unistd::Pid;
-use pageserver::http::models::{BranchCreateRequest, TenantCreateRequest};
 use postgres::{Config, NoTls};
-use reqwest::blocking::{Client, RequestBuilder, Response};
-use reqwest::{IntoUrl, Method};
-use thiserror::Error;
-use zenith_utils::http::error::HttpErrorBody;
-use zenith_utils::postgres_backend::AuthType;
-use zenith_utils::zid::ZTenantId;
 
 use crate::local_env::LocalEnv;
 use crate::read_pidfile;
 use pageserver::branches::BranchInfo;
 use zenith_utils::connstring::connection_address;
 
-#[derive(Error, Debug)]
-pub enum PageserverHttpError {
-    #[error("Reqwest error: {0}")]
-    Transport(#[from] reqwest::Error),
-
-    #[error("Error: {0}")]
-    Response(String),
-}
-
-type Result<T> = result::Result<T, PageserverHttpError>;
-
-pub trait ResponseErrorMessageExt: Sized {
-    fn error_from_body(self) -> Result<Self>;
-}
-
-impl ResponseErrorMessageExt for Response {
-    fn error_from_body(self) -> Result<Self> {
-        let status = self.status();
-        if !(status.is_client_error() || status.is_server_error()) {
-            return Ok(self);
-        }
-
-        // reqwest do not export it's error construction utility functions, so lets craft the message ourselves
-        let url = self.url().to_owned();
-        Err(PageserverHttpError::Response(
-            match self.json::<HttpErrorBody>() {
-                Ok(err_body) => format!("Error: {}", err_body.msg),
-                Err(_) => format!("Http error ({}) at {}.", status.as_u16(), url),
-            },
-        ))
-    }
-}
-
 //
 // Control routines for pageserver.
 //
 // Used in CLI and tests.
 //
-#[derive(Debug)]
 pub struct PageServerNode {
     pub kill_on_exit: bool,
-    pub pg_connection_config: Config,
+    pub connection_config: Option<Config>,
     pub env: LocalEnv,
-    pub http_client: Client,
-    pub http_base_url: String,
 }
 
 impl PageServerNode {
     pub fn from_env(env: &LocalEnv) -> PageServerNode {
-        let password = if env.auth_type == AuthType::ZenithJWT {
-            &env.auth_token
-        } else {
-            ""
-        };
-
         PageServerNode {
             kill_on_exit: false,
-            pg_connection_config: Self::pageserver_connection_config(
-                password,
-                env.pageserver_pg_port,
-            ),
+            connection_config: None, // default
             env: env.clone(),
-            http_client: Client::new(),
-            http_base_url: format!("http://127.0.0.1:{}/v1", env.pageserver_http_port),
         }
     }
 
-    fn pageserver_connection_config(password: &str, port: u16) -> Config {
-        format!("postgresql://no_user:{}@127.0.0.1:{}/no_db", password, port)
+    fn default_config() -> Config {
+        "postgresql://no_user@localhost:64000/no_db"
             .parse()
             .unwrap()
     }
 
-    pub fn init(&self, create_tenant: Option<&str>, enable_auth: bool) -> anyhow::Result<()> {
+    pub fn connection_config(&self) -> Config {
+        match &self.connection_config {
+            Some(config) => config.clone(),
+            None => Self::default_config(),
+        }
+    }
+
+    pub fn init(&self) -> Result<()> {
         let mut cmd = Command::new(self.env.pageserver_bin()?);
-        let listen_pg = format!("127.0.0.1:{}", self.env.pageserver_pg_port);
-        let listen_http = format!("127.0.0.1:{}", self.env.pageserver_http_port);
-        let mut args = vec![
-            "--init",
-            "-D",
-            self.env.base_data_dir.to_str().unwrap(),
-            "--postgres-distrib",
-            self.env.pg_distrib_dir.to_str().unwrap(),
-            "--listen-pg",
-            &listen_pg,
-            "--listen-http",
-            &listen_http,
-        ];
-
-        if enable_auth {
-            args.extend(&["--auth-validation-public-key-path", "auth_public_key.pem"]);
-            args.extend(&["--auth-type", "ZenithJWT"]);
-        }
-
-        if let Some(tenantid) = create_tenant {
-            args.extend(&["--create-tenant", tenantid])
-        }
-
         let status = cmd
-            .args(args)
+            .args(&[
+                "--init",
+                "-D",
+                self.env.base_data_dir.to_str().unwrap(),
+                "--postgres-distrib",
+                self.env.pg_distrib_dir.to_str().unwrap(),
+            ])
             .env_clear()
             .env("RUST_BACKTRACE", "1")
             .status()
@@ -142,13 +78,12 @@ impl PageServerNode {
         self.repo_path().join("pageserver.pid")
     }
 
-    pub fn start(&self) -> anyhow::Result<()> {
-        print!(
-            "Starting pageserver at '{}' in '{}'",
-            connection_address(&self.pg_connection_config),
+    pub fn start(&self) -> Result<()> {
+        println!(
+            "Starting pageserver at '{}' in {}",
+            connection_address(&self.connection_config()),
             self.repo_path().display()
         );
-        io::stdout().flush().unwrap();
 
         let mut cmd = Command::new(self.env.pageserver_bin()?);
         cmd.args(&["-D", self.repo_path().to_str().unwrap()])
@@ -165,179 +100,116 @@ impl PageServerNode {
 
         // It takes a while for the page server to start up. Wait until it is
         // open for business.
-        const RETRIES: i8 = 15;
-        for retries in 1..RETRIES {
-            match self.check_status() {
-                Ok(_) => {
-                    println!("\nPageserver started");
-                    return Ok(());
-                }
-                Err(err) => {
-                    match err {
-                        PageserverHttpError::Transport(err) => {
-                            if err.is_connect() && retries < 5 {
-                                print!(".");
-                                io::stdout().flush().unwrap();
-                            } else {
-                                if retries == 5 {
-                                    println!() // put a line break after dots for second message
-                                }
-                                println!(
-                                    "Pageserver not responding yet, err {} retrying ({})...",
-                                    err, retries
-                                );
-                            }
-                        }
-                        PageserverHttpError::Response(msg) => {
-                            bail!("pageserver failed to start: {} ", msg)
-                        }
-                    }
-                    thread::sleep(Duration::from_secs(1));
-                }
-            }
-        }
-        bail!("pageserver failed to start in {} seconds", RETRIES);
-    }
-
-    pub fn stop(&self, immediate: bool) -> anyhow::Result<()> {
-        let pid = read_pidfile(&self.pid_file())?;
-        let pid = Pid::from_raw(pid);
-        if immediate {
-            println!("Stop pageserver immediately");
-            if kill(pid, Signal::SIGQUIT).is_err() {
-                bail!("Failed to kill pageserver with pid {}", pid);
-            }
-        } else {
-            println!("Stop pageserver gracefully");
-            if kill(pid, Signal::SIGTERM).is_err() {
-                bail!("Failed to stop pageserver with pid {}", pid);
-            }
-        }
-
-        let address = connection_address(&self.pg_connection_config);
-
-        // TODO Remove this "timeout" and handle it on caller side instead.
-        // Shutting down may take a long time,
-        // if pageserver checkpoints a lot of data
-        for _ in 0..100 {
-            if let Err(_e) = TcpStream::connect(&address) {
-                println!("Pageserver stopped receiving connections");
-
-                //Now check status
-                match self.check_status() {
-                    Ok(_) => {
-                        println!("Pageserver status is OK. Wait a bit.");
-                        thread::sleep(Duration::from_secs(1));
-                    }
-                    Err(err) => {
-                        println!("Pageserver status is: {}", err);
-                        return Ok(());
-                    }
-                }
+        for retries in 1..15 {
+            let client = self.page_server_psql_client();
+            if client.is_ok() {
+                break;
             } else {
-                println!("Pageserver still receives connections");
+                println!("Pageserver not responding yet, retrying ({})...", retries);
                 thread::sleep(Duration::from_secs(1));
             }
         }
 
+        println!("Pageserver started");
+
+        Ok(())
+    }
+
+    pub fn stop(&self) -> Result<()> {
+        let pid = read_pidfile(&self.pid_file())?;
+        let pid = Pid::from_raw(pid);
+        if kill(pid, Signal::SIGTERM).is_err() {
+            bail!("Failed to kill pageserver with pid {}", pid);
+        }
+
+        // wait for pageserver stop
+        let address = connection_address(&self.connection_config());
+        for _ in 0..5 {
+            let stream = TcpStream::connect(&address);
+            thread::sleep(Duration::from_secs(1));
+            if let Err(_e) = stream {
+                println!("Pageserver stopped");
+                return Ok(());
+            }
+            println!("Stopping pageserver on {}", address);
+        }
+
         bail!("Failed to stop pageserver with pid {}", pid);
     }
 
     pub fn page_server_psql(&self, sql: &str) -> Vec<postgres::SimpleQueryMessage> {
-        let mut client = self.pg_connection_config.connect(NoTls).unwrap();
+        let mut client = self.connection_config().connect(NoTls).unwrap();
 
         println!("Pageserver query: '{}'", sql);
         client.simple_query(sql).unwrap()
     }
 
-    pub fn page_server_psql_client(&self) -> result::Result<postgres::Client, postgres::Error> {
-        self.pg_connection_config.connect(NoTls)
+    pub fn page_server_psql_client(&self) -> Result<postgres::Client, postgres::Error> {
+        self.connection_config().connect(NoTls)
     }
 
-    fn http_request<U: IntoUrl>(&self, method: Method, url: U) -> RequestBuilder {
-        let mut builder = self.http_client.request(method, url);
-        if self.env.auth_type == AuthType::ZenithJWT {
-            builder = builder.bearer_auth(&self.env.auth_token)
-        }
-        builder
-    }
-
-    pub fn check_status(&self) -> Result<()> {
-        self.http_request(Method::GET, format!("{}/{}", self.http_base_url, "status"))
-            .send()?
-            .error_from_body()?;
-        Ok(())
-    }
-
-    pub fn tenant_list(&self) -> Result<Vec<String>> {
-        Ok(self
-            .http_request(Method::GET, format!("{}/{}", self.http_base_url, "tenant"))
-            .send()?
-            .error_from_body()?
-            .json()?)
-    }
-
-    pub fn tenant_create(&self, tenantid: ZTenantId) -> Result<()> {
-        Ok(self
-            .http_request(Method::POST, format!("{}/{}", self.http_base_url, "tenant"))
-            .json(&TenantCreateRequest {
-                tenant_id: tenantid,
+    pub fn branches_list(&self) -> Result<Vec<BranchInfo>> {
+        let mut client = self.page_server_psql_client()?;
+        let query_result = client.simple_query("branch_list")?;
+        let branches_json = query_result
+            .first()
+            .map(|msg| match msg {
+                postgres::SimpleQueryMessage::Row(row) => row.get(0),
+                _ => None,
             })
-            .send()?
-            .error_from_body()?
-            .json()?)
+            .flatten()
+            .ok_or_else(|| anyhow!("missing branches"))?;
+
+        let res: Vec<BranchInfo> = serde_json::from_str(branches_json)?;
+        Ok(res)
     }
 
-    pub fn branch_list(&self, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
-        Ok(self
-            .http_request(
-                Method::GET,
-                format!("{}/branch/{}", self.http_base_url, tenantid),
-            )
-            .send()?
-            .error_from_body()?
-            .json()?)
-    }
+    pub fn branch_create(&self, name: &str, startpoint: &str) -> Result<BranchInfo> {
+        let mut client = self.page_server_psql_client()?;
+        let query_result =
+            client.simple_query(format!("branch_create {} {}", name, startpoint).as_str())?;
 
-    pub fn branch_create(
-        &self,
-        branch_name: &str,
-        startpoint: &str,
-        tenantid: &ZTenantId,
-    ) -> Result<BranchInfo> {
-        Ok(self
-            .http_request(Method::POST, format!("{}/branch", self.http_base_url))
-            .json(&BranchCreateRequest {
-                tenant_id: tenantid.to_owned(),
-                name: branch_name.to_owned(),
-                start_point: startpoint.to_owned(),
+        let branch_json = query_result
+            .first()
+            .map(|msg| match msg {
+                postgres::SimpleQueryMessage::Row(row) => row.get(0),
+                _ => None,
             })
-            .send()?
-            .error_from_body()?
-            .json()?)
+            .flatten()
+            .ok_or_else(|| anyhow!("missing branch"))?;
+
+        let res: BranchInfo = serde_json::from_str(branch_json).map_err(|e| {
+            anyhow!(
+                "failed to parse branch_create response: {}: {}",
+                branch_json,
+                e
+            )
+        })?;
+
+        Ok(res)
     }
 
-    pub fn branch_get_by_name(
-        &self,
-        tenantid: &ZTenantId,
-        branch_name: &str,
-    ) -> Result<BranchInfo> {
-        Ok(self
-            .http_request(
-                Method::GET,
-                format!("{}/branch/{}/{}", self.http_base_url, tenantid, branch_name),
-            )
-            .send()?
-            .error_for_status()?
-            .json()?)
+    // TODO: make this a separate request type and avoid loading all the branches
+    pub fn branch_get_by_name(&self, name: &str) -> Result<BranchInfo> {
+        let branch_infos = self.branches_list()?;
+        let branche_by_name: Result<HashMap<String, BranchInfo>> = branch_infos
+            .into_iter()
+            .map(|branch_info| Ok((branch_info.name.clone(), branch_info)))
+            .collect();
+        let branche_by_name = branche_by_name?;
+
+        let branch = branche_by_name
+            .get(name)
+            .ok_or_else(|| anyhow!("Branch {} not found", name))?;
+
+        Ok(branch.clone())
     }
 }
 
 impl Drop for PageServerNode {
     fn drop(&mut self) {
-        // TODO Looks like this flag is never set
         if self.kill_on_exit {
-            let _ = self.stop(true);
+            let _ = self.stop();
         }
     }
 }

docker-entrypoint.sh

@@ -1,13 +1,11 @@
 #!/bin/sh
-set -eux
-
 if [ "$1" = 'pageserver' ]; then
-    if [ ! -d "/data/tenants" ]; then
+    if [ ! -d "/data/timelines" ]; then
         echo "Initializing pageserver data directory"
         pageserver --init -D /data --postgres-distrib /usr/local
     fi
     echo "Staring pageserver at 0.0.0.0:6400"
-    pageserver -l 0.0.0.0:6400 --listen-http 0.0.0.0:9898 -D /data
+    pageserver -l 0.0.0.0:6400 -D /data
 else
     "$@"
 fi

docs/README.md

@@ -1,14 +0,0 @@
-# Zenith documentation
-
-## Table of contents
-
-- [authentication.md](authentication.md) — pageserver JWT authentication.
-- [docker.md](docker.md) — Docker images and building pipeline.
-- [glossary.md](glossary.md) — Glossary of all the terms used in codebase.
-- [multitenancy.md](multitenancy.md) — how multitenancy is organized in the pageserver and Zenith CLI.
-- [sourcetree.md](sourcetree.md) — Overview of the source tree layeout.
-- [pageserver/README](/pageserver/README) — pageserver overview.
-- [postgres_ffi/README](/postgres_ffi/README) — Postgres FFI overview.
-- [test_runner/README.md](/test_runner/README.md) — tests infrastructure overview.
-- [walkeeper/README](/walkeeper/README) — WAL service overview.
-- [core_changes.md](core_changes.md) - Description of Zenith changes in Postgres core

docs/authentication.md

@@ -1,30 +0,0 @@
-## Authentication
-
-### Overview
-
-Current state of authentication includes usage of JWT tokens in communication between compute and pageserver and between CLI and pageserver. JWT token is signed using RSA keys. CLI generates a key pair during call to `zenith init`. Using following openssl commands:
-
-```bash
-openssl genrsa -out private_key.pem 2048
-openssl rsa -in private_key.pem -pubout -outform PEM -out public_key.pem
-```
-
-CLI also generates signed token and saves it in the config for later access to pageserver. Now authentication is optional. Pageserver has two variables in config: `auth_validation_public_key_path` and `auth_type`, so when auth type present and set to `ZenithJWT` pageserver will require authentication for connections. Actual JWT is passed in password field of connection string. There is a caveat for psql, it silently truncates passwords to 100 symbols, so to correctly pass JWT via psql you have to either use PGPASSWORD environment variable, or store password in psql config file.
-
-Currently there is no authentication between compute and safekeepers, because this communication layer is under heavy refactoring. After this refactoring support for authentication will be added there too. Now safekeeper supports "hardcoded" token passed via environment variable to be able to use callmemaybe command in pageserver.
-
-Compute uses token passed via environment variable to communicate to pageserver and in the future to the safekeeper too.
-
-JWT authentication now supports two scopes: tenant and pageserverapi. Tenant scope is intended for use in tenant related api calls, e.g. create_branch. Compute launched for particular tenant also uses this scope. Scope pageserver api is intended to be used by console to manage pageserver. For now we have only one management operation - create tenant.
-
-Examples for token generation in python:
-
-```python
-# generate pageserverapi token
-management_token = jwt.encode({"scope": "pageserverapi"}, auth_keys.priv, algorithm="RS256")
-
-# generate tenant token
-tenant_token = jwt.encode({"scope": "tenant", "tenant_id": ps.initial_tenant}, auth_keys.priv, algorithm="RS256")
-```
-
-Utility functions to work with jwts in rust are located in zenith_utils/src/auth.rs

docs/core_changes.md

@@ -1,202 +0,0 @@
-1. Add t_cid to XLOG record
-- Why?
-  The cmin/cmax on a heap page is a real bummer. I don't see any other way to fix that than bite the bullet and modify the WAL-logging routine to include the cmin/cmax.
-
-  To recap, the problem is that the XLOG_HEAP_INSERT record does not include the command id of the inserted row. And same with deletion/update. So in the primary, a row is inserted with current xmin + cmin. But in the replica, the cmin is always set to 1. That works, because the command id is only relevant to the inserting transaction itself. After commit/abort, no one cares abut it anymore.
-
-- Alternatives?
-  I don't know
-
-2. Add PD_WAL_LOGGED.
-- Why?
-  Postgres sometimes writes data to the page before it is wal-logged. If such page ais swapped out, we  will loose this change. The problem is currently solved by setting PD_WAL_LOGGED bit in page header. When page without this bit set is written to the SMGR, then it is forced to be written to the WAL as FPI using log_newpage_copy() function.
-
-  There was wrong assumption that it can happen only during construction of some exotic indexes (like gist). It is not true. The same situation can happen with COPY,VACUUM and when record hint bits are set.
-
-- Discussion:
-  https://discord.com/channels/869525774699462656/882681420986851359
-
-- Alternatives:
-  Do not store this flag in page header, but associate this bit with shared buffer. Logically it is more correct but in practice we will get not advantages: neither in space, neither in CPU overhead.
-
-
-3. XLogReadBufferForRedo not always loads and pins requested buffer. So we need to add extra checks that buffer is really pinned. Also do not use BufferGetBlockNumber for buffer returned by XLogReadBufferForRedo.
-- Why?
-  XLogReadBufferForRedo is not pinning pages which are not requested by wal-redo. It is specific only for wal-redo Postgres.
-
-- Alternatives?
-  No
-
-
-4. Eliminate reporting of some warnings related with hint bits, for example
-"page is not marked all-visible but visibility map bit is set in relation".
-- Why?
-  Hint bit may be not WAL logged.
-
-- Alternative?
-  Always wal log any page changes.
-
-
-5. Maintain last written LSN.
-- Why?
-  When compute node requests page from page server, we need to specify LSN. Ideally it should be LSN
-  of WAL record performing last update of this pages. But we do not know it, because we do not have page.
-  We can use current WAL flush position, but in this case there is high probability that page server
-  will be blocked until this peace of WAL is delivered.
-  As better approximation we can keep max LSN of written page. It will be better to take in account LSNs only of evicted pages,
-  but SMGR API doesn't provide such knowledge.
-
-- Alternatives?
-  Maintain map of LSNs of evicted pages.
-
-
-6. Launching Postgres without WAL.
-- Why?
-  According to Zenith architecture compute node is stateless. So when we are launching
-  compute node, we need to provide some dummy PG_DATADIR. Relation pages
-  can be requested on demand from page server. But Postgres still need some non-relational data:
-  control and configuration files, SLRUs,...
-  It is currently implemented  using basebackup (do not mix with pg_basebackup) which is created
-  by pageserver. It includes in this tarball config/control files, SLRUs and required directories.
-  As far as pageserver do not have original (non-scattered) WAL segments, it includes in
-  this tarball dummy WAL segment which contains only SHUTDOWN_CHECKPOINT record at the beginning of segment,
-  which redo field points to the end of wal. It allows to load checkpoint record in more or less
-  standard way with minimal changes of Postgres, but then some special handling is needed,
-  including restoring previous record position from zenith.signal file.
-  Also we have to correctly initialize header of last WAL page (pointed by checkpoint.redo)
-  to pass checks performed by XLogReader.
-
-- Alternatives?
-  We may not include fake WAL segment in tarball at all and modify xlog.c to load checkpoint record
-  in special way. But it may only increase number of changes in xlog.c
-
-7. Add redo_read_buffer_filter callback to XLogReadBufferForRedoExtended
-- Why?
-  We need a way in wal-redo Postgres to ignore pages which are not requested by pageserver.
-  So wal-redo Postgres reconstructs only requested page and for all other returns BLK_DONE
-  which means that recovery for them is not needed.
-
-- Alternatives?
-  No
-
-8. Enforce WAL logging of sequence updates.
-- Why?
-  Due to performance reasons Postgres don't want to log each fetching of a value from a sequence,
-  so we pre-log a few fetches in advance. In the event of crash we can lose
-  (skip over) as many values as we pre-logged.
-  But it doesn't work with Zenith because page with sequence value can be evicted from buffer cache
-  and we will get a gap in sequence values even without crash.
-
-- Alternatives:
-  Do not try to preserve sequential order but avoid performance penalty.
-
-
-9. Treat unlogged tables as normal (permanent) tables.
-- Why?
-  Unlogged tables are not transient, so them have to survive node restart (unlike temporary tables).
-  But as far as compute node is stateless, we need to persist their data to storage node.
-  And it can only be done through the WAL.
-
-- Alternatives?
-  * Store unlogged tables locally (violates requirement of stateless compute nodes).
-  * Prohibit unlogged tables at all.
-
-
-10. Support start Postgres in wal-redo mode
-- Why?
-  To be able to apply WAL record and reconstruct pages at page server.
-
-- Alternatives?
-  * Rewrite redo handlers in Rust
-  * Do not reconstruct pages at page server at all and do it at compute node.
-
-
-11. WAL proposer
-- Why?
-  WAL proposer is communicating with safekeeper and ensures WAL durability by quorum writes.
-  It is currently implemented as patch to standard WAL sender.
-
-- Alternatives?
-  Can be moved to extension if some extra callbacks will be added to wal sender code.
-
-
-12. Secure Computing BPF API wrapper.
-- Why?
-  Pageserver delegates complex WAL decoding duties to Postgres,
-  which means that the latter might fall victim to carefully designed
-  malicious WAL records and start doing harmful things to the system.
-  To prevent this, it has been decided to limit possible interactions
-  with the outside world using the Secure Computing BPF mode.
-
-- Alternatives:
-  * Rewrite redo handlers in Rust.
-  * Add more checks to guarantee correctness of WAL records.
-  * Move seccomp.c to extension
-  * Many other discussed approaches to neutralize incorrect WAL records vulnerabilities.
-
-
-13. Callbacks for replica feedbacks
-- Why?
-  Allowing waproposer to interact with walsender code.
-
-- Alternatives
-  Copy walsender code to walproposer.
-
-
-14. Support multiple SMGR implementations.
-- Why?
-  Postgres provides abstract API for storage manager but it has only one implementation
-  and provides no way to replace it with custom storage manager.
-
-- Alternatives?
-  None.
-
-
-15. Calculate database size as sum of all database relations.
-- Why?
-  Postgres is calculating database size by traversing data directory
-  but as far as Zenith compute node is stateless we can not do it.
-
-- Alternatives?
-  Send this request directly to pageserver and calculate real (physical) size
-  of Zenith representation of database/timeline, rather than sum logical size of all relations.
-
-
------------------------------------------------
-Not currently committed but proposed:
-
-1. Disable ring buffer buffer manager strategies
-- Why?
-  Postgres tries to avoid cache flushing by bulk operations (copy, seqscan, vacuum,...).
-  Even if there are free space in buffer cache, pages may be evicted.
-  Negative effect of it can be somehow compensated by file system cache, but in case of Zenith
-  cost of requesting page from page server is much higher.
-
-- Alternatives?
-  Instead of just prohibiting ring buffer we may try to implement more flexible eviction policy,
-  for example copy evicted page from ring buffer to some other buffer if there is free space
-  in buffer cache.
-
-2. Disable marking page as dirty when hint bits are set.
-- Why?
-  Postgres has to modify page twice: first time when some tuple is updated and second time when
-  hint bits are set. Wal logging hint bits updates requires FPI which significantly increase size of WAL.
-
-- Alternatives?
-  Add special WAL record for setting page hints.
-
-3. Prefetching
-- Why?
-  As far as pages in Zenith are loaded on demand, to reduce node startup time
-  and also sppedup some massive queries we need some mechanism for bulk loading to
-  reduce page request round-trip overhead.
-
-  Currently Postgres is supporting prefetching only for bitmap scan.
-  In Zenith we also use prefetch for sequential and index scan. For sequential scan we prefetch
-  some number of following pages. For index scan we prefetch pages of heap relation addressed by TIDs.
-
-4. Prewarming.
-- Why?
-  Short downtime (or, in other words, fast compute node restart time) is one of the key feature of Zenith.
-  But overhead of request-response round-trip for loading pages on demand can make started node warm-up quite slow.
-  We can capture state of compute node buffer cache and send bulk request for this pages at startup.

docs/docker.md

@@ -1,38 +0,0 @@
-# Docker images of Zenith
-
-## Images
-
-Currently we build two main images:
-
-- [zenithdb/zenith](https://hub.docker.com/repository/docker/zenithdb/zenith) — image with pre-built `pageserver`, `safekeeper` and `proxy` binaries and all the required runtime dependencies. Built from [/Dockerfile](/Dockerfile).
-- [zenithdb/compute-node](https://hub.docker.com/repository/docker/zenithdb/compute-node) — compute node image with pre-built Postgres binaries from [zenithdb/postgres](https://github.com/zenithdb/postgres).
-
-And two intermediate images used either to reduce build time or to deliver some additional binary tools from other repos:
-
-- [zenithdb/build](https://hub.docker.com/repository/docker/zenithdb/build) — image with all the dependencies required to build Zenith and compute node images. This image is based on `rust:slim-buster`, so it also has a proper `rust` environment. Built from [/Dockerfile.build](/Dockerfile.build).
-- [zenithdb/compute-tools](https://hub.docker.com/repository/docker/zenithdb/compute-tools) — compute node configuration management tools.
-
-## Building pipeline
-
-1. Image `zenithdb/compute-tools` is re-built automatically.
-
-2. Image `zenithdb/build` is built manually. If you want to introduce any new compile time dependencies to Zenith or compute node you have to update this image as well, build it and push to Docker Hub.
-
-Build:
-```sh
-docker build -t zenithdb/build:buster -f Dockerfile.build .
-```
-
-Login:
-```sh
-docker login
-```
-
-Push to Docker Hub:
-```sh
-docker push zenithdb/build:buster
-```
-
-3. Image `zenithdb/compute-node` is built independently in the [zenithdb/postgres](https://github.com/zenithdb/postgres) repo.
-
-4. Image `zenithdb/zenith` is built in this repo after a successful `release` tests run and pushed to Docker Hub automatically.

docs/glossary.md

@@ -1,218 +0,0 @@
-# Glossary
-
-### Authentication
-
-### Base image (page image)
-
-### Basebackup
-
-A tarball with files needed to bootstrap a compute node[] and a corresponding command to create it.
-NOTE:It has nothing to do with PostgreSQL pg_basebackup.
-
-### Branch
-
-We can create branch at certain LSN using `zenith branch` command.
-Each Branch lives in a corresponding timeline[] and has an ancestor[].
-
-
-### Checkpoint (PostgreSQL)
-
-NOTE: This is an overloaded term.
-
-A checkpoint record in the WAL marks a point in the WAL sequence at which it is guaranteed that all data files have been updated with all information from shared memory modified before that checkpoint; 
-
-### Checkpoint (Layered repository)
-
-NOTE: This is an overloaded term.
-
-Whenever enough WAL has been accumulated in memory, the page server []
-writes out the changes from in-memory layers into new layer files[]. This process
-is called "checkpointing". The page server only creates layer files for
-relations that have been modified since the last checkpoint. 
-
-Configuration parameter `checkpoint_distance` defines the distance
-from current LSN to perform checkpoint of in-memory layers.
-Default is `DEFAULT_CHECKPOINT_DISTANCE`.
-Set this parameter to `0` to force checkpoint of every layer.
-
-Configuration parameter `checkpoint_period` defines the interval between checkpoint iterations.
-Default is `DEFAULT_CHECKPOINT_PERIOD`.
-### Compute node
-
-Stateless Postgres node that stores data in pageserver.
-
-### Garbage collection
-
-The process of removing old on-disk layers that are not needed by any timeline anymore.
-### Fork
-
-Each of the separate segmented file sets in which a relation is stored. The main fork is where the actual data resides. There also exist two secondary forks for metadata: the free space map and the visibility map.
-Each PostgreSQL fork is considered a separate relish.
-
-### Layer
-
-Each layer corresponds to the specific version of a relish Segment in a range of LSNs.
-There are two kinds of layers, in-memory and on-disk layers. In-memory
-layers are used to ingest incoming WAL, and provide fast access
-to the recent page versions. On-disk layers are stored as files on disk, and
-are immutable.
-### Layer file (on-disk layer)
-
-Layered repository on-disk format is based on immutable files.  The
-files are called "layer files". Each file corresponds to one RELISH_SEG_SIZE
-segment of a PostgreSQL relation fork. There are two kinds of layer
-files: image files and delta files. An image file contains a
-"snapshot" of the segment at a particular LSN, and a delta file
-contains WAL records applicable to the segment, in a range of LSNs.
-
-### Layer map
-
-The layer map tracks what layers exist for all the relishes in a timeline.
-### Layered repository
-
-Zenith repository implementation that keeps data in layers.
-### LSN
-
-
-### Page (block)
-
-The basic structure used to store relation data. All pages are of the same size.
-This is the unit of data exchange between compute node and pageserver.
-
-### Pageserver
-
-Zenith storage engine: repositories + wal receiver + page service + wal redo.
-
-### Page service
-
-The Page Service listens for GetPage@LSN requests from the Compute Nodes,
-and responds with pages from the repository.
-
-
-### PITR (Point-in-time-recovery)
-
-PostgreSQL's ability to restore up to a specified LSN.
-
-### Primary node
-
-
-### Proxy
-
-Postgres protocol proxy/router.
-This service listens psql port, can check auth via external service
-and create new databases and accounts (control plane API in our case).
-
-### Relation
-
-The generic term in PostgreSQL for all objects in a database that have a name and a list of attributes defined in a specific order.
-
-### Relish
-
-We call each relation and other file that is stored in the
-repository a "relish". It comes from "rel"-ish, as in "kind of a
-rel", because it covers relations as well as other things that are
-not relations, but are treated similarly for the purposes of the
-storage layer.
-
-### Replication slot
-
-
-### Replica node
-
-
-### Repository
-
-Repository stores multiple timelines, forked off from the same initial call to 'initdb'
-and has associated WAL redo service.
-One repository corresponds to one Tenant.
-
-### Retention policy
-
-How much history do we need to keep around for PITR and read-only nodes?
-
-### Segment (PostgreSQL)
-
-NOTE: This is an overloaded term.
-
-A physical file that stores data for a given relation. File segments are
-limited in size by a compile-time setting (1 gigabyte by default), so if a
-relation exceeds that size, it is split into multiple segments.
-
-### Segment (Layered Repository)
-
-NOTE: This is an overloaded term.
-
-Segment is a RELISH_SEG_SIZE slice of relish (identified by a SegmentTag).
-
-### SLRU
-
-SLRUs include pg_clog, pg_multixact/members, and
-pg_multixact/offsets. There are other SLRUs in PostgreSQL, but
-they don't need to be stored permanently (e.g. pg_subtrans),
-or we do not support them in zenith yet (pg_commit_ts).
-Each SLRU segment is considered a separate relish[].
-
-### Tenant (Multitenancy)
-Tenant represents a single customer, interacting with Zenith.
-Wal redo[] activity, timelines[], layers[] are managed for each tenant independently.
-One pageserver[] can serve multiple tenants at once.
-One safekeeper 
-
-See `docs/multitenancy.md` for more.
-
-### Timeline
-
-Timeline accepts page changes and serves get_page_at_lsn() and
-get_rel_size() requests. The term "timeline" is used internally
-in the system, but to users they are exposed as "branches", with
-human-friendly names.
-
-NOTE: this has nothing to do with PostgreSQL WAL timelines.
-
-### XLOG
-
-PostgreSQL alias for WAL[].
-
-### WAL (Write-ahead log)
-
-The journal that keeps track of the changes in the database cluster as user- and system-invoked operations take place. It comprises many individual WAL records[] written sequentially to WAL files[].
-
-### WAL acceptor, WAL proposer
-
-In the context of the consensus algorithm, the Postgres
-compute node is also known as the WAL proposer, and the safekeeper is also known
-as the acceptor. Those are the standard terms in the Paxos algorithm.
-
-### WAL receiver (WAL decoder)
-
-The WAL receiver connects to the external WAL safekeeping service (or
-directly to the primary) using PostgreSQL physical streaming
-replication, and continuously receives WAL. It decodes the WAL records,
-and stores them to the repository.
-
-We keep one WAL receiver active per timeline.
-
-### WAL record
-
-A low-level description of an individual data change.
-
-### WAL redo
-
-A service that runs PostgreSQL in a special wal_redo mode
-to apply given WAL records over an old page image and return new page image.
-
-### WAL safekeeper
-
-One node that participates in the quorum. All the safekeepers
-together form the WAL service.
-
-### WAL segment (WAL file)
-
-Also known as WAL segment or WAL segment file. Each of the sequentially-numbered files that provide storage space for WAL. The files are all of the same predefined size and are written in sequential order, interspersing changes as they occur in multiple simultaneous sessions.
-
-### WAL service
-
-The service as whole that ensures that WAL is stored durably.
-
-### Web console
-

docs/multitenancy.md

@@ -1,59 +0,0 @@
-## Multitenancy
-
-### Overview
-
-Zenith supports multitenancy. One pageserver can serve multiple tenants at once. Tenants can be managed via zenith CLI. During page server setup tenant can be created using ```zenith init --create-tenant``` Also tenants can be added into the system on the fly without pageserver restart. This can be done using the following cli command: ```zenith tenant create``` Tenants use random identifiers which can be represented as a 32 symbols hexadecimal string. So zenith tenant create accepts desired tenant id as an optional argument. The concept of timelines/branches is working independently per tenant.
-
-### Tenants in other commands
-
-By default during `zenith init` new tenant is created on the pageserver. Newly created tenant's id is saved to cli config, so other commands can use it automatically if no direct arugment `--tenantid=<tenantid>` is provided. So generally tenantid more frequently appears in internal pageserver interface. Its commands take tenantid argument to distinguish to which tenant operation should be applied. CLI support creation of new tenants.
-
-Examples for cli:
-
-```sh
-zenith tenant list
-
-zenith tenant create // generates new id
-
-zenith tenant create ee6016ec31116c1b7c33dfdfca38892f
-
-zenith pg create main // default tenant from zenith init
-
-zenith pg create main --tenantid=ee6016ec31116c1b7c33dfdfca38892f
-
-zenith branch --tenantid=ee6016ec31116c1b7c33dfdfca38892f
-```
-
-### Data layout
-
-On the page server tenants introduce one level of indirection, so data directory structured the following way:
-```
-<pageserver working directory>
-├── pageserver.log
-├── pageserver.pid
-├── pageserver.toml
-└── tenants
-   ├── 537cffa58a4fa557e49e19951b5a9d6b
-   ├── de182bc61fb11a5a6b390a8aed3a804a
-   └── ee6016ec31116c1b7c33dfdfca38891f
-```
-Wal redo activity and timelines are managed for each tenant independently.
-
-For local environment used for example in tests there also new level of indirection for tenants. It touches `pgdatadirs` directory. Now it contains `tenants` subdirectory so the structure looks the following way:
-
-```
-pgdatadirs
-└── tenants
-   ├── de182bc61fb11a5a6b390a8aed3a804a
-   │  └── main
-   └── ee6016ec31116c1b7c33dfdfca38892f
-      └── main
-```
-
-### Changes to postgres
-
-Tenant id is passed to postgres via GUC the same way as the timeline. Tenant id is added to commands issued to pageserver, namely: pagestream, callmemaybe. Tenant id is also exists in ServerInfo structure, this is needed to pass the value to wal receiver to be able to forward it to the pageserver.
-
-### Safety
-
-For now particular tenant can only appear on a particular pageserver. Set of safekeepers are also pinned to particular (tenantid, timeline) pair so there can only be one writer for particular (tenantid, timeline).

docs/sourcetree.md

@@ -1,81 +0,0 @@
-## Source tree layout
-
-Below you will find a brief overview of each subdir in the source tree in alphabetical order.
-
-`/control_plane`:
-
-Local control plane.
-Functions to start, configure and stop pageserver and postgres instances running as a local processes.
-Intended to be used in integration tests and in CLI tools for local installations.
-
-`/docs`:
-
-Documentaion of the Zenith features and concepts.
-Now it is mostly dev documentation.
-
-`/monitoring`:
-
-TODO
-
-`/pageserver`:
-
-Zenith storage service.
-The pageserver has a few different duties:
-
-- Store and manage the data.
-- Generate a tarball with files needed to bootstrap ComputeNode.
-- Respond to GetPage@LSN requests from the Compute Nodes.
-- Receive WAL from the WAL service and decode it.
-- Replay WAL that's applicable to the chunks that the Page Server maintains
-
-For more detailed info, see `/pageserver/README`
-
-`/postgres_ffi`:
-
-Utility functions for interacting with PostgreSQL file formats.
-Misc constants, copied from PostgreSQL headers.
-
-`/proxy`:
-
-Postgres protocol proxy/router.
-This service listens psql port, can check auth via external service
-and create new databases and accounts (control plane API in our case).
-
-`/test_runner`:
-
-Integration tests, written in Python using the `pytest` framework.
-
-`/vendor/postgres`:
-
-PostgreSQL source tree, with the modifications needed for Zenith.
-
-`/vendor/postgres/contrib/zenith`:
-
-PostgreSQL extension that implements storage manager API and network communications with remote page server.
-
-`/vendor/postgres/contrib/zenith_test_utils`:
-
-PostgreSQL extension that contains functions needed for testing and debugging.
-
-`/walkeeper`:
-
-The zenith WAL service that receives WAL from a primary compute nodes and streams it to the pageserver.
-It acts as a holding area and redistribution center for recently generated WAL.
-
-For more detailed info, see `/walkeeper/README`
-
-`/workspace_hack`:
-The workspace_hack crate exists only to pin down some dependencies.
-
-`/zenith`
-
-Main entry point for the 'zenith' CLI utility.
-TODO: Doesn't it belong to control_plane?
-
-`/zenith_metrics`:
-
-Helpers for exposing Prometheus metrics from the server.
-
-`/zenith_utils`:
-
-Helpers that are shared between other crates in this repository.

monitoring/docker-compose.yml

@@ -1,25 +0,0 @@
-version: "3"
-services:
-
-  prometheus:
-    container_name: prometheus
-    image: prom/prometheus:latest
-    volumes:
-      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
-    # ports:
-    #   - "9090:9090"
-    # TODO: find a proper portable solution
-    network_mode: "host"
-
-  grafana:
-    image: grafana/grafana:latest
-    volumes:
-      - ./grafana.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
-    environment:
-      - GF_AUTH_ANONYMOUS_ENABLED=true
-      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
-      - GF_AUTH_DISABLE_LOGIN_FORM=true
-    # ports:
-    #   - "3000:3000"
-    # TODO: find a proper portable solution
-    network_mode: "host"

monitoring/grafana.yaml

@@ -1,12 +0,0 @@
-apiVersion: 1
-
-datasources:
-- name: Prometheus
-  type: prometheus
-  access: proxy
-  orgId: 1
-  url: http://localhost:9090
-  basicAuth: false
-  isDefault: false
-  version: 1
-  editable: false

monitoring/prometheus.yaml

@@ -1,5 +0,0 @@
-scrape_configs:
-  - job_name: 'default'
-    scrape_interval: 10s
-    static_configs:
-      - targets: ['localhost:9898']

pageserver/Cargo.toml

@@ -4,44 +4,46 @@ version = "0.1.0"
 authors = ["Stas Kelvich <stas@zenith.tech>"]
 edition = "2018"
 
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
 [dependencies]
-bookfile = "^0.3"
 chrono = "0.4.19"
 rand = "0.8.3"
 regex = "1.4.5"
 bytes = { version = "1.0.1", features = ['serde'] }
 byteorder = "1.4.3"
 futures = "0.3.13"
-hyper = "0.14"
 lazy_static = "1.4.0"
+slog-stdlog = "4.1.0"
+slog-async = "2.6.0"
+slog-scope = "4.4.0"
+slog-term = "2.8.0"
+slog = "2.7.0"
 log = "0.4.14"
 clap = "2.33.0"
+termion = "1.5.6"
+tui = "0.14.0"
 daemonize = "0.4.1"
-tokio = { version = "1.11", features = ["process", "macros", "fs", "rt"] }
+rust-s3 = { version = "0.27.0-rc4", features = ["no-verify-ssl"] }
+tokio = { version = "1.3.0", features = ["full"] }
+tokio-stream = { version = "0.1.4" }
 postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-routerify = "2"
+# by default rust-rocksdb tries to build a lot of compression algos. Use lz4 only for now as it is simplest dependency.
+rocksdb = { version = "0.16.0", features = ["lz4"], default-features = false }
 anyhow = "1.0"
 crc32c = "0.6.0"
+walkdir = "2"
 thiserror = "1.0"
-hex = { version = "0.4.3", features = ["serde"] }
+hex = "0.4.3"
 tar = "0.4.33"
 humantime = "2.1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
+fs_extra = "1.2.0"
 toml = "0.5"
-scopeguard = "1.1.0"
-rust-s3 = { version = "0.27.0-rc4", features = ["no-verify-ssl"] }
-async-trait = "0.1"
-const_format = "0.2.21"
-tracing = "0.1.27"
-signal-hook = {version = "0.3.10", features = ["extended-siginfo"] }
 
 postgres_ffi = { path = "../postgres_ffi" }
-zenith_metrics = { path = "../zenith_metrics" }
 zenith_utils = { path = "../zenith_utils" }
 workspace_hack = { path = "../workspace_hack" }
-
-[dev-dependencies]
-hex-literal = "0.3"

pageserver/README

@@ -1,5 +1,16 @@
 ## Page server architecture
 
+The Page Server is responsible for all operations on a number of
+"chunks" of relation data. A chunk corresponds to a PostgreSQL
+relation segment (i.e. one max. 1 GB file in the data directory), but
+it holds all the different versions of every page in the segment that
+are still needed by the system.
+
+Currently we do not specifically organize data in chunks.
+All page images and corresponding WAL records are stored as entries in a key-value storage,
+where StorageKey is a zenith_timeline_id + BufferTag + LSN.
+
+
 The Page Server has a few different duties:
 
 - Respond to GetPage@LSN requests from the Compute Nodes
@@ -7,12 +18,10 @@ The Page Server has a few different duties:
 - Replay WAL that's applicable to the chunks that the Page Server maintains
 - Backup to S3
 
-S3 is the main fault-tolerant storage of all data, as there are no Page Server
-replicas. We use a separate fault-tolerant WAL service to reduce latency. It
-keeps track of WAL records which are not syncted to S3 yet.
 
 The Page Server consists of multiple threads that operate on a shared
-repository of page versions:
+cache of page versions:
+
 
                                            | WAL
                                            V
@@ -25,14 +34,16 @@ repository of page versions:
                   +---------+                              ..........            |    |
                   |         |                              .        .            |    |
  GetPage@LSN      |         |                              . backup .  ------->  | S3 |
-------------->    |  Page   |         repository           .        .            |    |
+------------->    |  Page   |         page cache           .        .            |    |
                   | Service |                              ..........            |    |
    page           |         |                                                    +----+
 <-------------    |         |
-                  +---------+      +--------------------+
-		                   |   Checkpointing /  |
-				   | Garbage collection |
-                                   +--------------------+
+                  +---------+
+
+                             ...................................
+                             .                                 .
+                             . Garbage Collection / Compaction .
+                             ...................................
 
 Legend:
 
@@ -52,7 +63,7 @@ Page Service
 ------------
 
 The Page Service listens for GetPage@LSN requests from the Compute Nodes,
-and responds with pages from the repository.
+and responds with pages from the page cache.
 
 
 WAL Receiver
@@ -61,59 +72,47 @@ WAL Receiver
 The WAL receiver connects to the external WAL safekeeping service (or
 directly to the primary) using PostgreSQL physical streaming
 replication, and continuously receives WAL. It decodes the WAL records,
-and stores them to the repository.
+and stores them to the page cache.
 
 
-Repository
+Page Cache
 ----------
 
-The repository stores all the page versions, or WAL records needed to
-reconstruct them. Each tenant has a separate Repository, which is
-stored in the .zenith/tenants/<tenantid> directory.
+The Page Cache is a switchboard to access different Repositories.
 
-Repository is an abstract trait, defined in `repository.rs`. It is
-implemented by the LayeredRepository object in
-`layered_repository.rs`. There is only that one implementation of the
-Repository trait, but it's still a useful abstraction that keeps the
-interface for the low-level storage functionality clean. The layered
-storage format is described in layered_repository/README.md.
+#### Repository
+Repository corresponds to one .zenith directory.
+Repository is needed to manage Timelines.
 
-Each repository consists of multiple Timelines. Timeline is a
-workhorse that accepts page changes from the WAL, and serves
-get_page_at_lsn() and get_rel_size() requests. Note: this has nothing
-to do with PostgreSQL WAL timeline. The term "timeline" is mostly
-interchangeable with "branch", there is a one-to-one mapping from
-branch to timeline. A timeline has a unique ID within the tenant,
-represented as 16-byte hex string that never changes, whereas a
-branch is a user-given name for a timeline.
+#### Timeline
+Timeline is a page cache workhorse that accepts page changes
+and serves get_page_at_lsn() and get_rel_size() requests.
+Note: this has nothing to do with PostgreSQL WAL timeline.
 
-Each repository also has a WAL redo manager associated with it, see
-`walredo.rs`. The WAL redo manager is used to replay PostgreSQL WAL
-records, whenever we need to reconstruct a page version from WAL to
-satisfy a GetPage@LSN request, or to avoid accumulating too much WAL
-for a page. The WAL redo manager uses a Postgres process running in
-special zenith wal-redo mode to do the actual WAL redo, and
-communicates with the process using a pipe.
+#### Branch
+We can create branch at certain LSN.
+Each Branch lives in a corresponding timeline and has an ancestor.
+
+To get full snapshot of data at certain moment we need to traverse timeline and its ancestors.
+
+#### ObjectRepository
+ObjectRepository implements Repository and has associated ObjectStore and WAL redo service.
+
+#### ObjectStore
+ObjectStore is an interface for key-value store for page images and wal records.
+Currently it has one implementation - RocksDB.
+
+#### WAL redo service
+WAL redo service - service that runs PostgreSQL in a special wal_redo mode
+to apply given WAL records over an old page image and return new page image.
 
 
-Checkpointing / Garbage Collection
-----------------------------------
+TODO: Garbage Collection / Compaction
+-------------------------------------
 
-Periodically, the checkpointer thread wakes up and performs housekeeping
-duties on the repository. It has two duties:
-
-### Checkpointing
-
-Flush WAL that has accumulated in memory to disk, so that the old WAL
-can be truncated away in the WAL safekeepers. Also, to free up memory
-for receiving new WAL. This process is called "checkpointing". It's
-similar to checkpointing in PostgreSQL or other DBMSs, but in the page
-server, checkpointing happens on a per-segment basis.
-
-### Garbage collection
-
-Remove old on-disk layer files that are no longer needed according to the
-PITR retention policy
+Periodically, the Garbage Collection / Compaction thread runs
+and applies pending WAL records, and removes old page versions that
+are no longer needed.
 
 
 TODO: Backup service

pageserver/src/basebackup.rs

@@ -4,23 +4,22 @@
 //! TODO: this module has nothing to do with PostgreSQL pg_basebackup.
 //! It could use a better name.
 //!
-//! Stateless Postgres compute node is launched by sending a tarball
-//! which contains non-relational data (multixacts, clog, filenodemaps, twophase files),
-//! generated pg_control and dummy segment of WAL.
-//! This module is responsible for creation of such tarball
-//! from data stored in object storage.
+//! Stateless Postgres compute node is launched by sending tarball which contains non-relational data (multixacts, clog, filenodemaps, twophase files)
+//! and generate pg_control and dummy segment of WAL. This module is responsible for creation of such tarball from snapshot directory and
+//! data stored in object storage.
 //!
-use anyhow::Result;
+use crate::ZTimelineId;
 use bytes::{BufMut, BytesMut};
 use log::*;
-use std::io;
 use std::io::Write;
 use std::sync::Arc;
 use std::time::SystemTime;
-use tar::{Builder, EntryType, Header};
+use tar::{Builder, Header};
+use walkdir::WalkDir;
 
-use crate::relish::*;
+use crate::object_key::*;
 use crate::repository::Timeline;
+use postgres_ffi::relfile_utils::*;
 use postgres_ffi::xlog_utils::*;
 use postgres_ffi::*;
 use zenith_utils::lsn::Lsn;
@@ -31,105 +30,92 @@ use zenith_utils::lsn::Lsn;
 pub struct Basebackup<'a> {
     ar: Builder<&'a mut dyn Write>,
     timeline: &'a Arc<dyn Timeline>,
-    pub lsn: Lsn,
+    lsn: Lsn,
     prev_record_lsn: Lsn,
+    snappath: String,
+    slru_buf: [u8; pg_constants::SLRU_SEG_SIZE],
+    slru_segno: u32,
+    slru_path: &'static str,
 }
 
-// Create basebackup with non-rel data in it. Omit relational data.
-//
-// Currently we use empty lsn in two cases:
-//  * During the basebackup right after timeline creation
-//  * When working without safekeepers. In this situation it is important to match the lsn
-//    we are taking basebackup on with the lsn that is used in pageserver's walreceiver
-//    to start the replication.
 impl<'a> Basebackup<'a> {
     pub fn new(
         write: &'a mut dyn Write,
+        timelineid: ZTimelineId,
         timeline: &'a Arc<dyn Timeline>,
-        req_lsn: Option<Lsn>,
-    ) -> Result<Basebackup<'a>> {
-        // Compute postgres doesn't have any previous WAL files, but the first
-        // record that it's going to write needs to include the LSN of the
-        // previous record (xl_prev). We include prev_record_lsn in the
-        // "zenith.signal" file, so that postgres can read it during startup.
-        //
-        // We don't keep full history of record boundaries in the page server,
-        // however, only the predecessor of the latest record on each
-        // timeline. So we can only provide prev_record_lsn when you take a
-        // base backup at the end of the timeline, i.e. at last_record_lsn.
-        // Even at the end of the timeline, we sometimes don't have a valid
-        // prev_lsn value; that happens if the timeline was just branched from
-        // an old LSN and it doesn't have any WAL of its own yet. We will set
-        // prev_lsn to Lsn(0) if we cannot provide the correct value.
-        let (backup_prev, backup_lsn) = if let Some(req_lsn) = req_lsn {
-            // Backup was requested at a particular LSN. Wait for it to arrive.
-            timeline.wait_lsn(req_lsn)?;
-
-            // If the requested point is the end of the timeline, we can
-            // provide prev_lsn. (get_last_record_rlsn() might return it as
-            // zero, though, if no WAL has been generated on this timeline
-            // yet.)
-            let end_of_timeline = timeline.get_last_record_rlsn();
-            if req_lsn == end_of_timeline.last {
-                (end_of_timeline.prev, req_lsn)
-            } else {
-                (Lsn(0), req_lsn)
-            }
-        } else {
-            // Backup was requested at end of the timeline.
-            let end_of_timeline = timeline.get_last_record_rlsn();
-            (end_of_timeline.prev, end_of_timeline.last)
-        };
-
-        info!(
-            "taking basebackup lsn={}, prev_lsn={}",
-            backup_prev, backup_lsn
-        );
-
-        Ok(Basebackup {
+        lsn: Lsn,
+        prev_record_lsn: Lsn,
+        snapshot_lsn: Lsn,
+    ) -> Basebackup<'a> {
+        Basebackup {
             ar: Builder::new(write),
             timeline,
-            lsn: backup_lsn,
-            prev_record_lsn: backup_prev,
-        })
+            lsn,
+            prev_record_lsn,
+            snappath: format!("timelines/{}/snapshots/{:016X}", timelineid, snapshot_lsn.0),
+            slru_path: "",
+            slru_segno: u32::MAX,
+            slru_buf: [0u8; pg_constants::SLRU_SEG_SIZE],
+        }
     }
 
     pub fn send_tarball(&mut self) -> anyhow::Result<()> {
-        // Create pgdata subdirs structure
-        for dir in pg_constants::PGDATA_SUBDIRS.iter() {
-            let header = new_tar_header_dir(*dir)?;
-            self.ar.append(&header, &mut io::empty())?;
-        }
+        debug!("sending tarball of snapshot in {}", self.snappath);
+        for entry in WalkDir::new(&self.snappath) {
+            let entry = entry?;
+            let fullpath = entry.path();
+            let relpath = entry.path().strip_prefix(&self.snappath).unwrap();
 
-        // Send empty config files.
-        for filepath in pg_constants::PGDATA_SPECIAL_FILES.iter() {
-            if *filepath == "pg_hba.conf" {
-                let data = pg_constants::PG_HBA.as_bytes();
-                let header = new_tar_header(filepath, data.len() as u64)?;
-                self.ar.append(&header, data)?;
+            if relpath.to_str().unwrap() == "" {
+                continue;
+            }
+
+            if entry.file_type().is_dir() {
+                trace!(
+                    "sending dir {} as {}",
+                    fullpath.display(),
+                    relpath.display()
+                );
+                self.ar.append_dir(relpath, fullpath)?;
+            } else if entry.file_type().is_symlink() {
+                error!("ignoring symlink in snapshot dir");
+            } else if entry.file_type().is_file() {
+                if !is_rel_file_path(relpath.to_str().unwrap()) {
+                    if entry.file_name() != "pg_filenode.map" // this files will be generated from object storage
+                        && !relpath.starts_with("pg_xact/")
+                        && !relpath.starts_with("pg_multixact/")
+                    {
+                        trace!("sending {}", relpath.display());
+                        self.ar.append_path_with_name(fullpath, relpath)?;
+                    }
+                } else {
+                    // relation pages are loaded on demand and should not be included in tarball
+                    trace!("not sending {}", relpath.display());
+                }
             } else {
-                let header = new_tar_header(filepath, 0)?;
-                self.ar.append(&header, &mut io::empty())?;
+                error!("unknown file type: {}", fullpath.display());
             }
         }
 
-        // Gather non-relational files from object storage pages.
+        // Generate non-relational files.
+        // Iteration is sorted order: all objects of the same time are grouped and traversed
+        // in key ascending order. For example all pg_xact records precede pg_multixact records and are sorted by block number.
+        // It allows to easily construct SLRU segments (32 blocks).
         for obj in self.timeline.list_nonrels(self.lsn)? {
             match obj {
-                RelishTag::Slru { slru, segno } => {
-                    self.add_slru_segment(slru, segno)?;
+                ObjectTag::Clog(slru) => self.add_slru_segment("pg_xact", &obj, slru.blknum)?,
+                ObjectTag::MultiXactMembers(slru) => {
+                    self.add_slru_segment("pg_multixact/members", &obj, slru.blknum)?
                 }
-                RelishTag::FileNodeMap { spcnode, dbnode } => {
-                    self.add_relmap_file(spcnode, dbnode)?;
-                }
-                RelishTag::TwoPhase { xid } => {
-                    self.add_twophase_file(xid)?;
+                ObjectTag::MultiXactOffsets(slru) => {
+                    self.add_slru_segment("pg_multixact/offsets", &obj, slru.blknum)?
                 }
+                ObjectTag::FileNodeMap(db) => self.add_relmap_file(&obj, &db)?,
+                ObjectTag::TwoPhase(prepare) => self.add_twophase_file(&obj, prepare.xid)?,
                 _ => {}
             }
         }
-
-        // Generate pg_control and bootstrap WAL segment.
+        self.finish_slru_segment()?; // write last non-completed SLRU segment (if any)
         self.add_pgcontrol_file()?;
         self.ar.finish()?;
         debug!("all tarred up!");
@@ -137,77 +123,70 @@ impl<'a> Basebackup<'a> {
     }
 
     //
-    // Generate SLRU segment files from repository.
+    // Generate SLRU segment files from repository. Path identifies SLRU kind (pg_xact, pg_multixact/members, ...).
+    // Intially pass an empty string.
     //
-    fn add_slru_segment(&mut self, slru: SlruKind, segno: u32) -> anyhow::Result<()> {
-        let seg_size = self
+    fn add_slru_segment(
+        &mut self,
+        path: &'static str,
+        tag: &ObjectTag,
+        page: u32,
+    ) -> anyhow::Result<()> {
+        let img = self
             .timeline
-            .get_relish_size(RelishTag::Slru { slru, segno }, self.lsn)?;
-
-        if seg_size == None {
-            trace!(
-                "SLRU segment {}/{:>04X} was truncated",
-                slru.to_str(),
-                segno
-            );
-            return Ok(());
-        }
-
-        let nblocks = seg_size.unwrap();
-
-        let mut slru_buf: Vec<u8> =
-            Vec::with_capacity(nblocks as usize * pg_constants::BLCKSZ as usize);
-        for blknum in 0..nblocks {
-            let img =
-                self.timeline
-                    .get_page_at_lsn(RelishTag::Slru { slru, segno }, blknum, self.lsn)?;
+            .get_page_at_lsn_nowait(*tag, self.lsn, false)?;
+        // Zero length image indicates truncated segment: just skip it
+        if !img.is_empty() {
             assert!(img.len() == pg_constants::BLCKSZ as usize);
-
-            slru_buf.extend_from_slice(&img);
+            let segno = page / pg_constants::SLRU_PAGES_PER_SEGMENT;
+            if self.slru_path != "" && (self.slru_segno != segno || self.slru_path != path) {
+                // Switch to new segment: save old one
+                let segname = format!("{}/{:>04X}", self.slru_path, self.slru_segno);
+                let header = new_tar_header(&segname, pg_constants::SLRU_SEG_SIZE as u64)?;
+                self.ar.append(&header, &self.slru_buf[..])?;
+                self.slru_buf = [0u8; pg_constants::SLRU_SEG_SIZE]; // reinitialize segment buffer
+            }
+            self.slru_segno = segno;
+            self.slru_path = path;
+            let offs_start = (page % pg_constants::SLRU_PAGES_PER_SEGMENT) as usize
+                * pg_constants::BLCKSZ as usize;
+            let offs_end = offs_start + pg_constants::BLCKSZ as usize;
+            self.slru_buf[offs_start..offs_end].copy_from_slice(&img);
         }
+        Ok(())
+    }
 
-        let segname = format!("{}/{:>04X}", slru.to_str(), segno);
-        let header = new_tar_header(&segname, slru_buf.len() as u64)?;
-        self.ar.append(&header, slru_buf.as_slice())?;
-
-        trace!("Added to basebackup slru {} relsize {}", segname, nblocks);
+    //
+    // We flush SLRU segments to the tarball once them are completed.
+    // This method is used to flush last (may be incompleted) segment.
+    //
+    fn finish_slru_segment(&mut self) -> anyhow::Result<()> {
+        if self.slru_path != "" {
+            // is there is some incompleted segment
+            let segname = format!("{}/{:>04X}", self.slru_path, self.slru_segno);
+            let header = new_tar_header(&segname, pg_constants::SLRU_SEG_SIZE as u64)?;
+            self.ar.append(&header, &self.slru_buf[..])?;
+        }
         Ok(())
     }
 
     //
     // Extract pg_filenode.map files from repository
-    // Along with them also send PG_VERSION for each database.
     //
-    fn add_relmap_file(&mut self, spcnode: u32, dbnode: u32) -> anyhow::Result<()> {
-        let img = self.timeline.get_page_at_lsn(
-            RelishTag::FileNodeMap { spcnode, dbnode },
-            0,
-            self.lsn,
-        )?;
-        let path = if spcnode == pg_constants::GLOBALTABLESPACE_OID {
-            let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
-            let header = new_tar_header("PG_VERSION", version_bytes.len() as u64)?;
-            self.ar.append(&header, version_bytes)?;
-
-            let header = new_tar_header("global/PG_VERSION", version_bytes.len() as u64)?;
-            self.ar.append(&header, version_bytes)?;
-
+    fn add_relmap_file(&mut self, tag: &ObjectTag, db: &DatabaseTag) -> anyhow::Result<()> {
+        let img = self
+            .timeline
+            .get_page_at_lsn_nowait(*tag, self.lsn, false)?;
+        info!("add_relmap_file {:?}", db);
+        let path = if db.spcnode == pg_constants::GLOBALTABLESPACE_OID {
             String::from("global/pg_filenode.map") // filenode map for global tablespace
         } else {
             // User defined tablespaces are not supported
-            assert!(spcnode == pg_constants::DEFAULTTABLESPACE_OID);
-
-            // Append dir path for each database
-            let path = format!("base/{}", dbnode);
-            let header = new_tar_header_dir(&path)?;
-            self.ar.append(&header, &mut io::empty())?;
-
-            let dst_path = format!("base/{}/PG_VERSION", dbnode);
-            let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
-            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
-            self.ar.append(&header, version_bytes)?;
-
-            format!("base/{}/pg_filenode.map", dbnode)
+            assert!(db.spcnode == pg_constants::DEFAULTTABLESPACE_OID);
+            let src_path = format!("{}/base/1/PG_VERSION", self.snappath);
+            let dst_path = format!("base/{}/PG_VERSION", db.dbnode);
+            self.ar.append_path_with_name(&src_path, &dst_path)?;
+            format!("base/{}/pg_filenode.map", db.dbnode)
         };
         assert!(img.len() == 512);
         let header = new_tar_header(&path, img.len() as u64)?;
@@ -218,33 +197,35 @@ impl<'a> Basebackup<'a> {
     //
     // Extract twophase state files
     //
-    fn add_twophase_file(&mut self, xid: TransactionId) -> anyhow::Result<()> {
-        let img = self
-            .timeline
-            .get_page_at_lsn(RelishTag::TwoPhase { xid }, 0, self.lsn)?;
-
-        let mut buf = BytesMut::new();
-        buf.extend_from_slice(&img[..]);
-        let crc = crc32c::crc32c(&img[..]);
-        buf.put_u32_le(crc);
-        let path = format!("pg_twophase/{:>08X}", xid);
-        let header = new_tar_header(&path, buf.len() as u64)?;
-        self.ar.append(&header, &buf[..])?;
-
+    fn add_twophase_file(&mut self, tag: &ObjectTag, xid: TransactionId) -> anyhow::Result<()> {
+        // Include in tarball two-phase files only of in-progress transactions
+        if self.timeline.get_tx_status(xid, self.lsn)?
+            == pg_constants::TRANSACTION_STATUS_IN_PROGRESS
+        {
+            let img = self
+                .timeline
+                .get_page_at_lsn_nowait(*tag, self.lsn, false)?;
+            let mut buf = BytesMut::new();
+            buf.extend_from_slice(&img[..]);
+            let crc = crc32c::crc32c(&img[..]);
+            buf.put_u32_le(crc);
+            let path = format!("pg_twophase/{:>08X}", xid);
+            let header = new_tar_header(&path, buf.len() as u64)?;
+            self.ar.append(&header, &buf[..])?;
+        }
         Ok(())
     }
 
     //
-    // Add generated pg_control file and bootstrap WAL segment.
-    // Also send zenith.signal file with extra bootstrap data.
+    // Add generated pg_control file
     //
     fn add_pgcontrol_file(&mut self) -> anyhow::Result<()> {
-        let checkpoint_bytes = self
-            .timeline
-            .get_page_at_lsn(RelishTag::Checkpoint, 0, self.lsn)?;
+        let checkpoint_bytes =
+            self.timeline
+                .get_page_at_lsn_nowait(ObjectTag::Checkpoint, self.lsn, false)?;
         let pg_control_bytes =
             self.timeline
-                .get_page_at_lsn(RelishTag::ControlFile, 0, self.lsn)?;
+                .get_page_at_lsn_nowait(ObjectTag::ControlFile, self.lsn, false)?;
         let mut pg_control = ControlFileData::decode(&pg_control_bytes)?;
         let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
 
@@ -255,27 +236,21 @@ impl<'a> Basebackup<'a> {
             XLOG_SIZE_OF_XLOG_LONG_PHD as u32,
             pg_constants::WAL_SEGMENT_SIZE,
         );
-        checkpoint.redo = normalize_lsn(self.lsn, pg_constants::WAL_SEGMENT_SIZE).0;
+        checkpoint.redo = self.lsn.0 + self.lsn.calc_padding(8u32);
 
         //reset some fields we don't want to preserve
-        //TODO Check this.
-        //We may need to determine the value from twophase data.
         checkpoint.oldestActiveXid = 0;
 
         //save new values in pg_control
         pg_control.checkPoint = checkpoint_lsn;
         pg_control.checkPointCopy = checkpoint;
+        info!("pg_control.state = {}", pg_control.state);
         pg_control.state = pg_constants::DB_SHUTDOWNED;
 
         // add zenith.signal file
-        let xl_prev = if self.prev_record_lsn == Lsn(0) {
-            0xBAD0 // magic value to indicate that we don't know prev_lsn
-        } else {
-            self.prev_record_lsn.0
-        };
         self.ar.append(
             &new_tar_header("zenith.signal", 8)?,
-            &xl_prev.to_le_bytes()[..],
+            &self.prev_record_lsn.0.to_le_bytes()[..],
         )?;
 
         //send pg_control
@@ -292,12 +267,65 @@ impl<'a> Basebackup<'a> {
         let wal_file_path = format!("pg_wal/{}", wal_file_name);
         let header = new_tar_header(&wal_file_path, pg_constants::WAL_SEGMENT_SIZE as u64)?;
         let wal_seg = generate_wal_segment(&pg_control);
-        assert!(wal_seg.len() == pg_constants::WAL_SEGMENT_SIZE);
         self.ar.append(&header, &wal_seg[..])?;
         Ok(())
     }
 }
 
+///
+/// Parse a path, relative to the root of PostgreSQL data directory, as
+/// a PostgreSQL relation data file.
+///
+fn parse_rel_file_path(path: &str) -> Result<(), FilePathError> {
+    /*
+     * Relation data files can be in one of the following directories:
+     *
+     * global/
+     *		shared relations
+     *
+     * base/<db oid>/
+     *		regular relations, default tablespace
+     *
+     * pg_tblspc/<tblspc oid>/<tblspc version>/
+     *		within a non-default tablespace (the name of the directory
+     *		depends on version)
+     *
+     * And the relation data files themselves have a filename like:
+     *
+     * <oid>.<segment number>
+     */
+    if let Some(fname) = path.strip_prefix("global/") {
+        let (_relnode, _forknum, _segno) = parse_relfilename(fname)?;
+
+        Ok(())
+    } else if let Some(dbpath) = path.strip_prefix("base/") {
+        let mut s = dbpath.split('/');
+        let dbnode_str = s.next().ok_or(FilePathError::InvalidFileName)?;
+        let _dbnode = dbnode_str.parse::<u32>()?;
+        let fname = s.next().ok_or(FilePathError::InvalidFileName)?;
+        if s.next().is_some() {
+            return Err(FilePathError::InvalidFileName);
+        };
+
+        let (_relnode, _forknum, _segno) = parse_relfilename(fname)?;
+
+        Ok(())
+    } else if path.strip_prefix("pg_tblspc/").is_some() {
+        // TODO
+        error!("tablespaces not implemented yet");
+        Err(FilePathError::InvalidFileName)
+    } else {
+        Err(FilePathError::InvalidFileName)
+    }
+}
+
+//
+// Check if it is relational file
+//
+fn is_rel_file_path(path: &str) -> bool {
+    parse_rel_file_path(path).is_ok()
+}
+
 //
 // Create new tarball entry header
 //
@@ -316,20 +344,3 @@ fn new_tar_header(path: &str, size: u64) -> anyhow::Result<Header> {
     header.set_cksum();
     Ok(header)
 }
-
-fn new_tar_header_dir(path: &str) -> anyhow::Result<Header> {
-    let mut header = Header::new_gnu();
-    header.set_size(0);
-    header.set_path(path)?;
-    header.set_mode(0o755); // -rw-------
-    header.set_entry_type(EntryType::dir());
-    header.set_mtime(
-        // use currenttime as last modified time
-        SystemTime::now()
-            .duration_since(SystemTime::UNIX_EPOCH)
-            .unwrap()
-            .as_secs(),
-    );
-    header.set_cksum();
-    Ok(header)
-}

pageserver/src/bin/dump_layerfile.rs

@@ -1,25 +0,0 @@
-//! Main entry point for the dump_layerfile executable
-//!
-//! A handy tool for debugging, that's all.
-use anyhow::Result;
-use clap::{App, Arg};
-use pageserver::layered_repository::dump_layerfile_from_path;
-use std::path::PathBuf;
-
-fn main() -> Result<()> {
-    let arg_matches = App::new("Zenith dump_layerfile utility")
-        .about("Dump contents of one layer file, for debugging")
-        .arg(
-            Arg::with_name("path")
-                .help("Path to file to dump")
-                .required(true)
-                .index(1),
-        )
-        .get_matches();
-
-    let path = PathBuf::from(arg_matches.value_of("path").unwrap());
-
-    dump_layerfile_from_path(&path)?;
-
-    Ok(())
-}

pageserver/src/bin/pageserver.rs

@@ -2,75 +2,42 @@
 // Main entry point for the Page Server executable
 //
 
-use pageserver::defaults::*;
+use log::*;
 use serde::{Deserialize, Serialize};
 use std::{
     env,
+    fs::{File, OpenOptions},
+    io,
     net::TcpListener,
     path::{Path, PathBuf},
-    str::FromStr,
+    process::exit,
     thread,
+    time::Duration,
 };
-use tracing::*;
-use zenith_utils::{auth::JwtAuth, logging, postgres_backend::AuthType};
-
-use anyhow::{bail, ensure, Context, Result};
-use signal_hook::consts::signal::*;
-use signal_hook::consts::TERM_SIGNALS;
-use signal_hook::flag;
-use signal_hook::iterator::exfiltrator::WithOrigin;
-use signal_hook::iterator::SignalsInfo;
-use std::process::exit;
-use std::sync::atomic::AtomicBool;
-use std::sync::Arc;
 
+use anyhow::{Context, Result};
 use clap::{App, Arg, ArgMatches};
 use daemonize::Daemonize;
 
-use pageserver::{
-    branches,
-    defaults::{
-        DEFAULT_HTTP_LISTEN_ADDR, DEFAULT_PG_LISTEN_ADDR,
-        DEFAULT_RELISH_STORAGE_MAX_CONCURRENT_SYNC_LIMITS,
-    },
-    http, page_service, relish_storage, tenant_mgr, PageServerConf, RelishStorageConfig,
-    RelishStorageKind, S3Config, LOG_FILE_NAME,
-};
-use zenith_utils::http::endpoint;
-use zenith_utils::postgres_backend;
+use slog::{Drain, FnValue};
 
-use const_format::formatcp;
+use pageserver::{branches, page_cache, page_service, tui, PageServerConf};
+
+const DEFAULT_LISTEN_ADDR: &str = "127.0.0.1:64000";
+
+const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
+const DEFAULT_GC_PERIOD: Duration = Duration::from_secs(100);
+
+const DEFAULT_WAL_REDOERS: usize = 1;
 
 /// String arguments that can be declared via CLI or config file
 #[derive(Serialize, Deserialize)]
 struct CfgFileParams {
-    listen_pg_addr: Option<String>,
-    listen_http_addr: Option<String>,
-    checkpoint_distance: Option<String>,
-    checkpoint_period: Option<String>,
+    listen_addr: Option<String>,
     gc_horizon: Option<String>,
     gc_period: Option<String>,
+    wal_redoers: Option<String>,
     pg_distrib_dir: Option<String>,
-    auth_validation_public_key_path: Option<String>,
-    auth_type: Option<String>,
-    // see https://github.com/alexcrichton/toml-rs/blob/6c162e6562c3e432bf04c82a3d1d789d80761a86/examples/enum_external.rs for enum deserialisation examples
-    relish_storage: Option<RelishStorage>,
-    relish_storage_max_concurrent_sync: Option<String>,
-}
-
-#[derive(Serialize, Deserialize, Clone)]
-enum RelishStorage {
-    Local {
-        local_path: String,
-    },
-    AwsS3 {
-        bucket_name: String,
-        bucket_region: String,
-        #[serde(skip_serializing)]
-        access_key_id: Option<String>,
-        #[serde(skip_serializing)]
-        secret_access_key: Option<String>,
-    },
 }
 
 impl CfgFileParams {
@@ -80,33 +47,12 @@ impl CfgFileParams {
             arg_matches.value_of(arg_name).map(str::to_owned)
         };
 
-        let relish_storage = if let Some(local_path) = get_arg("relish-storage-local-path") {
-            Some(RelishStorage::Local { local_path })
-        } else if let Some((bucket_name, bucket_region)) =
-            get_arg("relish-storage-s3-bucket").zip(get_arg("relish-storage-region"))
-        {
-            Some(RelishStorage::AwsS3 {
-                bucket_name,
-                bucket_region,
-                access_key_id: get_arg("relish-storage-access-key"),
-                secret_access_key: get_arg("relish-storage-secret-access-key"),
-            })
-        } else {
-            None
-        };
-
         Self {
-            listen_pg_addr: get_arg("listen-pg"),
-            listen_http_addr: get_arg("listen-http"),
-            checkpoint_distance: get_arg("checkpoint_distance"),
-            checkpoint_period: get_arg("checkpoint_period"),
+            listen_addr: get_arg("listen"),
             gc_horizon: get_arg("gc_horizon"),
             gc_period: get_arg("gc_period"),
+            wal_redoers: get_arg("wal_redoers"),
             pg_distrib_dir: get_arg("postgres-distrib"),
-            auth_validation_public_key_path: get_arg("auth-validation-public-key-path"),
-            auth_type: get_arg("auth-type"),
-            relish_storage,
-            relish_storage_max_concurrent_sync: get_arg("relish-storage-max-concurrent-sync"),
         }
     }
 
@@ -114,45 +60,19 @@ impl CfgFileParams {
     fn or(self, other: CfgFileParams) -> Self {
         // TODO cleaner way to do this
         Self {
-            listen_pg_addr: self.listen_pg_addr.or(other.listen_pg_addr),
-            listen_http_addr: self.listen_http_addr.or(other.listen_http_addr),
-            checkpoint_distance: self.checkpoint_distance.or(other.checkpoint_distance),
-            checkpoint_period: self.checkpoint_period.or(other.checkpoint_period),
+            listen_addr: self.listen_addr.or(other.listen_addr),
             gc_horizon: self.gc_horizon.or(other.gc_horizon),
             gc_period: self.gc_period.or(other.gc_period),
+            wal_redoers: self.wal_redoers.or(other.wal_redoers),
             pg_distrib_dir: self.pg_distrib_dir.or(other.pg_distrib_dir),
-            auth_validation_public_key_path: self
-                .auth_validation_public_key_path
-                .or(other.auth_validation_public_key_path),
-            auth_type: self.auth_type.or(other.auth_type),
-            relish_storage: self.relish_storage.or(other.relish_storage),
-            relish_storage_max_concurrent_sync: self
-                .relish_storage_max_concurrent_sync
-                .or(other.relish_storage_max_concurrent_sync),
         }
     }
 
     /// Create a PageServerConf from these string parameters
     fn try_into_config(&self) -> Result<PageServerConf> {
-        let workdir = PathBuf::from(".");
-
-        let listen_pg_addr = match self.listen_pg_addr.as_ref() {
+        let listen_addr = match self.listen_addr.as_ref() {
             Some(addr) => addr.clone(),
-            None => DEFAULT_PG_LISTEN_ADDR.to_owned(),
-        };
-
-        let listen_http_addr = match self.listen_http_addr.as_ref() {
-            Some(addr) => addr.clone(),
-            None => DEFAULT_HTTP_LISTEN_ADDR.to_owned(),
-        };
-
-        let checkpoint_distance: u64 = match self.checkpoint_distance.as_ref() {
-            Some(checkpoint_distance_str) => checkpoint_distance_str.parse()?,
-            None => DEFAULT_CHECKPOINT_DISTANCE,
-        };
-        let checkpoint_period = match self.checkpoint_period.as_ref() {
-            Some(checkpoint_period_str) => humantime::parse_duration(checkpoint_period_str)?,
-            None => DEFAULT_CHECKPOINT_PERIOD,
+            None => DEFAULT_LISTEN_ADDR.to_owned(),
         };
 
         let gc_horizon: u64 = match self.gc_horizon.as_ref() {
@@ -164,109 +84,58 @@ impl CfgFileParams {
             None => DEFAULT_GC_PERIOD,
         };
 
+        let wal_redoers = match self.wal_redoers.as_ref() {
+            Some(wal_redoers_str) => wal_redoers_str.parse::<usize>()?,
+            None => DEFAULT_WAL_REDOERS,
+        };
+
         let pg_distrib_dir = match self.pg_distrib_dir.as_ref() {
             Some(pg_distrib_dir_str) => PathBuf::from(pg_distrib_dir_str),
             None => env::current_dir()?.join("tmp_install"),
         };
 
-        let auth_validation_public_key_path = self
-            .auth_validation_public_key_path
-            .as_ref()
-            .map(PathBuf::from);
-
-        let auth_type = self
-            .auth_type
-            .as_ref()
-            .map_or(Ok(AuthType::Trust), |auth_type| {
-                AuthType::from_str(auth_type)
-            })?;
-
         if !pg_distrib_dir.join("bin/postgres").exists() {
-            bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
+            anyhow::bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
         }
 
-        if auth_type == AuthType::ZenithJWT {
-            ensure!(
-                auth_validation_public_key_path.is_some(),
-                "Missing auth_validation_public_key_path when auth_type is ZenithJWT"
-            );
-            let path_ref = auth_validation_public_key_path.as_ref().unwrap();
-            ensure!(
-                path_ref.exists(),
-                format!("Can't find auth_validation_public_key at {:?}", path_ref)
-            );
-        }
-
-        let max_concurrent_sync = match self.relish_storage_max_concurrent_sync.as_deref() {
-            Some(relish_storage_max_concurrent_sync) => {
-                relish_storage_max_concurrent_sync.parse()?
-            }
-            None => DEFAULT_RELISH_STORAGE_MAX_CONCURRENT_SYNC_LIMITS,
-        };
-        let relish_storage_config = self.relish_storage.as_ref().map(|storage_params| {
-            let storage = match storage_params.clone() {
-                RelishStorage::Local { local_path } => {
-                    RelishStorageKind::LocalFs(PathBuf::from(local_path))
-                }
-                RelishStorage::AwsS3 {
-                    bucket_name,
-                    bucket_region,
-                    access_key_id,
-                    secret_access_key,
-                } => RelishStorageKind::AwsS3(S3Config {
-                    bucket_name,
-                    bucket_region,
-                    access_key_id,
-                    secret_access_key,
-                }),
-            };
-            RelishStorageConfig {
-                max_concurrent_sync,
-                storage,
-            }
-        });
-
         Ok(PageServerConf {
             daemonize: false,
+            interactive: false,
+            materialize: false,
 
-            listen_pg_addr,
-            listen_http_addr,
-            checkpoint_distance,
-            checkpoint_period,
+            listen_addr,
             gc_horizon,
             gc_period,
-
-            superuser: String::from(DEFAULT_SUPERUSER),
-
-            workdir,
+            wal_redoers,
+            workdir: PathBuf::from("."),
 
             pg_distrib_dir,
-
-            auth_validation_public_key_path,
-            auth_type,
-            relish_storage_config,
         })
     }
 }
 
 fn main() -> Result<()> {
-    zenith_metrics::set_common_metrics_prefix("pageserver");
     let arg_matches = App::new("Zenith page server")
         .about("Materializes WAL stream to pages and serves them to the postgres")
         .arg(
-            Arg::with_name("listen-pg")
+            Arg::with_name("listen")
                 .short("l")
-                .long("listen-pg")
-                .alias("listen") // keep some compatibility
+                .long("listen")
                 .takes_value(true)
-                .help(formatcp!("listen for incoming page requests on ip:port (default: {DEFAULT_PG_LISTEN_ADDR})")),
+                .help("listen for incoming page requests on ip:port (default: 127.0.0.1:5430)"),
         )
         .arg(
-            Arg::with_name("listen-http")
-                .long("listen-http")
-                .alias("http_endpoint") // keep some compatibility
-                .takes_value(true)
-                .help(formatcp!("http endpoint address for metrics and management API calls on ip:port (default: {DEFAULT_HTTP_LISTEN_ADDR})")),
+            Arg::with_name("interactive")
+                .short("i")
+                .long("interactive")
+                .takes_value(false)
+                .help("Interactive mode"),
+        )
+        .arg(
+            Arg::with_name("materialize")
+                .long("materialize")
+                .takes_value(false)
+                .help("Materialize pages constructed by get_page_at"),
         )
         .arg(
             Arg::with_name("daemonize")
@@ -281,18 +150,6 @@ fn main() -> Result<()> {
                 .takes_value(false)
                 .help("Initialize pageserver repo"),
         )
-        .arg(
-            Arg::with_name("checkpoint_distance")
-                .long("checkpoint_distance")
-                .takes_value(true)
-                .help("Distance from current LSN to perform checkpoint of in-memory layers"),
-        )
-        .arg(
-            Arg::with_name("checkpoint_period")
-                .long("checkpoint_period")
-                .takes_value(true)
-                .help("Interval between checkpoint iterations"),
-        )
         .arg(
             Arg::with_name("gc_horizon")
                 .long("gc_horizon")
@@ -305,6 +162,12 @@ fn main() -> Result<()> {
                 .takes_value(true)
                 .help("Interval between garbage collector iterations"),
         )
+        .arg(
+            Arg::with_name("wal_redoers")
+                .long("wal_redoers")
+                .takes_value(true)
+                .help("Number of wal-redo postgres instances"),
+        )
         .arg(
             Arg::with_name("workdir")
                 .short("D")
@@ -318,117 +181,39 @@ fn main() -> Result<()> {
                 .takes_value(true)
                 .help("Postgres distribution directory"),
         )
-        .arg(
-            Arg::with_name("create-tenant")
-                .long("create-tenant")
-                .takes_value(true)
-                .help("Create tenant during init")
-                .requires("init"),
-        )
-        .arg(
-            Arg::with_name("auth-validation-public-key-path")
-                .long("auth-validation-public-key-path")
-                .takes_value(true)
-                .help("Path to public key used to validate jwt signature"),
-        )
-        .arg(
-            Arg::with_name("auth-type")
-                .long("auth-type")
-                .takes_value(true)
-                .help("Authentication scheme type. One of: Trust, MD5, ZenithJWT"),
-        )
-        .arg(
-            Arg::with_name("relish-storage-local-path")
-                .long("relish-storage-local-path")
-                .takes_value(true)
-                .help("Path to the local directory, to be used as an external relish storage")
-                .conflicts_with_all(&[
-                    "relish-storage-s3-bucket",
-                    "relish-storage-region",
-                    "relish-storage-access-key",
-                    "relish-storage-secret-access-key",
-                ]),
-        )
-        .arg(
-            Arg::with_name("relish-storage-s3-bucket")
-                .long("relish-storage-s3-bucket")
-                .takes_value(true)
-                .help("Name of the AWS S3 bucket to use an external relish storage")
-                .requires("relish-storage-region"),
-        )
-        .arg(
-            Arg::with_name("relish-storage-region")
-                .long("relish-storage-region")
-                .takes_value(true)
-                .help("Region of the AWS S3 bucket"),
-        )
-        .arg(
-            Arg::with_name("relish-storage-access-key")
-                .long("relish-storage-access-key")
-                .takes_value(true)
-                .help("Credentials to access the AWS S3 bucket"),
-        )
-        .arg(
-            Arg::with_name("relish-storage-secret-access-key")
-                .long("relish-storage-secret-access-key")
-                .takes_value(true)
-                .help("Credentials to access the AWS S3 bucket"),
-        )
-        .arg(
-            Arg::with_name("relish-storage-max-concurrent-sync")
-                .long("relish-storage-max-concurrent-sync")
-                .takes_value(true)
-                .help("Maximum allowed concurrent synchronisations with storage"),
-        )
         .get_matches();
 
     let workdir = Path::new(arg_matches.value_of("workdir").unwrap_or(".zenith"));
-    let cfg_file_path = workdir
-        .canonicalize()
-        .with_context(|| format!("Error opening workdir '{}'", workdir.display()))?
-        .join("pageserver.toml");
+    let cfg_file_path = workdir.canonicalize()?.join("pageserver.toml");
 
     let args_params = CfgFileParams::from_args(&arg_matches);
 
     let init = arg_matches.is_present("init");
-    let create_tenant = arg_matches.value_of("create-tenant");
-
     let params = if init {
         // We're initializing the repo, so there's no config file yet
         args_params
     } else {
         // Supplement the CLI arguments with the config file
-        let cfg_file_contents = std::fs::read_to_string(&cfg_file_path)
-            .with_context(|| format!("No pageserver config at '{}'", cfg_file_path.display()))?;
-        let file_params: CfgFileParams = toml::from_str(&cfg_file_contents).with_context(|| {
-            format!(
-                "Failed to read '{}' as pageserver config",
-                cfg_file_path.display()
-            )
-        })?;
+        let cfg_file_contents = std::fs::read_to_string(&cfg_file_path)?;
+        let file_params: CfgFileParams = toml::from_str(&cfg_file_contents)?;
         args_params.or(file_params)
     };
 
-    // Set CWD to workdir for non-daemon modes
-    env::set_current_dir(&workdir).with_context(|| {
-        format!(
-            "Failed to set application's current dir to '{}'",
-            workdir.display()
-        )
-    })?;
-
     // Ensure the config is valid, even if just init-ing
-    let mut conf = params.try_into_config().with_context(|| {
-        format!(
-            "Pageserver config at '{}' is not valid",
-            cfg_file_path.display()
-        )
-    })?;
+    let mut conf = params.try_into_config()?;
 
     conf.daemonize = arg_matches.is_present("daemonize");
+    conf.interactive = arg_matches.is_present("interactive");
+    conf.materialize = arg_matches.is_present("materialize");
 
-    if init && conf.daemonize {
-        bail!("--daemonize cannot be used with --init")
+    if init && (conf.daemonize || conf.interactive) {
+        eprintln!("--daemonize and --interactive may not be used with --init");
+        exit(1);
+    }
+
+    if conf.daemonize && conf.interactive {
+        eprintln!("--daemonize is not allowed with --interactive: choose one");
+        exit(1);
     }
 
     // The configuration is all set up now. Turn it into a 'static
@@ -438,57 +223,59 @@ fn main() -> Result<()> {
 
     // Create repo and exit if init was requested
     if init {
-        branches::init_pageserver(conf, create_tenant).context("Failed to init pageserver")?;
+        branches::init_repo(conf, &workdir)?;
+
         // write the config file
-        let cfg_file_contents = toml::to_string_pretty(&params)
-            .context("Failed to create pageserver config contents for initialisation")?;
-        // TODO support enable-auth flag
-        std::fs::write(&cfg_file_path, cfg_file_contents).with_context(|| {
-            format!(
-                "Failed to initialize pageserver config at '{}'",
-                cfg_file_path.display()
-            )
-        })?;
-        Ok(())
-    } else {
-        start_pageserver(conf).context("Failed to start pageserver")
+        let cfg_file_contents = toml::to_string_pretty(&params)?;
+        std::fs::write(&cfg_file_path, cfg_file_contents)?;
+
+        return Ok(());
     }
+
+    // Set CWD to workdir for non-daemon modes
+    env::set_current_dir(&workdir)?;
+
+    start_pageserver(conf)
 }
 
 fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
-    // Initialize logger
-    let log_file = logging::init(LOG_FILE_NAME, conf.daemonize)?;
+    let log_filename = "pageserver.log";
+    // Don't open the same file for output multiple times;
+    // the different fds could overwrite each other's output.
+    let log_file = OpenOptions::new()
+        .create(true)
+        .append(true)
+        .open(&log_filename)
+        .with_context(|| format!("failed to open {:?}", &log_filename))?;
 
-    let term_now = Arc::new(AtomicBool::new(false));
-    for sig in TERM_SIGNALS {
-        // When terminated by a second term signal, exit with exit code 1.
-        // This will do nothing the first time (because term_now is false).
-        flag::register_conditional_shutdown(*sig, 1, Arc::clone(&term_now))?;
-        // But this will "arm" the above for the second time, by setting it to true.
-        // The order of registering these is important, if you put this one first, it will
-        // first arm and then terminate ‒ all in the first round.
-        flag::register(*sig, Arc::clone(&term_now))?;
-    }
+    // Initialize logger
+    let logger_file = log_file.try_clone().unwrap();
+    let _scope_guard = init_logging(&conf, logger_file)?;
+    let _log_guard = slog_stdlog::init()?;
+
+    // Note: this `info!(...)` macro comes from `log` crate
+    info!("standard logging redirected to slog");
+
+    let tui_thread = if conf.interactive {
+        // Initialize the UI
+        Some(
+            thread::Builder::new()
+                .name("UI thread".into())
+                .spawn(|| {
+                    let _ = tui::ui_main();
+                })
+                .unwrap(),
+        )
+    } else {
+        None
+    };
 
     // TODO: Check that it looks like a valid repository before going further
 
-    // bind sockets before daemonizing so we report errors early and do not return until we are listening
-    info!(
-        "Starting pageserver http handler on {}",
-        conf.listen_http_addr
-    );
-    let http_listener = TcpListener::bind(conf.listen_http_addr.clone())?;
-
-    info!(
-        "Starting pageserver pg protocol handler on {}",
-        conf.listen_pg_addr
-    );
-    let pageserver_listener = TcpListener::bind(conf.listen_pg_addr.clone())?;
-
     if conf.daemonize {
         info!("daemonizing...");
 
-        // There shouldn't be any logging to stdin/stdout. Redirect it to the main log so
+        // There should'n be any logging to stdin/stdout. Redirect it to the main log so
         // that we will see any accidental manual fprintf's or backtraces.
         let stdout = log_file.try_clone().unwrap();
         let stderr = log_file;
@@ -501,87 +288,84 @@ fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
 
         match daemonize.start() {
             Ok(_) => info!("Success, daemonized"),
-            Err(err) => error!(%err, "could not daemonize"),
+            Err(e) => error!("Error, {}", e),
         }
     }
 
-    // keep join handles for spawned threads
-    // don't spawn threads before daemonizing
-    let mut join_handles = Vec::new();
+    // Check that we can bind to address before starting threads to simplify shutdown
+    // sequence if port is occupied.
+    info!("Starting pageserver on {}", conf.listen_addr);
+    let pageserver_listener = TcpListener::bind(conf.listen_addr.clone())?;
 
-    if let Some(handle) = relish_storage::run_storage_sync_thread(conf)? {
-        join_handles.push(handle);
-    }
-    // Initialize tenant manager.
-    tenant_mgr::init(conf);
-
-    // initialize authentication for incoming connections
-    let auth = match &conf.auth_type {
-        AuthType::Trust | AuthType::MD5 => None,
-        AuthType::ZenithJWT => {
-            // unwrap is ok because check is performed when creating config, so path is set and file exists
-            let key_path = conf.auth_validation_public_key_path.as_ref().unwrap();
-            Some(JwtAuth::from_key_path(key_path)?.into())
-        }
-    };
-    info!("Using auth: {:#?}", conf.auth_type);
-
-    // Spawn a new thread for the http endpoint
-    // bind before launching separate thread so the error reported before startup exits
-    let cloned = auth.clone();
-    let http_endpoint_thread = thread::Builder::new()
-        .name("http_endpoint_thread".into())
-        .spawn(move || {
-            let router = http::make_router(conf, cloned);
-            endpoint::serve_thread_main(router, http_listener)
-        })?;
-
-    join_handles.push(http_endpoint_thread);
+    // Initialize page cache, this will spawn walredo_thread
+    page_cache::init(conf);
 
     // Spawn a thread to listen for connections. It will spawn further threads
     // for each connection.
     let page_service_thread = thread::Builder::new()
         .name("Page Service thread".into())
-        .spawn(move || {
-            page_service::thread_main(conf, auth, pageserver_listener, conf.auth_type)
-        })?;
+        .spawn(move || page_service::thread_main(conf, pageserver_listener))?;
 
-    for info in SignalsInfo::<WithOrigin>::new(TERM_SIGNALS)?.into_iter() {
-        match info.signal {
-            SIGQUIT => {
-                info!("Got SIGQUIT. Terminate pageserver in immediate shutdown mode");
-                exit(111);
-            }
-            SIGTERM => {
-                info!("Got SIGINT/SIGTERM. Terminate gracefully in fast shutdown mode");
-                // Terminate postgres backends
-                postgres_backend::set_pgbackend_shutdown_requested();
-                // Stop all tenants and flush their data
-                tenant_mgr::shutdown_all_tenants()?;
-                // Wait for pageservice thread to complete the job
-                page_service_thread
-                    .join()
-                    .expect("thread panicked")
-                    .expect("thread exited with an error");
-
-                // Shut down http router
-                endpoint::shutdown();
-
-                // Wait for all threads
-                for handle in join_handles.into_iter() {
-                    handle
-                        .join()
-                        .expect("thread panicked")
-                        .expect("thread exited with an error");
-                }
-                info!("Pageserver shut down successfully completed");
-                exit(0);
-            }
-            _ => {
-                debug!("Unknown signal.");
-            }
-        }
+    if let Some(tui_thread) = tui_thread {
+        // The TUI thread exits when the user asks to Quit.
+        tui_thread.join().unwrap();
+    } else {
+        page_service_thread
+            .join()
+            .expect("Page service thread has panicked")?
     }
 
     Ok(())
 }
+
+fn init_logging(
+    conf: &PageServerConf,
+    log_file: File,
+) -> Result<slog_scope::GlobalLoggerGuard, io::Error> {
+    if conf.interactive {
+        Ok(tui::init_logging())
+    } else if conf.daemonize {
+        let decorator = slog_term::PlainSyncDecorator::new(log_file);
+        let drain = slog_term::FullFormat::new(decorator).build();
+        let drain = slog::Filter::new(drain, |record: &slog::Record| {
+            if record.level().is_at_least(slog::Level::Info) {
+                return true;
+            }
+            false
+        });
+        let drain = std::sync::Mutex::new(drain).fuse();
+        let logger = slog::Logger::root(
+            drain,
+            slog::o!(
+                "location" =>
+                FnValue(move |record| {
+                    format!("{}, {}:{}",
+                            record.module(),
+                            record.file(),
+                            record.line()
+                            )
+                    }
+                )
+            ),
+        );
+        Ok(slog_scope::set_global_logger(logger))
+    } else {
+        let decorator = slog_term::TermDecorator::new().build();
+        let drain = slog_term::FullFormat::new(decorator).build().fuse();
+        let drain = slog_async::Async::new(drain).chan_size(1000).build().fuse();
+        let drain = slog::Filter::new(drain, |record: &slog::Record| {
+            if record.level().is_at_least(slog::Level::Info) {
+                return true;
+            }
+            if record.level().is_at_least(slog::Level::Debug)
+                && record.module().starts_with("pageserver")
+            {
+                return true;
+            }
+            false
+        })
+        .fuse();
+        let logger = slog::Logger::root(drain, slog::o!());
+        Ok(slog_scope::set_global_logger(logger))
+    }
+}

pageserver/src/branches.rs

@@ -4,91 +4,33 @@
 // TODO: move all paths construction to conf impl
 //
 
-use anyhow::{bail, ensure, Context, Result};
-use postgres_ffi::ControlFileData;
+use anyhow::{anyhow, bail, Context, Result};
+use fs::File;
+use postgres_ffi::{pg_constants, xlog_utils, ControlFileData};
+use rand::Rng;
 use serde::{Deserialize, Serialize};
+use std::env;
+use std::io::{Read, Write};
 use std::{
-    fs,
-    path::Path,
+    collections::HashMap,
+    fs, io,
+    path::{Path, PathBuf},
     process::{Command, Stdio},
     str::FromStr,
-    sync::Arc,
 };
-use tracing::*;
-
-use zenith_utils::crashsafe_dir;
-use zenith_utils::logging;
 use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
-use crate::tenant_mgr;
-use crate::walredo::WalRedoManager;
-use crate::{repository::Repository, PageServerConf};
-use crate::{restore_local_repo, LOG_FILE_NAME};
+use crate::page_cache;
+use crate::restore_local_repo;
+use crate::{repository::Repository, PageServerConf, ZTimelineId};
 
 #[derive(Serialize, Deserialize, Clone)]
 pub struct BranchInfo {
     pub name: String,
-    #[serde(with = "hex")]
     pub timeline_id: ZTimelineId,
-    pub latest_valid_lsn: Lsn,
+    pub latest_valid_lsn: Option<Lsn>,
     pub ancestor_id: Option<String>,
     pub ancestor_lsn: Option<String>,
-    pub current_logical_size: usize,
-    pub current_logical_size_non_incremental: usize,
-}
-
-impl BranchInfo {
-    pub fn from_path<T: AsRef<Path>>(
-        path: T,
-        conf: &PageServerConf,
-        tenantid: &ZTenantId,
-        repo: &Arc<dyn Repository>,
-    ) -> Result<Self> {
-        let name = path
-            .as_ref()
-            .file_name()
-            .unwrap()
-            .to_str()
-            .unwrap()
-            .to_string();
-        let timeline_id = std::fs::read_to_string(path)?.parse::<ZTimelineId>()?;
-
-        let timeline = repo.get_timeline(timeline_id)?;
-
-        let ancestor_path = conf.ancestor_path(&timeline_id, tenantid);
-        let mut ancestor_id: Option<String> = None;
-        let mut ancestor_lsn: Option<String> = None;
-
-        if ancestor_path.exists() {
-            let ancestor = std::fs::read_to_string(ancestor_path)?;
-            let mut strings = ancestor.split('@');
-
-            ancestor_id = Some(
-                strings
-                    .next()
-                    .with_context(|| "wrong branch ancestor point in time format")?
-                    .to_owned(),
-            );
-            ancestor_lsn = Some(
-                strings
-                    .next()
-                    .with_context(|| "wrong branch ancestor point in time format")?
-                    .to_owned(),
-            );
-        }
-
-        Ok(BranchInfo {
-            name,
-            timeline_id,
-            latest_valid_lsn: timeline.get_last_record_lsn(),
-            ancestor_id,
-            ancestor_lsn,
-            current_logical_size: timeline.get_current_logical_size(),
-            current_logical_size_non_incremental: timeline
-                .get_current_logical_size_non_incremental(timeline.get_last_record_lsn())?,
-        })
-    }
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -97,242 +39,238 @@ pub struct PointInTime {
     pub lsn: Lsn,
 }
 
-pub fn init_pageserver(conf: &'static PageServerConf, create_tenant: Option<&str>) -> Result<()> {
-    // Initialize logger
-    // use true as daemonize parameter because otherwise we pollute zenith cli output with a few pages long output of info messages
-    let _log_file = logging::init(LOG_FILE_NAME, true)?;
-
-    // We don't use the real WAL redo manager, because we don't want to spawn the WAL redo
-    // process during repository initialization.
-    //
-    // FIXME: That caused trouble, because the WAL redo manager spawned a thread that launched
-    // initdb in the background, and it kept running even after the "zenith init" had exited.
-    // In tests, we started the  page server immediately after that, so that initdb was still
-    // running in the background, and we failed to run initdb again in the same directory. This
-    // has been solved for the rapid init+start case now, but the general race condition remains
-    // if you restart the server quickly. The WAL redo manager doesn't use a separate thread
-    // anymore, but I think that could still happen.
-    let dummy_redo_mgr = Arc::new(crate::walredo::DummyRedoManager {});
-
-    if let Some(tenantid) = create_tenant {
-        let tenantid = ZTenantId::from_str(tenantid)?;
-        println!("initializing tenantid {}", tenantid);
-        create_repo(conf, tenantid, dummy_redo_mgr).with_context(|| "failed to create repo")?;
-    }
-    crashsafe_dir::create_dir_all(conf.tenants_path())?;
-
-    println!("pageserver init succeeded");
-    Ok(())
-}
-
-pub fn create_repo(
-    conf: &'static PageServerConf,
-    tenantid: ZTenantId,
-    wal_redo_manager: Arc<dyn WalRedoManager + Send + Sync>,
-) -> Result<Arc<dyn Repository>> {
-    let repo_dir = conf.tenant_path(&tenantid);
-    if repo_dir.exists() {
-        bail!("repo for {} already exists", tenantid)
-    }
-
+pub fn init_repo(conf: &'static PageServerConf, repo_dir: &Path) -> Result<()> {
     // top-level dir may exist if we are creating it through CLI
-    crashsafe_dir::create_dir_all(&repo_dir)
+    fs::create_dir_all(repo_dir)
         .with_context(|| format!("could not create directory {}", repo_dir.display()))?;
 
-    crashsafe_dir::create_dir(conf.timelines_path(&tenantid))?;
-    crashsafe_dir::create_dir_all(conf.branches_path(&tenantid))?;
-    crashsafe_dir::create_dir_all(conf.tags_path(&tenantid))?;
+    env::set_current_dir(repo_dir)?;
 
-    info!("created directory structure in {}", repo_dir.display());
+    fs::create_dir(std::path::Path::new("timelines"))?;
+    fs::create_dir(std::path::Path::new("refs"))?;
+    fs::create_dir(std::path::Path::new("refs").join("branches"))?;
+    fs::create_dir(std::path::Path::new("refs").join("tags"))?;
 
-    let tli = create_timeline(conf, None, &tenantid)?;
+    println!("created directory structure in {}", repo_dir.display());
 
-    let repo = Arc::new(crate::layered_repository::LayeredRepository::new(
-        conf,
-        wal_redo_manager,
-        tenantid,
-        false,
-    ));
+    // Run initdb
+    //
+    // We create the cluster temporarily in a "tmp" directory inside the repository,
+    // and move it to the right location from there.
+    let tmppath = std::path::Path::new("tmp");
 
-    // Load data into pageserver
-    // TODO To implement zenith import we need to
-    //      move data loading out of create_repo()
-    bootstrap_timeline(conf, tenantid, tli, repo.as_ref())?;
-
-    Ok(repo)
-}
-
-// Returns checkpoint LSN from controlfile
-fn get_lsn_from_controlfile(path: &Path) -> Result<Lsn> {
-    // Read control file to extract the LSN
-    let controlfile_path = path.join("global").join("pg_control");
-    let controlfile = ControlFileData::decode(&fs::read(controlfile_path)?)?;
-    let lsn = controlfile.checkPoint;
-
-    Ok(Lsn(lsn))
-}
-
-// Create the cluster temporarily in 'initdbpath' directory inside the repository
-// to get bootstrap data for timeline initialization.
-//
-fn run_initdb(conf: &'static PageServerConf, initdbpath: &Path) -> Result<()> {
-    info!("running initdb in {}... ", initdbpath.display());
+    print!("running initdb... ");
+    io::stdout().flush()?;
 
     let initdb_path = conf.pg_bin_dir().join("initdb");
-    let initdb_output = Command::new(initdb_path)
-        .args(&["-D", initdbpath.to_str().unwrap()])
-        .args(&["-U", &conf.superuser])
+    let initdb_otput = Command::new(initdb_path)
+        .args(&["-D", tmppath.to_str().unwrap()])
         .arg("--no-instructions")
-        // This is only used for a temporary installation that is deleted shortly after,
-        // so no need to fsync it
-        .arg("--no-sync")
         .env_clear()
         .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
         .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
         .stdout(Stdio::null())
         .output()
         .with_context(|| "failed to execute initdb")?;
-    if !initdb_output.status.success() {
+    if !initdb_otput.status.success() {
         anyhow::bail!(
             "initdb failed: '{}'",
-            String::from_utf8_lossy(&initdb_output.stderr)
+            String::from_utf8_lossy(&initdb_otput.stderr)
         );
     }
+    println!("initdb succeeded");
 
-    Ok(())
-}
+    // Read control file to extract the LSN and system id
+    let controlfile_path = tmppath.join("global").join("pg_control");
+    let controlfile = ControlFileData::decode(&fs::read(controlfile_path)?)?;
+    // let systemid = controlfile.system_identifier;
+    let lsn = controlfile.checkPoint;
+    let lsnstr = format!("{:016X}", lsn);
 
-//
-// - run initdb to init temporary instance and get bootstrap data
-// - after initialization complete, remove the temp dir.
-//
-fn bootstrap_timeline(
-    conf: &'static PageServerConf,
-    tenantid: ZTenantId,
-    tli: ZTimelineId,
-    repo: &dyn Repository,
-) -> Result<()> {
-    let _enter = info_span!("bootstrapping", timeline = %tli, tenant = %tenantid).entered();
+    // Bootstrap the repository by loading the newly-initdb'd cluster into 'main' branch.
+    let tli = create_timeline(conf, None)?;
+    let timelinedir = conf.timeline_path(tli);
 
-    let initdb_path = conf.tenant_path(&tenantid).join("tmp");
+    // We don't use page_cache here, because we don't want to spawn the WAL redo thread during
+    // repository initialization.
+    //
+    // FIXME: That caused trouble, because the WAL redo thread launched initdb in the background,
+    // and it kept running even after the "zenith init" had exited. In tests, we started the
+    // page server immediately after that, so that initdb was still running in the background,
+    // and we failed to run initdb again in the same directory. This has been solved for the
+    // rapid init+start case now, but the general race condition remains if you restart the
+    // server quickly.
+    let storage = crate::rocksdb_storage::RocksObjectStore::create(conf)?;
+    //let storage = crate::inmem_storage::InmemObjectStore::create(conf)?;
 
-    // Init temporarily repo to get bootstrap data
-    run_initdb(conf, &initdb_path)?;
-    let pgdata_path = initdb_path;
-
-    let lsn = get_lsn_from_controlfile(&pgdata_path)?.align();
-
-    // Import the contents of the data directory at the initial checkpoint
-    // LSN, and any WAL after that.
-    let timeline = repo.create_empty_timeline(tli)?;
-    restore_local_repo::import_timeline_from_postgres_datadir(
-        &pgdata_path,
-        timeline.writer().as_ref(),
-        lsn,
-    )?;
-    timeline.checkpoint()?;
-
-    println!(
-        "created initial timeline {} timeline.lsn {}",
-        tli,
-        timeline.get_last_record_lsn()
+    let repo = crate::object_repository::ObjectRepository::new(
+        conf,
+        std::sync::Arc::new(storage),
+        std::sync::Arc::new(crate::walredo::DummyRedoManager {}),
     );
+    let timeline = repo.create_empty_timeline(tli, Lsn(lsn))?;
+
+    restore_local_repo::import_timeline_from_postgres_datadir(&tmppath, &*timeline, Lsn(lsn))?;
+
+    // Move the initial WAL file
+    fs::rename(
+        tmppath.join("pg_wal").join("000000010000000000000001"),
+        timelinedir
+            .join("wal")
+            .join("000000010000000000000001.partial"),
+    )?;
+    println!("created initial timeline {}", tli);
 
     let data = tli.to_string();
-    fs::write(conf.branch_path("main", &tenantid), data)?;
+    fs::write(conf.branch_path("main"), data)?;
     println!("created main branch");
 
-    // Remove temp dir. We don't need it anymore
-    fs::remove_dir_all(pgdata_path)?;
+    // Remove pg_wal
+    fs::remove_dir_all(tmppath.join("pg_wal"))?;
+
+    // Move the data directory as an initial base backup.
+    // FIXME: It would be enough to only copy the non-relation files here, the relation
+    // data was already loaded into the repository.
+    let target = timelinedir.join("snapshots").join(&lsnstr);
+    fs::rename(tmppath, &target)?;
+
+    println!(
+        "new zenith repository was created in {}",
+        repo_dir.display()
+    );
 
     Ok(())
 }
 
-pub(crate) fn get_tenants(conf: &PageServerConf) -> Result<Vec<String>> {
-    let tenants_dir = conf.tenants_path();
-
-    std::fs::read_dir(&tenants_dir)?
-        .map(|dir_entry_res| {
-            let dir_entry = dir_entry_res?;
-            ensure!(dir_entry.file_type()?.is_dir());
-            Ok(dir_entry.file_name().to_str().unwrap().to_owned())
-        })
-        .collect()
-}
-
-pub(crate) fn get_branches(conf: &PageServerConf, tenantid: &ZTenantId) -> Result<Vec<BranchInfo>> {
-    let repo = tenant_mgr::get_repository_for_tenant(*tenantid)?;
+pub(crate) fn get_branches(conf: &PageServerConf) -> Result<Vec<BranchInfo>> {
+    let repo = page_cache::get_repository();
 
     // Each branch has a corresponding record (text file) in the refs/branches
     // with timeline_id.
-    let branches_dir = conf.branches_path(tenantid);
+    let branches_dir = std::path::Path::new("refs").join("branches");
 
     std::fs::read_dir(&branches_dir)?
         .map(|dir_entry_res| {
             let dir_entry = dir_entry_res?;
-            BranchInfo::from_path(dir_entry.path(), conf, tenantid, &repo)
+            let name = dir_entry.file_name().to_str().unwrap().to_string();
+            let timeline_id = std::fs::read_to_string(dir_entry.path())?.parse::<ZTimelineId>()?;
+
+            let latest_valid_lsn = repo
+                .get_timeline(timeline_id)
+                .map(|timeline| timeline.get_last_valid_lsn())
+                .ok();
+
+            let ancestor_path = conf.ancestor_path(timeline_id);
+            let mut ancestor_id: Option<String> = None;
+            let mut ancestor_lsn: Option<String> = None;
+
+            if ancestor_path.exists() {
+                let ancestor = std::fs::read_to_string(ancestor_path)?;
+                let mut strings = ancestor.split('@');
+
+                ancestor_id = Some(
+                    strings
+                        .next()
+                        .with_context(|| "wrong branch ancestor point in time format")?
+                        .to_owned(),
+                );
+                ancestor_lsn = Some(
+                    strings
+                        .next()
+                        .with_context(|| "wrong branch ancestor point in time format")?
+                        .to_owned(),
+                );
+            }
+
+            Ok(BranchInfo {
+                name,
+                timeline_id,
+                latest_valid_lsn,
+                ancestor_id,
+                ancestor_lsn,
+            })
         })
         .collect()
 }
 
+pub(crate) fn get_system_id(conf: &PageServerConf) -> Result<u64> {
+    // let branches = get_branches();
+
+    let branches_dir = std::path::Path::new("refs").join("branches");
+    let branches = std::fs::read_dir(&branches_dir)?
+        .map(|dir_entry_res| {
+            let dir_entry = dir_entry_res?;
+            let name = dir_entry.file_name().to_str().unwrap().to_string();
+            let timeline_id = std::fs::read_to_string(dir_entry.path())?.parse::<ZTimelineId>()?;
+            Ok((name, timeline_id))
+        })
+        .collect::<Result<HashMap<String, ZTimelineId>>>()?;
+
+    let main_tli = branches
+        .get("main")
+        .ok_or_else(|| anyhow!("Branch main not found"))?;
+
+    let (_, main_snap_dir) = find_latest_snapshot(conf, *main_tli)?;
+    let controlfile_path = main_snap_dir.join("global").join("pg_control");
+    let controlfile = ControlFileData::decode(&fs::read(controlfile_path)?)?;
+    Ok(controlfile.system_identifier)
+}
+
 pub(crate) fn create_branch(
     conf: &PageServerConf,
     branchname: &str,
     startpoint_str: &str,
-    tenantid: &ZTenantId,
 ) -> Result<BranchInfo> {
-    let repo = tenant_mgr::get_repository_for_tenant(*tenantid)?;
+    let repo = page_cache::get_repository();
 
-    if conf.branch_path(branchname, tenantid).exists() {
+    if conf.branch_path(&branchname).exists() {
         anyhow::bail!("branch {} already exists", branchname);
     }
 
-    let mut startpoint = parse_point_in_time(conf, startpoint_str, tenantid)?;
-    let timeline = repo.get_timeline(startpoint.timelineid)?;
+    let mut startpoint = parse_point_in_time(conf, startpoint_str)?;
+
     if startpoint.lsn == Lsn(0) {
         // Find end of WAL on the old timeline
-        let end_of_wal = timeline.get_last_record_lsn();
-        info!("branching at end of WAL: {}", end_of_wal);
+        let end_of_wal = repo
+            .get_timeline(startpoint.timelineid)?
+            .get_last_record_lsn();
+        println!("branching at end of WAL: {}", end_of_wal);
         startpoint.lsn = end_of_wal;
-    } else {
-        // Wait for the WAL to arrive and be processed on the parent branch up
-        // to the requested branch point. The repository code itself doesn't
-        // require it, but if we start to receive WAL on the new timeline,
-        // decoding the new WAL might need to look up previous pages, relation
-        // sizes etc. and that would get confused if the previous page versions
-        // are not in the repository yet.
-        timeline.wait_lsn(startpoint.lsn)?;
-    }
-    startpoint.lsn = startpoint.lsn.align();
-    if timeline.get_start_lsn() > startpoint.lsn {
-        anyhow::bail!(
-            "invalid startpoint {} for the branch {}: less than timeline start {}",
-            startpoint.lsn,
-            branchname,
-            timeline.get_start_lsn()
-        );
     }
 
     // create a new timeline directory for it
-    let newtli = create_timeline(conf, Some(startpoint), tenantid)?;
+    let newtli = create_timeline(conf, Some(startpoint))?;
+    let newtimelinedir = conf.timeline_path(newtli);
 
     // Let the Repository backend do its initialization
     repo.branch_timeline(startpoint.timelineid, newtli, startpoint.lsn)?;
 
+    // Copy the latest snapshot (TODO: before the startpoint) and all WAL
+    // TODO: be smarter and avoid the copying...
+    let (_maxsnapshot, oldsnapshotdir) = find_latest_snapshot(conf, startpoint.timelineid)?;
+    let copy_opts = fs_extra::dir::CopyOptions::new();
+    fs_extra::dir::copy(oldsnapshotdir, newtimelinedir.join("snapshots"), &copy_opts)?;
+
+    let oldtimelinedir = conf.timeline_path(startpoint.timelineid);
+    copy_wal(
+        &oldtimelinedir.join("wal"),
+        &newtimelinedir.join("wal"),
+        startpoint.lsn,
+        pg_constants::WAL_SEGMENT_SIZE,
+    )?;
+
     // Remember the human-readable branch name for the new timeline.
     // FIXME: there's a race condition, if you create a branch with the same
     // name concurrently.
     let data = newtli.to_string();
-    fs::write(conf.branch_path(branchname, tenantid), data)?;
+    fs::write(conf.branch_path(&branchname), data)?;
 
     Ok(BranchInfo {
         name: branchname.to_string(),
         timeline_id: newtli,
-        latest_valid_lsn: startpoint.lsn,
+        latest_valid_lsn: Some(startpoint.lsn),
         ancestor_id: None,
         ancestor_lsn: None,
-        current_logical_size: 0,
-        current_logical_size_non_incremental: 0,
     })
 }
 
@@ -355,11 +293,7 @@ pub(crate) fn create_branch(
 //    mytag
 //
 //
-fn parse_point_in_time(
-    conf: &PageServerConf,
-    s: &str,
-    tenantid: &ZTenantId,
-) -> Result<PointInTime> {
+fn parse_point_in_time(conf: &PageServerConf, s: &str) -> Result<PointInTime> {
     let mut strings = s.split('@');
     let name = strings.next().unwrap();
 
@@ -374,21 +308,21 @@ fn parse_point_in_time(
 
     // Check if it's a tag
     if lsn.is_none() {
-        let tagpath = conf.tag_path(name, tenantid);
+        let tagpath = conf.tag_path(name);
         if tagpath.exists() {
             let pointstr = fs::read_to_string(tagpath)?;
 
-            return parse_point_in_time(conf, &pointstr, tenantid);
+            return parse_point_in_time(conf, &pointstr);
         }
     }
 
     // Check if it's a branch
     // Check if it's branch @ LSN
-    let branchpath = conf.branch_path(name, tenantid);
+    let branchpath = conf.branch_path(name);
     if branchpath.exists() {
         let pointstr = fs::read_to_string(branchpath)?;
 
-        let mut result = parse_point_in_time(conf, &pointstr, tenantid)?;
+        let mut result = parse_point_in_time(conf, &pointstr)?;
 
         result.lsn = lsn.unwrap_or(Lsn(0));
         return Ok(result);
@@ -397,7 +331,7 @@ fn parse_point_in_time(
     // Check if it's a timelineid
     // Check if it's timelineid @ LSN
     if let Ok(timelineid) = ZTimelineId::from_str(name) {
-        let tlipath = conf.timeline_path(&timelineid, tenantid);
+        let tlipath = conf.timeline_path(timelineid);
         if tlipath.exists() {
             return Ok(PointInTime {
                 timelineid,
@@ -409,18 +343,17 @@ fn parse_point_in_time(
     bail!("could not parse point-in-time {}", s);
 }
 
-fn create_timeline(
-    conf: &PageServerConf,
-    ancestor: Option<PointInTime>,
-    tenantid: &ZTenantId,
-) -> Result<ZTimelineId> {
+fn create_timeline(conf: &PageServerConf, ancestor: Option<PointInTime>) -> Result<ZTimelineId> {
     // Create initial timeline
+    let mut tli_buf = [0u8; 16];
+    rand::thread_rng().fill(&mut tli_buf);
+    let timelineid = ZTimelineId::from(tli_buf);
 
-    let timelineid = ZTimelineId::generate();
-
-    let timelinedir = conf.timeline_path(&timelineid, tenantid);
+    let timelinedir = conf.timeline_path(timelineid);
 
     fs::create_dir(&timelinedir)?;
+    fs::create_dir(&timelinedir.join("snapshots"))?;
+    fs::create_dir(&timelinedir.join("wal"))?;
 
     if let Some(ancestor) = ancestor {
         let data = format!("{}@{}", ancestor.timelineid, ancestor.lsn);
@@ -429,3 +362,72 @@ fn create_timeline(
 
     Ok(timelineid)
 }
+
+///
+/// Copy all WAL segments from one directory to another, up to given LSN.
+///
+/// If the given LSN is in the middle of a segment, the last segment containing it
+/// is written out as .partial, and padded with zeros.
+///
+fn copy_wal(src_dir: &Path, dst_dir: &Path, upto: Lsn, wal_seg_size: usize) -> Result<()> {
+    let last_segno = upto.segment_number(wal_seg_size);
+    let last_segoff = upto.segment_offset(wal_seg_size);
+
+    for entry in fs::read_dir(src_dir).unwrap().flatten() {
+        let entry_name = entry.file_name();
+        let fname = entry_name.to_str().unwrap();
+
+        // Check if the filename looks like an xlog file, or a .partial file.
+        if !xlog_utils::IsXLogFileName(fname) && !xlog_utils::IsPartialXLogFileName(fname) {
+            continue;
+        }
+        let (segno, _tli) = xlog_utils::XLogFromFileName(fname, wal_seg_size as usize);
+
+        let copylen;
+        let mut dst_fname = PathBuf::from(fname);
+        if segno > last_segno {
+            // future segment, skip
+            continue;
+        } else if segno < last_segno {
+            copylen = wal_seg_size;
+            dst_fname.set_extension("");
+        } else {
+            copylen = last_segoff;
+            dst_fname.set_extension("partial");
+        }
+
+        let src_file = File::open(entry.path())?;
+        let mut dst_file = File::create(dst_dir.join(&dst_fname))?;
+        std::io::copy(&mut src_file.take(copylen as u64), &mut dst_file)?;
+
+        if copylen < wal_seg_size {
+            std::io::copy(
+                &mut std::io::repeat(0).take((wal_seg_size - copylen) as u64),
+                &mut dst_file,
+            )?;
+        }
+    }
+    Ok(())
+}
+
+// Find the latest snapshot for a timeline
+fn find_latest_snapshot(conf: &PageServerConf, timeline: ZTimelineId) -> Result<(Lsn, PathBuf)> {
+    let snapshotsdir = conf.snapshots_path(timeline);
+    let paths = fs::read_dir(&snapshotsdir)?;
+    let mut maxsnapshot = Lsn(0);
+    let mut snapshotdir: Option<PathBuf> = None;
+    for path in paths {
+        let path = path?;
+        let filename = path.file_name().to_str().unwrap().to_owned();
+        if let Ok(lsn) = Lsn::from_hex(&filename) {
+            maxsnapshot = std::cmp::max(lsn, maxsnapshot);
+            snapshotdir = Some(path.path());
+        }
+    }
+    if maxsnapshot == Lsn(0) {
+        // TODO: check ancestor timeline
+        anyhow::bail!("no snapshot found in {}", snapshotsdir.display());
+    }
+
+    Ok((maxsnapshot, snapshotdir.unwrap()))
+}

pageserver/src/http/mod.rs

@@ -1,3 +0,0 @@
-pub mod models;
-pub mod routes;
-pub use routes::make_router;

pageserver/src/http/models.rs

@@ -1,17 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-use crate::ZTenantId;
-
-#[derive(Serialize, Deserialize)]
-pub struct BranchCreateRequest {
-    #[serde(with = "hex")]
-    pub tenant_id: ZTenantId,
-    pub name: String,
-    pub start_point: String,
-}
-
-#[derive(Serialize, Deserialize)]
-pub struct TenantCreateRequest {
-    #[serde(with = "hex")]
-    pub tenant_id: ZTenantId,
-}

pageserver/src/http/openapi_spec.yml

@@ -1,291 +0,0 @@
-openapi: "3.0.2"
-info:
-  title: Page Server API
-  version: "1.0"
-servers:
-  - url: ""
-paths:
-  /v1/status:
-    description: Healthcheck endpoint
-    get:
-      description: Healthcheck
-      security: []
-      responses:
-        "200":
-          description: OK
-          content:
-            application/json:
-              schema:
-                type: object
-  /v1/branch/{tenant_id}:
-    parameters:
-      - name: tenant_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-    get:
-      description: Get branches for tenant
-      responses:
-        "200":
-          description: BranchInfo
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  $ref: "#/components/schemas/BranchInfo"
-        "400":
-          description: Error when no tenant id found in path
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-  /v1/branch/{tenant_id}/{branch_name}:
-    parameters:
-      - name: tenant_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-      - name: branch_name
-        in: path
-        required: true
-        schema:
-          type: string
-    get:
-      description: Get branches for tenant
-      responses:
-        "200":
-          description: BranchInfo
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BranchInfo"
-        "400":
-          description: Error when no tenant id found in path or no branch name
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-  /v1/branch/:
-    post:
-      description: Create branch
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              required:
-                - "tenant_id"
-                - "name"
-                - "start_point"
-              properties:
-                tenant_id:
-                  type: string
-                  format: hex
-                name:
-                  type: string
-                start_point:
-                  type: string
-      responses:
-        "201":
-          description: BranchInfo
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  $ref: "#/components/schemas/BranchInfo"
-        "400":
-          description: Malformed branch create request
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-  /v1/tenant/:
-    get:
-      description: Get tenants list
-      responses:
-        "200":
-          description: OK
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  type: string
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-    post:
-      description: Create tenant
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              required:
-                - "tenant_id"
-              properties:
-                tenant_id:
-                  type: string
-                  format: hex
-      responses:
-        "201":
-          description: CREATED
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  type: string
-        "400":
-          description: Malformed tenant create request
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-
-components:
-  securitySchemes:
-    JWT:
-      type: http
-      scheme: bearer
-      bearerFormat: JWT
-  schemas:
-    BranchInfo:
-      type: object
-      required:
-        - name
-        - timeline_id
-        - latest_valid_lsn
-        - current_logical_size
-        - current_logical_size_non_incremental
-      properties:
-        name:
-          type: string
-        timeline_id:
-          type: string
-          format: hex
-        ancestor_id:
-          type: string
-        ancestor_lsn:
-          type: string
-        current_logical_size:
-          type: integer
-        current_logical_size_non_incremental:
-          type: integer
-    Error:
-      type: object
-      required:
-        - msg
-      properties:
-        msg:
-          type: string
-    UnauthorizedError:
-      type: object
-      required:
-        - msg
-      properties:
-        msg:
-          type: string
-    ForbiddenError:
-      type: object
-      required:
-        - msg
-      properties:
-        msg:
-          type: string
-
-security:
-  - JWT: []

pageserver/src/http/routes.rs

@@ -1,183 +0,0 @@
-use std::sync::Arc;
-
-use anyhow::Result;
-use hyper::header;
-use hyper::StatusCode;
-use hyper::{Body, Request, Response, Uri};
-use routerify::{ext::RequestExt, RouterBuilder};
-use tracing::*;
-use zenith_utils::auth::JwtAuth;
-use zenith_utils::http::endpoint::attach_openapi_ui;
-use zenith_utils::http::endpoint::auth_middleware;
-use zenith_utils::http::endpoint::check_permission;
-use zenith_utils::http::error::ApiError;
-use zenith_utils::http::{
-    endpoint,
-    error::HttpErrorBody,
-    json::{json_request, json_response},
-    request::get_request_param,
-    request::parse_request_param,
-};
-
-use super::models::BranchCreateRequest;
-use super::models::TenantCreateRequest;
-use crate::branches::BranchInfo;
-use crate::{branches, tenant_mgr, PageServerConf, ZTenantId};
-
-#[derive(Debug)]
-struct State {
-    conf: &'static PageServerConf,
-    auth: Option<Arc<JwtAuth>>,
-    allowlist_routes: Vec<Uri>,
-}
-
-impl State {
-    fn new(conf: &'static PageServerConf, auth: Option<Arc<JwtAuth>>) -> Self {
-        let allowlist_routes = ["/v1/status", "/v1/doc", "/swagger.yml"]
-            .iter()
-            .map(|v| v.parse().unwrap())
-            .collect::<Vec<_>>();
-        Self {
-            conf,
-            auth,
-            allowlist_routes,
-        }
-    }
-}
-
-#[inline(always)]
-fn get_state(request: &Request<Body>) -> &State {
-    request
-        .data::<Arc<State>>()
-        .expect("unknown state type")
-        .as_ref()
-}
-
-#[inline(always)]
-fn get_config(request: &Request<Body>) -> &'static PageServerConf {
-    get_state(request).conf
-}
-
-// healthcheck handler
-async fn status_handler(_: Request<Body>) -> Result<Response<Body>, ApiError> {
-    Ok(Response::builder()
-        .status(StatusCode::OK)
-        .header(header::CONTENT_TYPE, "application/json")
-        .body(Body::from("{}"))
-        .map_err(ApiError::from_err)?)
-}
-
-async fn branch_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let request_data: BranchCreateRequest = json_request(&mut request).await?;
-
-    check_permission(&request, Some(request_data.tenant_id))?;
-
-    let response_data = tokio::task::spawn_blocking(move || {
-        let _enter = info_span!("/branch_create", name = %request_data.name, tenant = %request_data.tenant_id, startpoint=%request_data.start_point).entered();
-        branches::create_branch(
-            get_config(&request),
-            &request_data.name,
-            &request_data.start_point,
-            &request_data.tenant_id,
-        )
-    })
-    .await
-    .map_err(ApiError::from_err)??;
-    Ok(json_response(StatusCode::CREATED, response_data)?)
-}
-
-async fn branch_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let tenantid: ZTenantId = parse_request_param(&request, "tenant_id")?;
-
-    check_permission(&request, Some(tenantid))?;
-
-    let response_data = tokio::task::spawn_blocking(move || {
-        let _enter = info_span!("branch_list", tenant = %tenantid).entered();
-        crate::branches::get_branches(get_config(&request), &tenantid)
-    })
-    .await
-    .map_err(ApiError::from_err)??;
-    Ok(json_response(StatusCode::OK, response_data)?)
-}
-
-// TODO add to swagger
-async fn branch_detail_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let tenantid: ZTenantId = parse_request_param(&request, "tenant_id")?;
-    let branch_name: String = get_request_param(&request, "branch_name")?.to_string();
-    let conf = get_state(&request).conf;
-    let path = conf.branch_path(&branch_name, &tenantid);
-
-    let response_data = tokio::task::spawn_blocking(move || {
-        let _enter = info_span!("branch_detail", tenant = %tenantid, branch=%branch_name).entered();
-        let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
-        BranchInfo::from_path(path, conf, &tenantid, &repo)
-    })
-    .await
-    .map_err(ApiError::from_err)??;
-
-    Ok(json_response(StatusCode::OK, response_data)?)
-}
-
-async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    // check for management permission
-    check_permission(&request, None)?;
-
-    let response_data = tokio::task::spawn_blocking(move || {
-        let _enter = info_span!("tenant_list").entered();
-        crate::branches::get_tenants(get_config(&request))
-    })
-    .await
-    .map_err(ApiError::from_err)??;
-
-    Ok(json_response(StatusCode::OK, response_data)?)
-}
-
-async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    // check for management permission
-    check_permission(&request, None)?;
-
-    let request_data: TenantCreateRequest = json_request(&mut request).await?;
-
-    let response_data = tokio::task::spawn_blocking(move || {
-        let _enter = info_span!("tenant_create", tenant = %request_data.tenant_id).entered();
-        tenant_mgr::create_repository_for_tenant(get_config(&request), request_data.tenant_id)
-    })
-    .await
-    .map_err(ApiError::from_err)??;
-    Ok(json_response(StatusCode::CREATED, response_data)?)
-}
-
-async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
-    json_response(
-        StatusCode::NOT_FOUND,
-        HttpErrorBody::from_msg("page not found".to_owned()),
-    )
-}
-
-pub fn make_router(
-    conf: &'static PageServerConf,
-    auth: Option<Arc<JwtAuth>>,
-) -> RouterBuilder<hyper::Body, ApiError> {
-    let spec = include_bytes!("openapi_spec.yml");
-    let mut router = attach_openapi_ui(endpoint::make_router(), spec, "/swagger.yml", "/v1/doc");
-    if auth.is_some() {
-        router = router.middleware(auth_middleware(|request| {
-            let state = get_state(request);
-            if state.allowlist_routes.contains(request.uri()) {
-                None
-            } else {
-                state.auth.as_deref()
-            }
-        }))
-    }
-
-    router
-        .data(Arc::new(State::new(conf, auth)))
-        .get("/v1/status", status_handler)
-        .get("/v1/branch/:tenant_id", branch_list_handler)
-        .get("/v1/branch/:tenant_id/:branch_name", branch_detail_handler)
-        .post("/v1/branch", branch_create_handler)
-        .get("/v1/tenant", tenant_list_handler)
-        .post("/v1/tenant", tenant_create_handler)
-        .any(handler_404)
-}

pageserver/src/inmem_storage.rs

@@ -0,0 +1,345 @@
+//!
+//! An implementation of the ObjectStore interface, backed by BTreeMap
+//!
+use crate::object_key::*;
+use crate::object_store::ObjectStore;
+use crate::repository::RelTag;
+use crate::PageServerConf;
+use crate::ZTimelineId;
+use anyhow::{bail, Result};
+use serde::{Deserialize, Serialize};
+use std::collections::{BTreeMap, HashSet};
+use std::fs::File;
+use std::io::prelude::*;
+use std::ops::Bound::*;
+use std::sync::RwLock;
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
+pub struct StorageKey {
+    obj_key: ObjectKey,
+    lsn: Lsn,
+}
+
+impl StorageKey {
+    /// The first key for a given timeline
+    fn timeline_start(timeline: ZTimelineId) -> Self {
+        Self {
+            obj_key: ObjectKey {
+                timeline,
+                tag: ObjectTag::FirstTag,
+            },
+            lsn: Lsn(0),
+        }
+    }
+}
+
+pub struct InmemObjectStore {
+    conf: &'static PageServerConf,
+    db: RwLock<BTreeMap<StorageKey, Vec<u8>>>,
+}
+
+impl ObjectStore for InmemObjectStore {
+    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>> {
+        let db = self.db.read().unwrap();
+        let val = db.get(&StorageKey {
+            obj_key: key.clone(),
+            lsn,
+        });
+        if let Some(val) = val {
+            Ok(val.clone())
+        } else {
+            bail!("could not find page {:?}", key);
+        }
+    }
+
+    fn get_next_key(&self, key: &ObjectKey) -> Result<Option<ObjectKey>> {
+        let search_key = StorageKey {
+            obj_key: key.clone(),
+            lsn: Lsn(0),
+        };
+        let db = self.db.read().unwrap();
+        for pair in db.range(&search_key..) {
+            let key = pair.0;
+            return Ok(Some(key.obj_key.clone()));
+        }
+        Ok(None)
+    }
+
+    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()> {
+        let mut db = self.db.write().unwrap();
+        db.insert(
+            StorageKey {
+                obj_key: key.clone(),
+                lsn,
+            },
+            value.to_vec(),
+        );
+        Ok(())
+    }
+
+    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()> {
+        let mut db = self.db.write().unwrap();
+        db.remove(&StorageKey {
+            obj_key: key.clone(),
+            lsn,
+        });
+        Ok(())
+    }
+
+    /// Iterate through page versions of given page, starting from the given LSN.
+    /// The versions are walked in descending LSN order.
+    fn object_versions<'a>(
+        &'a self,
+        key: &ObjectKey,
+        lsn: Lsn,
+    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>> {
+        let from = StorageKey {
+            obj_key: key.clone(),
+            lsn: Lsn(0),
+        };
+        let till = StorageKey {
+            obj_key: key.clone(),
+            lsn,
+        };
+        let db = self.db.read().unwrap();
+        let versions: Vec<(Lsn, Vec<u8>)> = db
+            .range(from..=till)
+            .map(|pair| (pair.0.lsn, pair.1.clone()))
+            .collect();
+        Ok(Box::new(InmemObjectVersionIter::new(versions)))
+    }
+
+    /// Iterate through all timeline objects
+    fn list_objects<'a>(
+        &'a self,
+        timeline: ZTimelineId,
+        nonrel_only: bool,
+        lsn: Lsn,
+    ) -> Result<Box<dyn Iterator<Item = ObjectTag> + 'a>> {
+        let curr_key = StorageKey::timeline_start(timeline);
+
+        Ok(Box::new(InmemObjectIter {
+            store: &self,
+            curr_key,
+            timeline,
+            nonrel_only,
+            lsn,
+        }))
+    }
+
+    /// Get a list of all distinct relations in given tablespace and database.
+    ///
+    /// TODO: This implementation is very inefficient, it scans
+    /// through all entries in the given database. In practice, this
+    /// is used for CREATE DATABASE, and usually the template database is small.
+    /// But if it's not, this will be slow.
+    fn list_rels(
+        &self,
+        timelineid: ZTimelineId,
+        spcnode: u32,
+        dbnode: u32,
+        lsn: Lsn,
+    ) -> Result<HashSet<RelTag>> {
+        // FIXME: This scans everything. Very slow
+
+        let mut rels: HashSet<RelTag> = HashSet::new();
+
+        let mut search_rel_tag = RelTag {
+            spcnode,
+            dbnode,
+            relnode: 0,
+            forknum: 0u8,
+        };
+        let db = self.db.read().unwrap();
+        'outer: loop {
+            let search_key = StorageKey {
+                obj_key: ObjectKey {
+                    timeline: timelineid,
+                    tag: ObjectTag::RelationMetadata(search_rel_tag),
+                },
+                lsn: Lsn(0),
+            };
+            for pair in db.range(&search_key..) {
+                let key = pair.0;
+
+                if let ObjectTag::RelationMetadata(rel_tag) = key.obj_key.tag {
+                    if spcnode != 0 && rel_tag.spcnode != spcnode
+                        || dbnode != 0 && rel_tag.dbnode != dbnode
+                    {
+                        break 'outer;
+                    }
+                    if key.lsn <= lsn {
+                        // visible in this snapshot
+                        rels.insert(rel_tag);
+                    }
+                    search_rel_tag = rel_tag;
+                    // skip to next relation
+                    // FIXME: What if relnode is u32::MAX ?
+                    search_rel_tag.relnode += 1;
+                    continue 'outer;
+                } else {
+                    // no more relation metadata entries
+                    break 'outer;
+                }
+            }
+        }
+
+        Ok(rels)
+    }
+
+    /// Iterate through versions of all objects in a timeline.
+    ///
+    /// Returns objects in increasing key-version order.
+    /// Returns all versions up to and including the specified LSN.
+    fn objects<'a>(
+        &'a self,
+        timeline: ZTimelineId,
+        lsn: Lsn,
+    ) -> Result<Box<dyn Iterator<Item = Result<(ObjectTag, Lsn, Vec<u8>)>> + 'a>> {
+        let curr_key = StorageKey::timeline_start(timeline);
+
+        Ok(Box::new(InmemObjects {
+            store: &self,
+            curr_key,
+            timeline,
+            lsn,
+        }))
+    }
+
+    fn compact(&self) {}
+}
+
+impl Drop for InmemObjectStore {
+    fn drop(&mut self) {
+        let path = self.conf.workdir.join("objstore.dmp");
+        let mut f = File::create(path).unwrap();
+        f.write(&self.db.ser().unwrap()).unwrap();
+    }
+}
+
+impl InmemObjectStore {
+    pub fn open(conf: &'static PageServerConf) -> Result<InmemObjectStore> {
+        let path = conf.workdir.join("objstore.dmp");
+        let mut f = File::open(path)?;
+        let mut buffer = Vec::new();
+        // read the whole file
+        f.read_to_end(&mut buffer)?;
+        let db = RwLock::new(BTreeMap::des(&buffer)?);
+        Ok(InmemObjectStore { conf: conf, db })
+    }
+
+    pub fn create(conf: &'static PageServerConf) -> Result<InmemObjectStore> {
+        Ok(InmemObjectStore {
+            conf: conf,
+            db: RwLock::new(BTreeMap::new()),
+        })
+    }
+}
+
+///
+/// Iterator for `object_versions`. Returns all page versions of a given block, in
+/// reverse LSN order.
+///
+struct InmemObjectVersionIter {
+    versions: Vec<(Lsn, Vec<u8>)>,
+    curr: usize,
+}
+impl InmemObjectVersionIter {
+    fn new(versions: Vec<(Lsn, Vec<u8>)>) -> InmemObjectVersionIter {
+        let curr = versions.len();
+        InmemObjectVersionIter { versions, curr }
+    }
+}
+impl Iterator for InmemObjectVersionIter {
+    type Item = (Lsn, Vec<u8>);
+
+    fn next(&mut self) -> std::option::Option<Self::Item> {
+        if self.curr == 0 {
+            None
+        } else {
+            self.curr -= 1;
+            Some(self.versions[self.curr].clone())
+        }
+    }
+}
+
+struct InmemObjects<'r> {
+    store: &'r InmemObjectStore,
+    curr_key: StorageKey,
+    timeline: ZTimelineId,
+    lsn: Lsn,
+}
+
+impl<'r> Iterator for InmemObjects<'r> {
+    // TODO consider returning Box<[u8]>
+    type Item = Result<(ObjectTag, Lsn, Vec<u8>)>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.next_result().transpose()
+    }
+}
+
+impl<'r> InmemObjects<'r> {
+    fn next_result(&mut self) -> Result<Option<(ObjectTag, Lsn, Vec<u8>)>> {
+        let db = self.store.db.read().unwrap();
+        for pair in db.range((Excluded(&self.curr_key), Unbounded)) {
+            let key = pair.0;
+            if key.obj_key.timeline != self.timeline {
+                return Ok(None);
+            }
+            if key.lsn > self.lsn {
+                // TODO can speed up by seeking iterator
+                continue;
+            }
+            self.curr_key = key.clone();
+            let value = pair.1.clone();
+            return Ok(Some((key.obj_key.tag, key.lsn, value)));
+        }
+        Ok(None)
+    }
+}
+
+///
+/// Iterator for `list_objects`. Returns all objects preceeding specified LSN
+///
+struct InmemObjectIter<'a> {
+    store: &'a InmemObjectStore,
+    curr_key: StorageKey,
+    timeline: ZTimelineId,
+    nonrel_only: bool,
+    lsn: Lsn,
+}
+
+impl<'a> Iterator for InmemObjectIter<'a> {
+    type Item = ObjectTag;
+
+    fn next(&mut self) -> std::option::Option<Self::Item> {
+        let db = self.store.db.read().unwrap();
+        'outer: loop {
+            for pair in db.range((Excluded(&self.curr_key), Unbounded)) {
+                let key = pair.0;
+                if key.obj_key.timeline != self.timeline {
+                    return None;
+                }
+                self.curr_key = key.clone();
+                self.curr_key.lsn = Lsn(u64::MAX); // next seek should skip all versions
+                if key.lsn <= self.lsn {
+                    // visible in this snapshot
+                    if self.nonrel_only {
+                        match key.obj_key.tag {
+                            ObjectTag::RelationMetadata(_) => return None,
+                            ObjectTag::RelationBuffer(_) => return None,
+                            _ => return Some(key.obj_key.tag),
+                        }
+                    } else {
+                        return Some(key.obj_key.tag);
+                    }
+                }
+                continue 'outer;
+            }
+            return None;
+        }
+    }
+}

pageserver/src/layered_repository/README.md

@@ -1,419 +0,0 @@
-# Overview
-
-The on-disk format is based on immutable files. The page server
-receives a stream of incoming WAL, parses the WAL records to determine
-which pages they apply to, and accumulates the incoming changes in
-memory. Every now and then, the accumulated changes are written out to
-new immutable files. This process is called checkpointing. Old versions
-of on-disk files that are not needed by any timeline are removed by GC
-process.
-
-# Terms used in layered repository
-
-- Relish - one PostgreSQL relation or similarly treated file.
-- Segment - one slice of a Relish that is stored in a LayeredTimeline.
-- Layer -  specific version of a relish Segment in a range of LSNs.
-
-Layers can be InMemory or OnDisk:
-- InMemory layer is not durably stored and needs to rebuild from WAL on pageserver start.
-- OnDisk layer is durably stored.
-
-OnDisk layers can be Image or Delta:
-- ImageLayer represents an image or a snapshot of a segment at one particular LSN.
-- DeltaLayer represents a collection of WAL records or page images in a range of LSNs.
-
-Dropped segments are always represented on disk by DeltaLayer.
-
-LSN range defined by start_lsn and end_lsn:
-- start_lsn is inclusive.
-- end_lsn is exclusive.
-
-For an open in-memory layer, the end_lsn is MAX_LSN. For a frozen
-in-memory layer or a delta layer, it is a valid end bound. An image
-layer represents snapshot at one LSN, so end_lsn is always the
-snapshot LSN + 1
-
-Layers can be open or historical:
-- Open layer is a writeable one. Only InMemory layer can be open.
-FIXME: If open layer is dropped, it is not writeable, so it should be turned into historical, 
-but now it is not implemented - see bug #569.
-- Historical layer is the one that cannot be modified anymore. Now only OnDisk layers can be historical.
-
-- LayerMap - a map that tracks what layers exist for all the relishes in a timeline.
-
-LayerMap consists of two data structures:
-- segs - All the layers keyed by segment tag
-- open_layers - data structure that hold all open layers ordered by oldest_pending_lsn for quick access during checkpointing. oldest_pending_lsn is the LSN of the oldest page version stored in this layer.
-
-All operations that update InMemory Layers should update both structures to keep them up-to-date.
-
-- LayeredTimeline - implements Timeline interface.
-
-All methods of LayeredTimeline are aware of its ancestors and return data taking them into account.
-TODO: Are there any exceptions to this?
-For example, timeline.list_rels(lsn) will return all segments that are visible in this timeline at the LSN,
-including ones that were not modified in this timeline and thus don't have a layer in the timeline's LayerMap.
-
-TODO:
-Describe GC and checkpoint interval settings.
-
-# Layer files (On-disk layers)
-
-The files are called "layer files". Each layer file corresponds
-to one RELISH_SEG_SIZE slice of a PostgreSQL relation fork or
-non-rel file in a range of LSNs. The layer files
-for each timeline are stored in the timeline's subdirectory under
-.zenith/tenants/<tenantid>/timelines.
-
-There are two kind of layer file: base images, and deltas. A base
-image file contains a layer of a segment as it was at one LSN,
-whereas a delta file contains modifications to a segment - mostly in
-the form of WAL records - in a range of LSN
-
-base image file:
-
-    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>
-
-delta file:
-
-    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>_<end LSN>
-
-For example:
-
-    rel_1663_13990_2609_0_10_000000000169C348
-    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000
-
-In addition to the relations, with "rel_*" prefix, we use the same
-format for storing various smaller files from the PostgreSQL data
-directory. They will use different suffixes and the naming scheme up
-to the LSNs vary. The Zenith source code uses the term "relish" to
-mean "a relation, or other file that's treated like a relation in the
-storage" For example, a base image of a CLOG segment would be named
-like this:
-
-    pg_xact_0000_0_00000000198B06B0
-
-There is no difference in how the relation and non-relation files are
-managed, except that the first part of file names is different.
-Internally, the relations and non-relation files that are managed in
-the versioned store are together called "relishes".
-
-If a file has been dropped, the last layer file for it is created
-with the _DROPPED suffix, e.g.
-
-    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000_DROPPED
-
-
-## Notation used in this document
-
-The full path of a delta file looks like this:
-
-    .zenith/tenants/941ddc8604413b88b3d208bddf90396c/timelines/4af489b06af8eed9e27a841775616962/rel_1663_13990_2609_0_10_000000000169C348_0000000001702000
-
-For simplicity, the examples below use a simplified notation for the
-paths.  The tenant ID is left out, the timeline ID is replaced with
-the human-readable branch name, and spcnode+dbnode+relnode+forkum+segno
-with a human-readable table name. The LSNs are also shorter. For
-example, a base image file at LSN 100 and a delta file between 100-200
-for 'orders' table on 'main' branch is represented like this:
-
-    main/orders_100
-    main/orders_100_200
-
-
-# Creating layer files
-
-Let's start with a simple example with a system that contains one
-branch called 'main' and two tables, 'orders' and 'customers'. The end
-of WAL is currently at LSN 250. In this starting situation, you would
-have these files on disk:
-
-	main/orders_100
-	main/orders_100_200
-	main/orders_200
-	main/customers_100
-	main/customers_100_200
-	main/customers_200
-
-In addition to those files, the recent changes between LSN 200 and the
-end of WAL at 250 are kept in memory. If the page server crashes, the
-latest records between 200-250 need to be re-read from the WAL.
-
-Whenever enough WAL has been accumulated in memory, the page server
-writes out the changes in memory into new layer files. This process
-is called "checkpointing" (not to be confused with the PostgreSQL
-checkpoints, that's a different thing). The page server only creates
-layer files for relations that have been modified since the last
-checkpoint. For example, if the current end of WAL is at LSN 450, and
-the last checkpoint happened at LSN 400 but there hasn't been any
-recent changes to 'customers' table, you would have these files on
-disk:
-
-	main/orders_100
-	main/orders_100_200
-	main/orders_200
-	main/orders_200_300
-	main/orders_300
-	main/orders_300_400
-	main/orders_400
-	main/customers_100
-	main/customers_100_200
-	main/customers_200
-
-If the customers table is modified later, a new file is created for it
-at the next checkpoint. The new file will cover the "gap" from the
-last layer file, so the LSN ranges are always contiguous:
-
-	main/orders_100
-	main/orders_100_200
-	main/orders_200
-	main/orders_200_300
-	main/orders_300
-	main/orders_300_400
-	main/orders_400
-	main/customers_100
-	main/customers_100_200
-	main/customers_200
-	main/customers_200_500
-	main/customers_500
-
-## Reading page versions
-
-Whenever a GetPage@LSN request comes in from the compute node, the
-page server needs to reconstruct the requested page, as it was at the
-requested LSN. To do that, the page server first checks the recent
-in-memory layer; if the requested page version is found there, it can
-be returned immediatedly without looking at the files on
-disk. Otherwise the page server needs to locate the layer file that
-contains the requested page version.
-
-For example, if a request comes in for table 'orders' at LSN 250, the
-page server would load the 'main/orders_200_300' file into memory, and
-reconstruct and return the requested page from it, as it was at
-LSN 250. Because the layer file consists of a full image of the
-relation at the start LSN and the WAL, reconstructing the page
-involves replaying any WAL records applicable to the page between LSNs
-200-250, starting from the base image at LSN 200.
-
-# Multiple branches
-
-Imagine that a child branch is created at LSN 250:
-
-            @250
-    ----main--+-------------------------->
-               \
-                +---child-------------->
-
-
-Then, the 'orders' table is updated differently on the 'main' and
-'child' branches. You now have this situation on disk:
-
-    main/orders_100
-    main/orders_100_200
-    main/orders_200
-    main/orders_200_300
-    main/orders_300
-    main/orders_300_400
-    main/orders_400
-    main/customers_100
-    main/customers_100_200
-    main/customers_200
-    child/orders_250_300
-    child/orders_300
-    child/orders_300_400
-    child/orders_400
-
-Because the 'customers' table hasn't been modified on the child
-branch, there is no file for it there. If you request a page for it on
-the 'child' branch, the page server will not find any layer file
-for it in the 'child' directory, so it will recurse to look into the
-parent 'main' branch instead.
-
-From the 'child' branch's point of view, the history for each relation
-is linear, and the request's LSN identifies unambiguously which file
-you need to look at. For example, the history for the 'orders' table
-on the 'main' branch consists of these files:
-
-    main/orders_100
-    main/orders_100_200
-    main/orders_200
-    main/orders_200_300
-    main/orders_300
-    main/orders_300_400
-    main/orders_400
-
-And from the 'child' branch's point of view, it consists of these
-files:
-
-    main/orders_100
-    main/orders_100_200
-    main/orders_200
-    main/orders_200_300
-    child/orders_250_300
-    child/orders_300
-    child/orders_300_400
-    child/orders_400
-
-The branch metadata includes the point where the child branch was
-created, LSN 250. If a page request comes with LSN 275, we read the
-page version from the 'child/orders_250_300' file. We might also
-need to reconstruct the page version as it was at LSN 250, in order
-to replay the WAL up to LSN 275, using 'main/orders_200_300' and
-'main/orders_200'. The page versions between 250-300 in the
-'main/orders_200_300' file are ignored when operating on the child
-branch.
-
-Note: It doesn't make any difference if the child branch is created
-when the end of the main branch was at LSN 250, or later when the tip of
-the main branch had already moved on. The latter case, creating a
-branch at a historic LSN, is how we support PITR in Zenith.
-
-
-# Garbage collection
-
-In this scheme, we keep creating new layer files over time. We also
-need a mechanism to remove old files that are no longer needed,
-because disk space isn't infinite.
-
-What files are still needed? Currently, the page server supports PITR
-and branching from any branch at any LSN that is "recent enough" from
-the tip of the branch.  "Recent enough" is defined as an LSN horizon,
-which by default is 64 MB.  (See DEFAULT_GC_HORIZON). For this
-example, let's assume that the LSN horizon is 150 units.
-
-Let's look at the single branch scenario again. Imagine that the end
-of the branch is LSN 525, so that the GC horizon is currently at
-525-150 = 375
-
-	main/orders_100
-	main/orders_100_200
-	main/orders_200
-	main/orders_200_300
-	main/orders_300
-	main/orders_300_400
-	main/orders_400
-	main/orders_400_500
-	main/orders_500
-	main/customers_100
-	main/customers_100_200
-	main/customers_200
-
-We can remove the following files because the end LSNs of those files are
-older than GC horizon 375, and there are more recent layer files for the
-table:
-
-	main/orders_100       DELETE
-	main/orders_100_200   DELETE
-	main/orders_200       DELETE
-	main/orders_200_300   DELETE
-	main/orders_300       STILL NEEDED BY orders_300_400
-	main/orders_300_400   KEEP, NEWER THAN GC HORIZON
-	main/orders_400       .. 
-	main/orders_400_500   .. 
-	main/orders_500       .. 
-	main/customers_100      DELETE
-	main/customers_100_200  DELETE
-	main/customers_200      KEEP, NO NEWER VERSION
-
-'main/customers_100_200' is old enough, but it cannot be
-removed because there is no newer layer file for the table.
-
-Things get slightly more complicated with multiple branches. All of
-the above still holds, but in addition to recent files we must also
-retain older shapshot files that are still needed by child branches.
-For example, if child branch is created at LSN 150, and the 'customers'
-table is updated on the branch, you would have these files:
-
-	main/orders_100        KEEP, NEEDED BY child BRANCH
-	main/orders_100_200    KEEP, NEEDED BY child BRANCH
-	main/orders_200        DELETE
-	main/orders_200_300    DELETE
-	main/orders_300        KEEP, NEWER THAN GC HORIZON
-	main/orders_300_400    KEEP, NEWER THAN GC HORIZON
-	main/orders_400        KEEP, NEWER THAN GC HORIZON
-	main/orders_400_500    KEEP, NEWER THAN GC HORIZON
-	main/orders_500        KEEP, NEWER THAN GC HORIZON
-	main/customers_100       DELETE
-	main/customers_100_200   DELETE
-	main/customers_200       KEEP, NO NEWER VERSION
-	child/customers_150_300  DELETE
-	child/customers_300      KEEP, NO NEWER VERSION
-
-In this situation, 'main/orders_100' and 'main/orders_100_200' cannot
-be removed, even though they are older than the GC horizon, because
-they are still needed by the child branch. 'main/orders_200'
-and 'main/orders_200_300' can still be removed.
-
-If 'orders' is modified later on the 'child' branch, we will create a
-new base image and delta file for it on the child:
-
-	main/orders_100
-	main/orders_100_200
-
-	main/orders_300
-	main/orders_300_400
-	main/orders_400
-	main/orders_400_500
-	main/orders_500
-	main/customers_200
-	child/customers_300
-	child/orders_150_400
-	child/orders_400
-
-After this, the 'main/orders_100' and 'main/orders_100_200' file could
-be removed. It is no longer needed by the child branch, because there
-is a newer layer file there. TODO: This optimization hasn't been
-implemented! The GC algorithm will currently keep the file on the
-'main' branch anyway, for as long as the child branch exists.
-
-
-# TODO: On LSN ranges
-
-In principle, each relation can be checkpointed separately, i.e. the
-LSN ranges of the files don't need to line up. So this would be legal:
-
-	main/orders_100
-	main/orders_100_200
-	main/orders_200
-	main/orders_200_300
-	main/orders_300
-	main/orders_300_400
-	main/orders_400
-	main/customers_150
-	main/customers_150_250
-	main/customers_250
-	main/customers_250_500
-	main/customers_500
-
-However, the code currently always checkpoints all relations together.
-So that situation doesn't arise in practice.
-
-It would also be OK to have overlapping LSN ranges for the same relation:
-
-	main/orders_100
-	main/orders_100_200
-	main/orders_200
-	main/orders_200_300
-	main/orders_300
-	main/orders_250_350
-	main/orders_350
-	main/orders_300_400
-	main/orders_400
-
-The code that reads the layer files should cope with this, but this
-situation doesn't arise either, because the checkpointing code never
-does that.  It could be useful, however, as a transient state when
-garbage collecting around branch points, or explicit recovery
-points. For example, if we start with this:
-
-	main/orders_100
-	main/orders_100_200
-	main/orders_200
-	main/orders_200_300
-	main/orders_300
-
-And there is a branch or explicit recovery point at LSN 150, we could
-replace 'main/orders_100_200' with 'main/orders_150' to keep a
-layer only at that exact point that's still needed, removing the
-other page versions around it. But such compaction has not been
-implemented yet.

pageserver/src/layered_repository/blob.rs

@@ -1,45 +0,0 @@
-use std::{fs::File, io::Write};
-
-use anyhow::Result;
-use bookfile::{BookWriter, BoundedReader, ChapterId, ChapterWriter};
-use serde::{Deserialize, Serialize};
-
-#[derive(Serialize, Deserialize)]
-pub struct BlobRange {
-    offset: u64,
-    size: usize,
-}
-
-pub fn read_blob(reader: &BoundedReader<&'_ File>, range: &BlobRange) -> Result<Vec<u8>> {
-    let mut buf = vec![0u8; range.size];
-    reader.read_exact_at(&mut buf, range.offset)?;
-    Ok(buf)
-}
-
-pub struct BlobWriter<W> {
-    writer: ChapterWriter<W>,
-    offset: u64,
-}
-
-impl<W: Write> BlobWriter<W> {
-    // This function takes a BookWriter and creates a new chapter to ensure offset is 0.
-    pub fn new(book_writer: BookWriter<W>, chapter_id: impl Into<ChapterId>) -> Self {
-        let writer = book_writer.new_chapter(chapter_id);
-        Self { writer, offset: 0 }
-    }
-
-    pub fn write_blob(&mut self, blob: &[u8]) -> Result<BlobRange> {
-        self.writer.write_all(blob)?;
-
-        let range = BlobRange {
-            offset: self.offset,
-            size: blob.len(),
-        };
-        self.offset += blob.len() as u64;
-        Ok(range)
-    }
-
-    pub fn close(self) -> bookfile::Result<BookWriter<W>> {
-        self.writer.close()
-    }
-}

pageserver/src/layered_repository/delta_layer.rs

@@ -1,577 +0,0 @@
-//!
-//! A DeltaLayer represents a collection of WAL records or page images in a range of
-//! LSNs, for one segment. It is stored on a file on disk.
-//!
-//! Usually a delta layer only contains differences - in the form of WAL records against
-//! a base LSN. However, if a segment is newly created, by creating a new relation or
-//! extending an old one, there might be no base image. In that case, all the entries in
-//! the delta layer must be page images or WAL records with the 'will_init' flag set, so
-//! that they can be replayed without referring to an older page version. Also in some
-//! circumstances, the predecessor layer might actually be another delta layer. That
-//! can happen when you create a new branch in the middle of a delta layer, and the WAL
-//! records on the new branch are put in a new delta layer.
-//!
-//! When a delta file needs to be accessed, we slurp the metadata and relsize chapters
-//! into memory, into the DeltaLayerInner struct. See load() and unload() functions.
-//! To access a page/WAL record, we search `page_version_metas` for the block # and LSN.
-//! The byte ranges in the metadata can be used to find the page/WAL record in
-//! PAGE_VERSIONS_CHAPTER.
-//!
-//! On disk, the delta files are stored in timelines/<timelineid> directory.
-//! Currently, there are no subdirectories, and each delta file is named like this:
-//!
-//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>_<end LSN>
-//!
-//! For example:
-//!
-//!    1663_13990_2609_0_5_000000000169C348_000000000169C349
-//!
-//! If a relation is dropped, we add a '_DROPPED' to the end of the filename to indicate that.
-//! So the above example would become:
-//!
-//!    1663_13990_2609_0_5_000000000169C348_000000000169C349_DROPPED
-//!
-//! The end LSN indicates when it was dropped in that case, we don't store it in the
-//! file contents in any way.
-//!
-//! A detlta file is constructed using the 'bookfile' crate. Each file consists of two
-//! parts: the page versions and the relation sizes. They are stored as separate chapters.
-//!
-use crate::layered_repository::blob::BlobWriter;
-use crate::layered_repository::filename::{DeltaFileName, PathOrConf};
-use crate::layered_repository::storage_layer::{
-    Layer, PageReconstructData, PageReconstructResult, PageVersion, SegmentTag,
-};
-use crate::waldecoder;
-use crate::PageServerConf;
-use crate::{ZTenantId, ZTimelineId};
-use anyhow::{bail, ensure, Result};
-use log::*;
-use serde::{Deserialize, Serialize};
-use zenith_utils::vec_map::VecMap;
-// avoid binding to Write (conflicts with std::io::Write)
-// while being able to use std::fmt::Write's methods
-use std::fmt::Write as _;
-use std::fs;
-use std::fs::File;
-use std::io::{BufWriter, Write};
-use std::ops::Bound::Included;
-use std::path::{Path, PathBuf};
-use std::sync::{Mutex, MutexGuard};
-
-use bookfile::{Book, BookWriter};
-
-use zenith_utils::bin_ser::BeSer;
-use zenith_utils::lsn::Lsn;
-
-use super::blob::{read_blob, BlobRange};
-
-// Magic constant to identify a Zenith delta file
-pub const DELTA_FILE_MAGIC: u32 = 0x5A616E01;
-
-/// Mapping from (block #, lsn) -> page/WAL record
-/// byte ranges in PAGE_VERSIONS_CHAPTER
-static PAGE_VERSION_METAS_CHAPTER: u64 = 1;
-/// Page/WAL bytes - cannot be interpreted
-/// without PAGE_VERSION_METAS_CHAPTER
-static PAGE_VERSIONS_CHAPTER: u64 = 2;
-static REL_SIZES_CHAPTER: u64 = 3;
-
-/// Contains the [`Summary`] struct
-static SUMMARY_CHAPTER: u64 = 4;
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-struct Summary {
-    tenantid: ZTenantId,
-    timelineid: ZTimelineId,
-    seg: SegmentTag,
-
-    start_lsn: Lsn,
-    end_lsn: Lsn,
-
-    dropped: bool,
-}
-
-impl From<&DeltaLayer> for Summary {
-    fn from(layer: &DeltaLayer) -> Self {
-        Self {
-            tenantid: layer.tenantid,
-            timelineid: layer.timelineid,
-            seg: layer.seg,
-
-            start_lsn: layer.start_lsn,
-            end_lsn: layer.end_lsn,
-
-            dropped: layer.dropped,
-        }
-    }
-}
-
-///
-/// DeltaLayer is the in-memory data structure associated with an
-/// on-disk delta file.  We keep a DeltaLayer in memory for each
-/// file, in the LayerMap. If a layer is in "loaded" state, we have a
-/// copy of the file in memory, in 'inner'. Otherwise the struct is
-/// just a placeholder for a file that exists on disk, and it needs to
-/// be loaded before using it in queries.
-///
-pub struct DeltaLayer {
-    path_or_conf: PathOrConf,
-
-    pub tenantid: ZTenantId,
-    pub timelineid: ZTimelineId,
-    pub seg: SegmentTag,
-
-    //
-    // This entry contains all the changes from 'start_lsn' to 'end_lsn'. The
-    // start is inclusive, and end is exclusive.
-    //
-    pub start_lsn: Lsn,
-    pub end_lsn: Lsn,
-
-    dropped: bool,
-
-    inner: Mutex<DeltaLayerInner>,
-}
-
-pub struct DeltaLayerInner {
-    /// If false, the 'page_version_metas' and 'relsizes' have not been
-    /// loaded into memory yet.
-    loaded: bool,
-
-    /// All versions of all pages in the file are are kept here.
-    /// Indexed by block number and LSN.
-    page_version_metas: VecMap<(u32, Lsn), BlobRange>,
-
-    /// `relsizes` tracks the size of the relation at different points in time.
-    relsizes: VecMap<Lsn, u32>,
-}
-
-impl Layer for DeltaLayer {
-    fn get_timeline_id(&self) -> ZTimelineId {
-        self.timelineid
-    }
-
-    fn get_seg_tag(&self) -> SegmentTag {
-        self.seg
-    }
-
-    fn is_dropped(&self) -> bool {
-        self.dropped
-    }
-
-    fn get_start_lsn(&self) -> Lsn {
-        self.start_lsn
-    }
-
-    fn get_end_lsn(&self) -> Lsn {
-        self.end_lsn
-    }
-
-    fn filename(&self) -> PathBuf {
-        PathBuf::from(self.layer_name().to_string())
-    }
-
-    /// Look up given page in the cache.
-    fn get_page_reconstruct_data(
-        &self,
-        blknum: u32,
-        lsn: Lsn,
-        reconstruct_data: &mut PageReconstructData,
-    ) -> Result<PageReconstructResult> {
-        let mut need_image = true;
-
-        assert!(self.seg.blknum_in_seg(blknum));
-
-        {
-            // Open the file and lock the metadata in memory
-            // TODO: avoid opening the file for each read
-            let (_path, book) = self.open_book()?;
-            let page_version_reader = book.chapter_reader(PAGE_VERSIONS_CHAPTER)?;
-            let inner = self.load()?;
-
-            // Scan the metadata BTreeMap backwards, starting from the given entry.
-            let minkey = (blknum, Lsn(0));
-            let maxkey = (blknum, lsn);
-            let iter = inner
-                .page_version_metas
-                .slice_range((Included(&minkey), Included(&maxkey)))
-                .iter()
-                .rev();
-            for ((_blknum, pv_lsn), blob_range) in iter {
-                let pv = PageVersion::des(&read_blob(&page_version_reader, blob_range)?)?;
-
-                if let Some(img) = pv.page_image {
-                    // Found a page image, return it
-                    reconstruct_data.page_img = Some(img);
-                    need_image = false;
-                    break;
-                } else if let Some(rec) = pv.record {
-                    let will_init = rec.will_init;
-                    reconstruct_data.records.push((*pv_lsn, rec));
-                    if will_init {
-                        // This WAL record initializes the page, so no need to go further back
-                        need_image = false;
-                        break;
-                    }
-                } else {
-                    // No base image, and no WAL record. Huh?
-                    bail!("no page image or WAL record for requested page");
-                }
-            }
-
-            // release metadata lock and close the file
-        }
-
-        // If an older page image is needed to reconstruct the page, let the
-        // caller know.
-        if need_image {
-            Ok(PageReconstructResult::Continue(self.start_lsn))
-        } else {
-            Ok(PageReconstructResult::Complete)
-        }
-    }
-
-    /// Get size of the relation at given LSN
-    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
-        assert!(lsn >= self.start_lsn);
-        ensure!(
-            self.seg.rel.is_blocky(),
-            "get_seg_size() called on a non-blocky rel"
-        );
-
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let inner = self.load()?;
-        let slice = inner
-            .relsizes
-            .slice_range((Included(&Lsn(0)), Included(&lsn)));
-
-        if let Some((_entry_lsn, entry)) = slice.last() {
-            Ok(*entry)
-        } else {
-            Err(anyhow::anyhow!("could not find seg size in delta layer"))
-        }
-    }
-
-    /// Does this segment exist at given LSN?
-    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
-        // Is the requested LSN after the rel was dropped?
-        if self.dropped && lsn >= self.end_lsn {
-            return Ok(false);
-        }
-
-        // Otherwise, it exists.
-        Ok(true)
-    }
-
-    ///
-    /// Release most of the memory used by this layer. If it's accessed again later,
-    /// it will need to be loaded back.
-    ///
-    fn unload(&self) -> Result<()> {
-        let mut inner = self.inner.lock().unwrap();
-        inner.page_version_metas = VecMap::default();
-        inner.relsizes = VecMap::default();
-        inner.loaded = false;
-        Ok(())
-    }
-
-    fn delete(&self) -> Result<()> {
-        // delete underlying file
-        fs::remove_file(self.path())?;
-        Ok(())
-    }
-
-    fn is_incremental(&self) -> bool {
-        true
-    }
-
-    /// debugging function to print out the contents of the layer
-    fn dump(&self) -> Result<()> {
-        println!(
-            "----- delta layer for ten {} tli {} seg {} {}-{} ----",
-            self.tenantid, self.timelineid, self.seg, self.start_lsn, self.end_lsn
-        );
-
-        println!("--- relsizes ---");
-        let inner = self.load()?;
-        for (k, v) in inner.relsizes.as_slice() {
-            println!("  {}: {}", k, v);
-        }
-        println!("--- page versions ---");
-        let (_path, book) = self.open_book()?;
-        let chapter = book.chapter_reader(PAGE_VERSIONS_CHAPTER)?;
-        for ((blk, lsn), blob_range) in inner.page_version_metas.as_slice() {
-            let mut desc = String::new();
-
-            let buf = read_blob(&chapter, blob_range)?;
-            let pv = PageVersion::des(&buf)?;
-
-            if let Some(img) = pv.page_image.as_ref() {
-                write!(&mut desc, " img {} bytes", img.len())?;
-            }
-            if let Some(rec) = pv.record.as_ref() {
-                let wal_desc = waldecoder::describe_wal_record(&rec.rec);
-                write!(
-                    &mut desc,
-                    " rec {} bytes will_init: {} {}",
-                    rec.rec.len(),
-                    rec.will_init,
-                    wal_desc
-                )?;
-            }
-            println!("  blk {} at {}: {}", blk, lsn, desc);
-        }
-
-        Ok(())
-    }
-}
-
-impl DeltaLayer {
-    fn path_for(
-        path_or_conf: &PathOrConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        fname: &DeltaFileName,
-    ) -> PathBuf {
-        match path_or_conf {
-            PathOrConf::Path(path) => path.clone(),
-            PathOrConf::Conf(conf) => conf
-                .timeline_path(&timelineid, &tenantid)
-                .join(fname.to_string()),
-        }
-    }
-
-    /// Create a new delta file, using the given page versions and relsizes.
-    /// The page versions are passed by an iterator; the iterator must return
-    /// page versions in blknum+lsn order.
-    ///
-    /// This is used to write the in-memory layer to disk. The in-memory layer uses the same
-    /// data structure with two btreemaps as we do, so passing the btreemaps is currently
-    /// expedient.
-    #[allow(clippy::too_many_arguments)]
-    pub fn create<'a>(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        seg: SegmentTag,
-        start_lsn: Lsn,
-        end_lsn: Lsn,
-        dropped: bool,
-        page_versions: impl Iterator<Item = (u32, Lsn, &'a PageVersion)>,
-        relsizes: VecMap<Lsn, u32>,
-    ) -> Result<DeltaLayer> {
-        if seg.rel.is_blocky() {
-            assert!(!relsizes.is_empty());
-        }
-
-        let delta_layer = DeltaLayer {
-            path_or_conf: PathOrConf::Conf(conf),
-            timelineid,
-            tenantid,
-            seg,
-            start_lsn,
-            end_lsn,
-            dropped,
-            inner: Mutex::new(DeltaLayerInner {
-                loaded: true,
-                page_version_metas: VecMap::default(),
-                relsizes,
-            }),
-        };
-        let mut inner = delta_layer.inner.lock().unwrap();
-
-        // Write the in-memory btreemaps into a file
-        let path = delta_layer.path();
-
-        // Note: This overwrites any existing file. There shouldn't be any.
-        // FIXME: throw an error instead?
-        let file = File::create(&path)?;
-        let buf_writer = BufWriter::new(file);
-        let book = BookWriter::new(buf_writer, DELTA_FILE_MAGIC)?;
-
-        let mut page_version_writer = BlobWriter::new(book, PAGE_VERSIONS_CHAPTER);
-
-        for (blknum, lsn, page_version) in page_versions {
-            let buf = PageVersion::ser(page_version)?;
-            let blob_range = page_version_writer.write_blob(&buf)?;
-
-            inner
-                .page_version_metas
-                .append((blknum, lsn), blob_range)
-                .unwrap();
-        }
-
-        let book = page_version_writer.close()?;
-
-        // Write out page versions
-        let mut chapter = book.new_chapter(PAGE_VERSION_METAS_CHAPTER);
-        let buf = VecMap::ser(&inner.page_version_metas)?;
-        chapter.write_all(&buf)?;
-        let book = chapter.close()?;
-
-        // and relsizes to separate chapter
-        let mut chapter = book.new_chapter(REL_SIZES_CHAPTER);
-        let buf = VecMap::ser(&inner.relsizes)?;
-        chapter.write_all(&buf)?;
-        let book = chapter.close()?;
-
-        let mut chapter = book.new_chapter(SUMMARY_CHAPTER);
-        let summary = Summary {
-            tenantid,
-            timelineid,
-            seg,
-
-            start_lsn,
-            end_lsn,
-
-            dropped,
-        };
-        Summary::ser_into(&summary, &mut chapter)?;
-        let book = chapter.close()?;
-
-        // This flushes the underlying 'buf_writer'.
-        let writer = book.close()?;
-        writer.get_ref().sync_all()?;
-
-        trace!("saved {}", &path.display());
-
-        drop(inner);
-
-        Ok(delta_layer)
-    }
-
-    fn open_book(&self) -> Result<(PathBuf, Book<File>)> {
-        let path = Self::path_for(
-            &self.path_or_conf,
-            self.timelineid,
-            self.tenantid,
-            &self.layer_name(),
-        );
-
-        let file = File::open(&path)?;
-        let book = Book::new(file)?;
-
-        Ok((path, book))
-    }
-
-    ///
-    /// Load the contents of the file into memory
-    ///
-    fn load(&self) -> Result<MutexGuard<DeltaLayerInner>> {
-        // quick exit if already loaded
-        let mut inner = self.inner.lock().unwrap();
-
-        if inner.loaded {
-            return Ok(inner);
-        }
-
-        let (path, book) = self.open_book()?;
-
-        match &self.path_or_conf {
-            PathOrConf::Conf(_) => {
-                let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
-                let actual_summary = Summary::des(&chapter)?;
-
-                let expected_summary = Summary::from(self);
-
-                if actual_summary != expected_summary {
-                    bail!("in-file summary does not match expected summary. actual = {:?} expected = {:?}", actual_summary, expected_summary);
-                }
-            }
-            PathOrConf::Path(path) => {
-                let actual_filename = Path::new(path.file_name().unwrap());
-                let expected_filename = self.filename();
-
-                if actual_filename != expected_filename {
-                    println!(
-                        "warning: filename does not match what is expected from in-file summary"
-                    );
-                    println!("actual: {:?}", actual_filename);
-                    println!("expected: {:?}", expected_filename);
-                }
-            }
-        }
-
-        let chapter = book.read_chapter(PAGE_VERSION_METAS_CHAPTER)?;
-        let page_version_metas = VecMap::des(&chapter)?;
-
-        let chapter = book.read_chapter(REL_SIZES_CHAPTER)?;
-        let relsizes = VecMap::des(&chapter)?;
-
-        debug!("loaded from {}", &path.display());
-
-        *inner = DeltaLayerInner {
-            loaded: true,
-            page_version_metas,
-            relsizes,
-        };
-
-        Ok(inner)
-    }
-
-    /// Create a DeltaLayer struct representing an existing file on disk.
-    pub fn new(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        filename: &DeltaFileName,
-    ) -> DeltaLayer {
-        DeltaLayer {
-            path_or_conf: PathOrConf::Conf(conf),
-            timelineid,
-            tenantid,
-            seg: filename.seg,
-            start_lsn: filename.start_lsn,
-            end_lsn: filename.end_lsn,
-            dropped: filename.dropped,
-            inner: Mutex::new(DeltaLayerInner {
-                loaded: false,
-                page_version_metas: VecMap::default(),
-                relsizes: VecMap::default(),
-            }),
-        }
-    }
-
-    /// Create a DeltaLayer struct representing an existing file on disk.
-    ///
-    /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary.
-    pub fn new_for_path(path: &Path, book: &Book<File>) -> Result<Self> {
-        let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
-        let summary = Summary::des(&chapter)?;
-
-        Ok(DeltaLayer {
-            path_or_conf: PathOrConf::Path(path.to_path_buf()),
-            timelineid: summary.timelineid,
-            tenantid: summary.tenantid,
-            seg: summary.seg,
-            start_lsn: summary.start_lsn,
-            end_lsn: summary.end_lsn,
-            dropped: summary.dropped,
-            inner: Mutex::new(DeltaLayerInner {
-                loaded: false,
-                page_version_metas: VecMap::default(),
-                relsizes: VecMap::default(),
-            }),
-        })
-    }
-
-    fn layer_name(&self) -> DeltaFileName {
-        DeltaFileName {
-            seg: self.seg,
-            start_lsn: self.start_lsn,
-            end_lsn: self.end_lsn,
-            dropped: self.dropped,
-        }
-    }
-
-    /// Path to the layer file in pageserver workdir.
-    pub fn path(&self) -> PathBuf {
-        Self::path_for(
-            &self.path_or_conf,
-            self.timelineid,
-            self.tenantid,
-            &self.layer_name(),
-        )
-    }
-}

pageserver/src/layered_repository/filename.rs

@@ -1,315 +0,0 @@
-//!
-//! Helper functions for dealing with filenames of the image and delta layer files.
-//!
-use crate::layered_repository::storage_layer::SegmentTag;
-use crate::relish::*;
-use crate::PageServerConf;
-use crate::{ZTenantId, ZTimelineId};
-use std::fmt;
-use std::fs;
-use std::path::PathBuf;
-
-use anyhow::Result;
-use log::*;
-use zenith_utils::lsn::Lsn;
-
-use super::METADATA_FILE_NAME;
-
-// Note: LayeredTimeline::load_layer_map() relies on this sort order
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
-pub struct DeltaFileName {
-    pub seg: SegmentTag,
-    pub start_lsn: Lsn,
-    pub end_lsn: Lsn,
-    pub dropped: bool,
-}
-
-/// Represents the filename of a DeltaLayer
-///
-///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>
-///
-/// or if it was dropped:
-///
-///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>_DROPPED
-///
-impl DeltaFileName {
-    ///
-    /// Parse a string as a delta file name. Returns None if the filename does not
-    /// match the expected pattern.
-    ///
-    pub fn parse_str(fname: &str) -> Option<Self> {
-        let rel;
-        let mut parts;
-        if let Some(rest) = fname.strip_prefix("rel_") {
-            parts = rest.split('_');
-            rel = RelishTag::Relation(RelTag {
-                spcnode: parts.next()?.parse::<u32>().ok()?,
-                dbnode: parts.next()?.parse::<u32>().ok()?,
-                relnode: parts.next()?.parse::<u32>().ok()?,
-                forknum: parts.next()?.parse::<u8>().ok()?,
-            });
-        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::Clog,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::MultiXactMembers,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::MultiXactOffsets,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
-            parts = rest.split('_');
-            rel = RelishTag::FileNodeMap {
-                spcnode: parts.next()?.parse::<u32>().ok()?,
-                dbnode: parts.next()?.parse::<u32>().ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
-            parts = rest.split('_');
-            rel = RelishTag::TwoPhase {
-                xid: parts.next()?.parse::<u32>().ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
-            parts = rest.split('_');
-            rel = RelishTag::Checkpoint;
-        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
-            parts = rest.split('_');
-            rel = RelishTag::ControlFile;
-        } else {
-            return None;
-        }
-
-        let segno = parts.next()?.parse::<u32>().ok()?;
-
-        let seg = SegmentTag { rel, segno };
-
-        let start_lsn = Lsn::from_hex(parts.next()?).ok()?;
-        let end_lsn = Lsn::from_hex(parts.next()?).ok()?;
-
-        let mut dropped = false;
-        if let Some(suffix) = parts.next() {
-            if suffix == "DROPPED" {
-                dropped = true;
-            } else {
-                return None;
-            }
-        }
-        if parts.next().is_some() {
-            return None;
-        }
-
-        Some(DeltaFileName {
-            seg,
-            start_lsn,
-            end_lsn,
-            dropped,
-        })
-    }
-}
-
-impl fmt::Display for DeltaFileName {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let basename = match self.seg.rel {
-            RelishTag::Relation(reltag) => format!(
-                "rel_{}_{}_{}_{}",
-                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
-            ),
-            RelishTag::Slru {
-                slru: SlruKind::Clog,
-                segno,
-            } => format!("pg_xact_{:04X}", segno),
-            RelishTag::Slru {
-                slru: SlruKind::MultiXactMembers,
-                segno,
-            } => format!("pg_multixact_members_{:04X}", segno),
-            RelishTag::Slru {
-                slru: SlruKind::MultiXactOffsets,
-                segno,
-            } => format!("pg_multixact_offsets_{:04X}", segno),
-            RelishTag::FileNodeMap { spcnode, dbnode } => {
-                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
-            }
-            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
-            RelishTag::Checkpoint => "pg_control_checkpoint".to_string(),
-            RelishTag::ControlFile => "pg_control".to_string(),
-        };
-
-        write!(
-            f,
-            "{}_{}_{:016X}_{:016X}{}",
-            basename,
-            self.seg.segno,
-            u64::from(self.start_lsn),
-            u64::from(self.end_lsn),
-            if self.dropped { "_DROPPED" } else { "" }
-        )
-    }
-}
-
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
-pub struct ImageFileName {
-    pub seg: SegmentTag,
-    pub lsn: Lsn,
-}
-
-///
-/// Represents the filename of an ImageLayer
-///
-///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<LSN>
-///
-impl ImageFileName {
-    ///
-    /// Parse a string as an image file name. Returns None if the filename does not
-    /// match the expected pattern.
-    ///
-    pub fn parse_str(fname: &str) -> Option<Self> {
-        let rel;
-        let mut parts;
-        if let Some(rest) = fname.strip_prefix("rel_") {
-            parts = rest.split('_');
-            rel = RelishTag::Relation(RelTag {
-                spcnode: parts.next()?.parse::<u32>().ok()?,
-                dbnode: parts.next()?.parse::<u32>().ok()?,
-                relnode: parts.next()?.parse::<u32>().ok()?,
-                forknum: parts.next()?.parse::<u8>().ok()?,
-            });
-        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::Clog,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::MultiXactMembers,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
-            parts = rest.split('_');
-            rel = RelishTag::Slru {
-                slru: SlruKind::MultiXactOffsets,
-                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
-            parts = rest.split('_');
-            rel = RelishTag::FileNodeMap {
-                spcnode: parts.next()?.parse::<u32>().ok()?,
-                dbnode: parts.next()?.parse::<u32>().ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
-            parts = rest.split('_');
-            rel = RelishTag::TwoPhase {
-                xid: parts.next()?.parse::<u32>().ok()?,
-            };
-        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
-            parts = rest.split('_');
-            rel = RelishTag::Checkpoint;
-        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
-            parts = rest.split('_');
-            rel = RelishTag::ControlFile;
-        } else {
-            return None;
-        }
-
-        let segno = parts.next()?.parse::<u32>().ok()?;
-
-        let seg = SegmentTag { rel, segno };
-
-        let lsn = Lsn::from_hex(parts.next()?).ok()?;
-
-        if parts.next().is_some() {
-            return None;
-        }
-
-        Some(ImageFileName { seg, lsn })
-    }
-}
-
-impl fmt::Display for ImageFileName {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let basename = match self.seg.rel {
-            RelishTag::Relation(reltag) => format!(
-                "rel_{}_{}_{}_{}",
-                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
-            ),
-            RelishTag::Slru {
-                slru: SlruKind::Clog,
-                segno,
-            } => format!("pg_xact_{:04X}", segno),
-            RelishTag::Slru {
-                slru: SlruKind::MultiXactMembers,
-                segno,
-            } => format!("pg_multixact_members_{:04X}", segno),
-            RelishTag::Slru {
-                slru: SlruKind::MultiXactOffsets,
-                segno,
-            } => format!("pg_multixact_offsets_{:04X}", segno),
-            RelishTag::FileNodeMap { spcnode, dbnode } => {
-                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
-            }
-            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
-            RelishTag::Checkpoint => "pg_control_checkpoint".to_string(),
-            RelishTag::ControlFile => "pg_control".to_string(),
-        };
-
-        write!(
-            f,
-            "{}_{}_{:016X}",
-            basename,
-            self.seg.segno,
-            u64::from(self.lsn),
-        )
-    }
-}
-
-/// Scan timeline directory and create ImageFileName and DeltaFilename
-/// structs representing all files on disk
-///
-/// TODO: returning an Iterator would be more idiomatic
-pub fn list_files(
-    conf: &'static PageServerConf,
-    timelineid: ZTimelineId,
-    tenantid: ZTenantId,
-) -> Result<(Vec<ImageFileName>, Vec<DeltaFileName>)> {
-    let path = conf.timeline_path(&timelineid, &tenantid);
-
-    let mut deltafiles: Vec<DeltaFileName> = Vec::new();
-    let mut imgfiles: Vec<ImageFileName> = Vec::new();
-    for direntry in fs::read_dir(path)? {
-        let fname = direntry?.file_name();
-        let fname = fname.to_str().unwrap();
-
-        if let Some(deltafilename) = DeltaFileName::parse_str(fname) {
-            deltafiles.push(deltafilename);
-        } else if let Some(imgfilename) = ImageFileName::parse_str(fname) {
-            imgfiles.push(imgfilename);
-        } else if fname == METADATA_FILE_NAME || fname == "ancestor" || fname.ends_with(".old") {
-            // ignore these
-        } else {
-            warn!("unrecognized filename in timeline dir: {}", fname);
-        }
-    }
-    Ok((imgfiles, deltafiles))
-}
-
-/// Helper enum to hold a PageServerConf, or a path
-///
-/// This is used by DeltaLayer and ImageLayer. Normally, this holds a reference to the
-/// global config, and paths to layer files are constructed using the tenant/timeline
-/// path from the config. But in the 'dump_layerfile' binary, we need to construct a Layer
-/// struct for a file on disk, without having a page server running, so that we have no
-/// config. In that case, we use the Path variant to hold the full path to the file on
-/// disk.
-pub enum PathOrConf {
-    Path(PathBuf),
-    Conf(&'static PageServerConf),
-}

pageserver/src/layered_repository/image_layer.rs

@@ -1,490 +0,0 @@
-//! An ImageLayer represents an image or a snapshot of a segment at one particular LSN.
-//! It is stored in a file on disk.
-//!
-//! On disk, the image files are stored in timelines/<timelineid> directory.
-//! Currently, there are no subdirectories, and each image layer file is named like this:
-//!
-//! Note that segno is
-//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<LSN>
-//!
-//! For example:
-//!
-//!    1663_13990_2609_0_5_000000000169C348
-//!
-//! An image file is constructed using the 'bookfile' crate.
-//!
-//! Only metadata is loaded into memory by the load function.
-//! When images are needed, they are read directly from disk.
-//!
-//! For blocky relishes, the images are stored in BLOCKY_IMAGES_CHAPTER.
-//! All the images are required to be BLOCK_SIZE, which allows for random access.
-//!
-//! For non-blocky relishes, the image can be found in NONBLOCKY_IMAGE_CHAPTER.
-//!
-use crate::layered_repository::filename::{ImageFileName, PathOrConf};
-use crate::layered_repository::storage_layer::{
-    Layer, PageReconstructData, PageReconstructResult, SegmentTag,
-};
-use crate::layered_repository::LayeredTimeline;
-use crate::layered_repository::RELISH_SEG_SIZE;
-use crate::PageServerConf;
-use crate::{ZTenantId, ZTimelineId};
-use anyhow::{anyhow, bail, ensure, Result};
-use bytes::Bytes;
-use log::*;
-use serde::{Deserialize, Serialize};
-use std::convert::TryInto;
-use std::fs;
-use std::fs::File;
-use std::io::{BufWriter, Write};
-use std::path::{Path, PathBuf};
-use std::sync::{Mutex, MutexGuard};
-
-use bookfile::{Book, BookWriter};
-
-use zenith_utils::bin_ser::BeSer;
-use zenith_utils::lsn::Lsn;
-
-// Magic constant to identify a Zenith segment image file
-pub const IMAGE_FILE_MAGIC: u32 = 0x5A616E01 + 1;
-
-/// Contains each block in block # order
-const BLOCKY_IMAGES_CHAPTER: u64 = 1;
-const NONBLOCKY_IMAGE_CHAPTER: u64 = 2;
-
-/// Contains the [`Summary`] struct
-const SUMMARY_CHAPTER: u64 = 3;
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-struct Summary {
-    tenantid: ZTenantId,
-    timelineid: ZTimelineId,
-    seg: SegmentTag,
-
-    lsn: Lsn,
-}
-
-impl From<&ImageLayer> for Summary {
-    fn from(layer: &ImageLayer) -> Self {
-        Self {
-            tenantid: layer.tenantid,
-            timelineid: layer.timelineid,
-            seg: layer.seg,
-
-            lsn: layer.lsn,
-        }
-    }
-}
-
-const BLOCK_SIZE: usize = 8192;
-
-///
-/// ImageLayer is the in-memory data structure associated with an on-disk image
-/// file.  We keep an ImageLayer in memory for each file, in the LayerMap. If a
-/// layer is in "loaded" state, we have a copy of the file in memory, in 'inner'.
-/// Otherwise the struct is just a placeholder for a file that exists on disk,
-/// and it needs to be loaded before using it in queries.
-///
-pub struct ImageLayer {
-    path_or_conf: PathOrConf,
-    pub tenantid: ZTenantId,
-    pub timelineid: ZTimelineId,
-    pub seg: SegmentTag,
-
-    // This entry contains an image of all pages as of this LSN
-    pub lsn: Lsn,
-
-    inner: Mutex<ImageLayerInner>,
-}
-
-#[derive(Clone)]
-enum ImageType {
-    Blocky { num_blocks: u32 },
-    NonBlocky,
-}
-
-pub struct ImageLayerInner {
-    /// If false, the 'image_type' has not been
-    /// loaded into memory yet.
-    loaded: bool,
-
-    /// Derived from filename and bookfile chapter metadata
-    image_type: ImageType,
-}
-
-impl Layer for ImageLayer {
-    fn filename(&self) -> PathBuf {
-        PathBuf::from(self.layer_name().to_string())
-    }
-
-    fn get_timeline_id(&self) -> ZTimelineId {
-        self.timelineid
-    }
-
-    fn get_seg_tag(&self) -> SegmentTag {
-        self.seg
-    }
-
-    fn is_dropped(&self) -> bool {
-        false
-    }
-
-    fn get_start_lsn(&self) -> Lsn {
-        self.lsn
-    }
-
-    fn get_end_lsn(&self) -> Lsn {
-        // End-bound is exclusive
-        self.lsn + 1
-    }
-
-    /// Look up given page in the file
-    fn get_page_reconstruct_data(
-        &self,
-        blknum: u32,
-        lsn: Lsn,
-        reconstruct_data: &mut PageReconstructData,
-    ) -> Result<PageReconstructResult> {
-        assert!(lsn >= self.lsn);
-
-        let inner = self.load()?;
-
-        let base_blknum = blknum % RELISH_SEG_SIZE;
-
-        let (_path, book) = self.open_book()?;
-
-        let buf = match &inner.image_type {
-            ImageType::Blocky { num_blocks } => {
-                if base_blknum >= *num_blocks {
-                    return Ok(PageReconstructResult::Missing(lsn));
-                }
-
-                let mut buf = vec![0u8; BLOCK_SIZE];
-                let offset = BLOCK_SIZE as u64 * base_blknum as u64;
-
-                let chapter = book.chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
-                chapter.read_exact_at(&mut buf, offset)?;
-
-                buf
-            }
-            ImageType::NonBlocky => {
-                ensure!(base_blknum == 0);
-                book.read_chapter(NONBLOCKY_IMAGE_CHAPTER)?.into_vec()
-            }
-        };
-
-        reconstruct_data.page_img = Some(Bytes::from(buf));
-        Ok(PageReconstructResult::Complete)
-    }
-
-    /// Get size of the segment
-    fn get_seg_size(&self, _lsn: Lsn) -> Result<u32> {
-        let inner = self.load()?;
-        match inner.image_type {
-            ImageType::Blocky { num_blocks } => Ok(num_blocks),
-            ImageType::NonBlocky => Err(anyhow!("get_seg_size called for non-blocky segment")),
-        }
-    }
-
-    /// Does this segment exist at given LSN?
-    fn get_seg_exists(&self, _lsn: Lsn) -> Result<bool> {
-        Ok(true)
-    }
-
-    ///
-    /// Release most of the memory used by this layer. If it's accessed again later,
-    /// it will need to be loaded back.
-    ///
-    fn unload(&self) -> Result<()> {
-        let mut inner = self.inner.lock().unwrap();
-        inner.image_type = ImageType::Blocky { num_blocks: 0 };
-        inner.loaded = false;
-        Ok(())
-    }
-
-    fn delete(&self) -> Result<()> {
-        // delete underlying file
-        fs::remove_file(self.path())?;
-        Ok(())
-    }
-
-    fn is_incremental(&self) -> bool {
-        false
-    }
-
-    /// debugging function to print out the contents of the layer
-    fn dump(&self) -> Result<()> {
-        println!(
-            "----- image layer for ten {} tli {} seg {} at {} ----",
-            self.tenantid, self.timelineid, self.seg, self.lsn
-        );
-
-        let inner = self.load()?;
-
-        match inner.image_type {
-            ImageType::Blocky { num_blocks } => println!("({}) blocks ", num_blocks),
-            ImageType::NonBlocky => {
-                let (_path, book) = self.open_book()?;
-                let chapter = book.read_chapter(NONBLOCKY_IMAGE_CHAPTER)?;
-                println!("non-blocky ({} bytes)", chapter.len());
-            }
-        }
-
-        Ok(())
-    }
-}
-
-impl ImageLayer {
-    fn path_for(
-        path_or_conf: &PathOrConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        fname: &ImageFileName,
-    ) -> PathBuf {
-        match path_or_conf {
-            PathOrConf::Path(path) => path.to_path_buf(),
-            PathOrConf::Conf(conf) => conf
-                .timeline_path(&timelineid, &tenantid)
-                .join(fname.to_string()),
-        }
-    }
-
-    /// Create a new image file, using the given array of pages.
-    fn create(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        seg: SegmentTag,
-        lsn: Lsn,
-        base_images: Vec<Bytes>,
-    ) -> Result<ImageLayer> {
-        let image_type = if seg.rel.is_blocky() {
-            let num_blocks: u32 = base_images.len().try_into()?;
-            ImageType::Blocky { num_blocks }
-        } else {
-            assert_eq!(base_images.len(), 1);
-            ImageType::NonBlocky
-        };
-
-        let layer = ImageLayer {
-            path_or_conf: PathOrConf::Conf(conf),
-            timelineid,
-            tenantid,
-            seg,
-            lsn,
-            inner: Mutex::new(ImageLayerInner {
-                loaded: true,
-                image_type: image_type.clone(),
-            }),
-        };
-        let inner = layer.inner.lock().unwrap();
-
-        // Write the images into a file
-        let path = layer.path();
-        // Note: This overwrites any existing file. There shouldn't be any.
-        // FIXME: throw an error instead?
-        let file = File::create(&path)?;
-        let buf_writer = BufWriter::new(file);
-        let book = BookWriter::new(buf_writer, IMAGE_FILE_MAGIC)?;
-
-        let book = match &image_type {
-            ImageType::Blocky { .. } => {
-                let mut chapter = book.new_chapter(BLOCKY_IMAGES_CHAPTER);
-                for block_bytes in base_images {
-                    assert_eq!(block_bytes.len(), BLOCK_SIZE);
-                    chapter.write_all(&block_bytes)?;
-                }
-                chapter.close()?
-            }
-            ImageType::NonBlocky => {
-                let mut chapter = book.new_chapter(NONBLOCKY_IMAGE_CHAPTER);
-                chapter.write_all(&base_images[0])?;
-                chapter.close()?
-            }
-        };
-
-        let mut chapter = book.new_chapter(SUMMARY_CHAPTER);
-        let summary = Summary {
-            tenantid,
-            timelineid,
-            seg,
-
-            lsn,
-        };
-        Summary::ser_into(&summary, &mut chapter)?;
-        let book = chapter.close()?;
-
-        // This flushes the underlying 'buf_writer'.
-        let writer = book.close()?;
-        writer.get_ref().sync_all()?;
-
-        trace!("saved {}", path.display());
-
-        drop(inner);
-
-        Ok(layer)
-    }
-
-    // Create a new image file by materializing every page in a source layer
-    // at given LSN.
-    pub fn create_from_src(
-        conf: &'static PageServerConf,
-        timeline: &LayeredTimeline,
-        src: &dyn Layer,
-        lsn: Lsn,
-    ) -> Result<ImageLayer> {
-        let seg = src.get_seg_tag();
-        let timelineid = timeline.timelineid;
-
-        let startblk;
-        let size;
-        if seg.rel.is_blocky() {
-            size = src.get_seg_size(lsn)?;
-            startblk = seg.segno * RELISH_SEG_SIZE;
-        } else {
-            size = 1;
-            startblk = 0;
-        }
-
-        trace!(
-            "creating new ImageLayer for {} on timeline {} at {}",
-            seg,
-            timelineid,
-            lsn,
-        );
-
-        let mut base_images: Vec<Bytes> = Vec::new();
-        for blknum in startblk..(startblk + size) {
-            let img = timeline.materialize_page(seg, blknum, lsn, &*src)?;
-
-            base_images.push(img);
-        }
-
-        Self::create(conf, timelineid, timeline.tenantid, seg, lsn, base_images)
-    }
-
-    ///
-    /// Load the contents of the file into memory
-    ///
-    fn load(&self) -> Result<MutexGuard<ImageLayerInner>> {
-        // quick exit if already loaded
-        let mut inner = self.inner.lock().unwrap();
-
-        if inner.loaded {
-            return Ok(inner);
-        }
-
-        let (path, book) = self.open_book()?;
-
-        match &self.path_or_conf {
-            PathOrConf::Conf(_) => {
-                let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
-                let actual_summary = Summary::des(&chapter)?;
-
-                let expected_summary = Summary::from(self);
-
-                if actual_summary != expected_summary {
-                    bail!("in-file summary does not match expected summary. actual = {:?} expected = {:?}", actual_summary, expected_summary);
-                }
-            }
-            PathOrConf::Path(path) => {
-                let actual_filename = Path::new(path.file_name().unwrap());
-                let expected_filename = self.filename();
-
-                if actual_filename != expected_filename {
-                    println!(
-                        "warning: filename does not match what is expected from in-file summary"
-                    );
-                    println!("actual: {:?}", actual_filename);
-                    println!("expected: {:?}", expected_filename);
-                }
-            }
-        }
-
-        let image_type = if self.seg.rel.is_blocky() {
-            let chapter = book.chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
-            let images_len = chapter.len();
-            ensure!(images_len % BLOCK_SIZE as u64 == 0);
-            let num_blocks: u32 = (images_len / BLOCK_SIZE as u64).try_into()?;
-            ImageType::Blocky { num_blocks }
-        } else {
-            let _chapter = book.chapter_reader(NONBLOCKY_IMAGE_CHAPTER)?;
-            ImageType::NonBlocky
-        };
-
-        debug!("loaded from {}", &path.display());
-
-        *inner = ImageLayerInner {
-            loaded: true,
-            image_type,
-        };
-
-        Ok(inner)
-    }
-
-    fn open_book(&self) -> Result<(PathBuf, Book<File>)> {
-        let path = self.path();
-
-        let file = File::open(&path)?;
-        let book = Book::new(file)?;
-
-        Ok((path, book))
-    }
-
-    /// Create an ImageLayer struct representing an existing file on disk
-    pub fn new(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        filename: &ImageFileName,
-    ) -> ImageLayer {
-        ImageLayer {
-            path_or_conf: PathOrConf::Conf(conf),
-            timelineid,
-            tenantid,
-            seg: filename.seg,
-            lsn: filename.lsn,
-            inner: Mutex::new(ImageLayerInner {
-                loaded: false,
-                image_type: ImageType::Blocky { num_blocks: 0 },
-            }),
-        }
-    }
-
-    /// Create an ImageLayer struct representing an existing file on disk.
-    ///
-    /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary.
-    pub fn new_for_path(path: &Path, book: &Book<File>) -> Result<ImageLayer> {
-        let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
-        let summary = Summary::des(&chapter)?;
-
-        Ok(ImageLayer {
-            path_or_conf: PathOrConf::Path(path.to_path_buf()),
-            timelineid: summary.timelineid,
-            tenantid: summary.tenantid,
-            seg: summary.seg,
-            lsn: summary.lsn,
-            inner: Mutex::new(ImageLayerInner {
-                loaded: false,
-                image_type: ImageType::Blocky { num_blocks: 0 },
-            }),
-        })
-    }
-
-    fn layer_name(&self) -> ImageFileName {
-        ImageFileName {
-            seg: self.seg,
-            lsn: self.lsn,
-        }
-    }
-
-    /// Path to the layer file in pageserver workdir.
-    pub fn path(&self) -> PathBuf {
-        Self::path_for(
-            &self.path_or_conf,
-            self.timelineid,
-            self.tenantid,
-            &self.layer_name(),
-        )
-    }
-}

pageserver/src/layered_repository/inmemory_layer.rs

@@ -1,659 +0,0 @@
-//!
-//! An in-memory layer stores recently received page versions in memory. The page versions
-//! are held in a BTreeMap, and there's another BTreeMap to track the size of the relation.
-//!
-use crate::layered_repository::filename::DeltaFileName;
-use crate::layered_repository::storage_layer::{
-    Layer, PageReconstructData, PageReconstructResult, PageVersion, SegmentTag, RELISH_SEG_SIZE,
-};
-use crate::layered_repository::LayeredTimeline;
-use crate::layered_repository::ZERO_PAGE;
-use crate::layered_repository::{DeltaLayer, ImageLayer};
-use crate::repository::WALRecord;
-use crate::PageServerConf;
-use crate::{ZTenantId, ZTimelineId};
-use anyhow::{bail, ensure, Result};
-use bytes::Bytes;
-use log::*;
-use std::path::PathBuf;
-use std::sync::{Arc, RwLock};
-use zenith_utils::vec_map::VecMap;
-
-use zenith_utils::lsn::Lsn;
-
-use super::page_versions::PageVersions;
-
-pub struct InMemoryLayer {
-    conf: &'static PageServerConf,
-    tenantid: ZTenantId,
-    timelineid: ZTimelineId,
-    seg: SegmentTag,
-
-    ///
-    /// This layer contains all the changes from 'start_lsn'. The
-    /// start is inclusive.
-    ///
-    start_lsn: Lsn,
-
-    /// LSN of the oldest page version stored in this layer
-    oldest_pending_lsn: Lsn,
-
-    /// The above fields never change. The parts that do change are in 'inner',
-    /// and protected by mutex.
-    inner: RwLock<InMemoryLayerInner>,
-
-    /// Predecessor layer might be needed?
-    incremental: bool,
-}
-
-pub struct InMemoryLayerInner {
-    /// Frozen in-memory layers have an exclusive end LSN.
-    /// Writes are only allowed when this is None
-    end_lsn: Option<Lsn>,
-
-    /// If this relation was dropped, remember when that happened.
-    /// The drop LSN is recorded in [`end_lsn`].
-    dropped: bool,
-
-    ///
-    /// All versions of all pages in the layer are are kept here.
-    /// Indexed by block number and LSN.
-    ///
-    page_versions: PageVersions,
-
-    ///
-    /// `segsizes` tracks the size of the segment at different points in time.
-    ///
-    /// For a blocky rel, there is always one entry, at the layer's start_lsn,
-    /// so that determining the size never depends on the predecessor layer. For
-    /// a non-blocky rel, 'segsizes' is not used and is always empty.
-    ///
-    segsizes: VecMap<Lsn, u32>,
-}
-
-impl InMemoryLayerInner {
-    fn assert_writeable(&self) {
-        assert!(self.end_lsn.is_none());
-    }
-
-    fn get_seg_size(&self, lsn: Lsn) -> u32 {
-        // Scan the BTreeMap backwards, starting from the given entry.
-        let slice = self.segsizes.slice_range(..=lsn);
-
-        // We make sure there is always at least one entry
-        if let Some((_entry_lsn, entry)) = slice.last() {
-            *entry
-        } else {
-            panic!("could not find seg size in in-memory layer");
-        }
-    }
-}
-
-impl Layer for InMemoryLayer {
-    // An in-memory layer doesn't really have a filename as it's not stored on disk,
-    // but we construct a filename as if it was a delta layer
-    fn filename(&self) -> PathBuf {
-        let inner = self.inner.read().unwrap();
-
-        let end_lsn;
-        if let Some(drop_lsn) = inner.end_lsn {
-            end_lsn = drop_lsn;
-        } else {
-            end_lsn = Lsn(u64::MAX);
-        }
-
-        let delta_filename = DeltaFileName {
-            seg: self.seg,
-            start_lsn: self.start_lsn,
-            end_lsn,
-            dropped: inner.dropped,
-        }
-        .to_string();
-
-        PathBuf::from(format!("inmem-{}", delta_filename))
-    }
-
-    fn get_timeline_id(&self) -> ZTimelineId {
-        self.timelineid
-    }
-
-    fn get_seg_tag(&self) -> SegmentTag {
-        self.seg
-    }
-
-    fn get_start_lsn(&self) -> Lsn {
-        self.start_lsn
-    }
-
-    fn get_end_lsn(&self) -> Lsn {
-        let inner = self.inner.read().unwrap();
-
-        if let Some(end_lsn) = inner.end_lsn {
-            end_lsn
-        } else {
-            Lsn(u64::MAX)
-        }
-    }
-
-    fn is_dropped(&self) -> bool {
-        let inner = self.inner.read().unwrap();
-        inner.dropped
-    }
-
-    /// Look up given page in the cache.
-    fn get_page_reconstruct_data(
-        &self,
-        blknum: u32,
-        lsn: Lsn,
-        reconstruct_data: &mut PageReconstructData,
-    ) -> Result<PageReconstructResult> {
-        let mut need_image = true;
-
-        assert!(self.seg.blknum_in_seg(blknum));
-
-        {
-            let inner = self.inner.read().unwrap();
-
-            // Scan the page versions backwards, starting from `lsn`.
-            let iter = inner
-                .page_versions
-                .get_block_lsn_range(blknum, ..=lsn)
-                .iter()
-                .rev();
-            for (entry_lsn, entry) in iter {
-                if let Some(img) = &entry.page_image {
-                    reconstruct_data.page_img = Some(img.clone());
-                    need_image = false;
-                    break;
-                } else if let Some(rec) = &entry.record {
-                    reconstruct_data.records.push((*entry_lsn, rec.clone()));
-                    if rec.will_init {
-                        // This WAL record initializes the page, so no need to go further back
-                        need_image = false;
-                        break;
-                    }
-                } else {
-                    // No base image, and no WAL record. Huh?
-                    bail!("no page image or WAL record for requested page");
-                }
-            }
-            // release lock on 'inner'
-        }
-
-        // If an older page image is needed to reconstruct the page, let the
-        // caller know
-        if need_image {
-            if self.incremental {
-                Ok(PageReconstructResult::Continue(Lsn(self.start_lsn.0 - 1)))
-            } else {
-                Ok(PageReconstructResult::Missing(self.start_lsn))
-            }
-        } else {
-            Ok(PageReconstructResult::Complete)
-        }
-    }
-
-    /// Get size of the relation at given LSN
-    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
-        assert!(lsn >= self.start_lsn);
-        ensure!(
-            self.seg.rel.is_blocky(),
-            "get_seg_size() called on a non-blocky rel"
-        );
-
-        let inner = self.inner.read().unwrap();
-        Ok(inner.get_seg_size(lsn))
-    }
-
-    /// Does this segment exist at given LSN?
-    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
-        let inner = self.inner.read().unwrap();
-
-        // If the segment created after requested LSN,
-        // it doesn't exist in the layer. But we shouldn't
-        // have requested it in the first place.
-        assert!(lsn >= self.start_lsn);
-
-        // Is the requested LSN after the segment was dropped?
-        if let Some(end_lsn) = inner.end_lsn {
-            if lsn >= end_lsn {
-                return Ok(false);
-            }
-        }
-
-        // Otherwise, it exists
-        Ok(true)
-    }
-
-    /// Cannot unload anything in an in-memory layer, since there's no backing
-    /// store. To release memory used by an in-memory layer, use 'freeze' to turn
-    /// it into an on-disk layer.
-    fn unload(&self) -> Result<()> {
-        Ok(())
-    }
-
-    /// Nothing to do here. When you drop the last reference to the layer, it will
-    /// be deallocated.
-    fn delete(&self) -> Result<()> {
-        Ok(())
-    }
-
-    fn is_incremental(&self) -> bool {
-        self.incremental
-    }
-
-    /// debugging function to print out the contents of the layer
-    fn dump(&self) -> Result<()> {
-        let inner = self.inner.read().unwrap();
-
-        let end_str = inner
-            .end_lsn
-            .as_ref()
-            .map(Lsn::to_string)
-            .unwrap_or_default();
-
-        println!(
-            "----- in-memory layer for tli {} seg {} {}-{} {} ----",
-            self.timelineid, self.seg, self.start_lsn, end_str, inner.dropped,
-        );
-
-        for (k, v) in inner.segsizes.as_slice() {
-            println!("segsizes {}: {}", k, v);
-        }
-
-        for (blknum, lsn, pv) in inner.page_versions.ordered_page_version_iter(None) {
-            println!(
-                "blk {} at {}: {}/{}\n",
-                blknum,
-                lsn,
-                pv.page_image.is_some(),
-                pv.record.is_some()
-            );
-        }
-
-        Ok(())
-    }
-}
-
-/// A result of an inmemory layer data being written to disk.
-pub struct LayersOnDisk {
-    pub delta_layers: Vec<DeltaLayer>,
-    pub image_layers: Vec<ImageLayer>,
-}
-
-impl LayersOnDisk {
-    pub fn is_empty(&self) -> bool {
-        self.delta_layers.is_empty() && self.image_layers.is_empty()
-    }
-}
-
-impl InMemoryLayer {
-    /// Return the oldest page version that's stored in this layer
-    pub fn get_oldest_pending_lsn(&self) -> Lsn {
-        self.oldest_pending_lsn
-    }
-
-    ///
-    /// Create a new, empty, in-memory layer
-    ///
-    pub fn create(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        seg: SegmentTag,
-        start_lsn: Lsn,
-        oldest_pending_lsn: Lsn,
-    ) -> Result<InMemoryLayer> {
-        trace!(
-            "initializing new empty InMemoryLayer for writing {} on timeline {} at {}",
-            seg,
-            timelineid,
-            start_lsn
-        );
-
-        // The segment is initially empty, so initialize 'segsizes' with 0.
-        let mut segsizes = VecMap::default();
-        if seg.rel.is_blocky() {
-            segsizes.append(start_lsn, 0).unwrap();
-        }
-
-        Ok(InMemoryLayer {
-            conf,
-            timelineid,
-            tenantid,
-            seg,
-            start_lsn,
-            oldest_pending_lsn,
-            incremental: false,
-            inner: RwLock::new(InMemoryLayerInner {
-                end_lsn: None,
-                dropped: false,
-                page_versions: PageVersions::default(),
-                segsizes,
-            }),
-        })
-    }
-
-    // Write operations
-
-    /// Remember new page version, as a WAL record over previous version
-    pub fn put_wal_record(&self, lsn: Lsn, blknum: u32, rec: WALRecord) -> u32 {
-        self.put_page_version(
-            blknum,
-            lsn,
-            PageVersion {
-                page_image: None,
-                record: Some(rec),
-            },
-        )
-    }
-
-    /// Remember new page version, as a full page image
-    pub fn put_page_image(&self, blknum: u32, lsn: Lsn, img: Bytes) -> u32 {
-        self.put_page_version(
-            blknum,
-            lsn,
-            PageVersion {
-                page_image: Some(img),
-                record: None,
-            },
-        )
-    }
-
-    /// Common subroutine of the public put_wal_record() and put_page_image() functions.
-    /// Adds the page version to the in-memory tree
-    pub fn put_page_version(&self, blknum: u32, lsn: Lsn, pv: PageVersion) -> u32 {
-        assert!(self.seg.blknum_in_seg(blknum));
-
-        trace!(
-            "put_page_version blk {} of {} at {}/{}",
-            blknum,
-            self.seg.rel,
-            self.timelineid,
-            lsn
-        );
-        let mut inner = self.inner.write().unwrap();
-
-        inner.assert_writeable();
-
-        let old = inner.page_versions.append_or_update_last(blknum, lsn, pv);
-
-        if old.is_some() {
-            // We already had an entry for this LSN. That's odd..
-            warn!(
-                "Page version of rel {} blk {} at {} already exists",
-                self.seg.rel, blknum, lsn
-            );
-        }
-
-        // Also update the relation size, if this extended the relation.
-        if self.seg.rel.is_blocky() {
-            let newsize = blknum - self.seg.segno * RELISH_SEG_SIZE + 1;
-
-            // use inner get_seg_size, since calling self.get_seg_size will try to acquire the lock,
-            // which we've just acquired above
-            let oldsize = inner.get_seg_size(lsn);
-            if newsize > oldsize {
-                trace!(
-                    "enlarging segment {} from {} to {} blocks at {}",
-                    self.seg,
-                    oldsize,
-                    newsize,
-                    lsn
-                );
-
-                // If we are extending the relation by more than one page, initialize the "gap"
-                // with zeros
-                //
-                // XXX: What if the caller initializes the gap with subsequent call with same LSN?
-                // I don't think that can happen currently, but that is highly dependent on how
-                // PostgreSQL writes its WAL records and there's no guarantee of it. If it does
-                // happen, we would hit the "page version already exists" warning above on the
-                // subsequent call to initialize the gap page.
-                let gapstart = self.seg.segno * RELISH_SEG_SIZE + oldsize;
-                for gapblknum in gapstart..blknum {
-                    let zeropv = PageVersion {
-                        page_image: Some(ZERO_PAGE.clone()),
-                        record: None,
-                    };
-                    trace!(
-                        "filling gap blk {} with zeros for write of {}",
-                        gapblknum,
-                        blknum
-                    );
-                    let old = inner
-                        .page_versions
-                        .append_or_update_last(gapblknum, lsn, zeropv);
-                    // We already had an entry for this LSN. That's odd..
-
-                    if old.is_some() {
-                        warn!(
-                            "Page version of rel {} blk {} at {} already exists",
-                            self.seg.rel, blknum, lsn
-                        );
-                    }
-                }
-
-                inner.segsizes.append_or_update_last(lsn, newsize).unwrap();
-                return newsize - oldsize;
-            }
-        }
-
-        0
-    }
-
-    /// Remember that the relation was truncated at given LSN
-    pub fn put_truncation(&self, lsn: Lsn, segsize: u32) {
-        assert!(
-            self.seg.rel.is_blocky(),
-            "put_truncation() called on a non-blocky rel"
-        );
-
-        let mut inner = self.inner.write().unwrap();
-        inner.assert_writeable();
-
-        // check that this we truncate to a smaller size than segment was before the truncation
-        let oldsize = inner.get_seg_size(lsn);
-        assert!(segsize < oldsize);
-
-        let old = inner.segsizes.append_or_update_last(lsn, segsize).unwrap();
-
-        if old.is_some() {
-            // We already had an entry for this LSN. That's odd..
-            warn!("Inserting truncation, but had an entry for the LSN already");
-        }
-    }
-
-    /// Remember that the segment was dropped at given LSN
-    pub fn drop_segment(&self, lsn: Lsn) {
-        let mut inner = self.inner.write().unwrap();
-
-        assert!(inner.end_lsn.is_none());
-        assert!(!inner.dropped);
-        inner.dropped = true;
-        assert!(self.start_lsn < lsn);
-        inner.end_lsn = Some(lsn);
-
-        trace!("dropped segment {} at {}", self.seg, lsn);
-    }
-
-    ///
-    /// Initialize a new InMemoryLayer for, by copying the state at the given
-    /// point in time from given existing layer.
-    ///
-    pub fn create_successor_layer(
-        conf: &'static PageServerConf,
-        src: Arc<dyn Layer>,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        start_lsn: Lsn,
-        oldest_pending_lsn: Lsn,
-    ) -> Result<InMemoryLayer> {
-        let seg = src.get_seg_tag();
-
-        assert!(oldest_pending_lsn.is_aligned());
-        assert!(oldest_pending_lsn >= start_lsn);
-
-        trace!(
-            "initializing new InMemoryLayer for writing {} on timeline {} at {}",
-            seg,
-            timelineid,
-            start_lsn,
-        );
-
-        // Copy the segment size at the start LSN from the predecessor layer.
-        let mut segsizes = VecMap::default();
-        if seg.rel.is_blocky() {
-            let size = src.get_seg_size(start_lsn)?;
-            segsizes.append(start_lsn, size).unwrap();
-        }
-
-        Ok(InMemoryLayer {
-            conf,
-            timelineid,
-            tenantid,
-            seg,
-            start_lsn,
-            oldest_pending_lsn,
-            incremental: true,
-            inner: RwLock::new(InMemoryLayerInner {
-                end_lsn: None,
-                dropped: false,
-                page_versions: PageVersions::default(),
-                segsizes,
-            }),
-        })
-    }
-
-    pub fn is_writeable(&self) -> bool {
-        let inner = self.inner.read().unwrap();
-        inner.end_lsn.is_none()
-    }
-
-    /// Make the layer non-writeable. Only call once.
-    /// Records the end_lsn for non-dropped layers.
-    /// `end_lsn` is inclusive
-    pub fn freeze(&self, end_lsn: Lsn) {
-        let mut inner = self.inner.write().unwrap();
-
-        if inner.end_lsn.is_some() {
-            assert!(inner.dropped);
-        } else {
-            assert!(!inner.dropped);
-            assert!(self.start_lsn < end_lsn + 1);
-            inner.end_lsn = Some(Lsn(end_lsn.0 + 1));
-
-            if let Some((lsn, _)) = inner.segsizes.as_slice().last() {
-                assert!(lsn <= &end_lsn, "{:?} {:?}", lsn, end_lsn);
-            }
-
-            for (_blk, lsn, _pv) in inner.page_versions.ordered_page_version_iter(None) {
-                assert!(lsn <= end_lsn);
-            }
-        }
-    }
-
-    /// Write the this frozen in-memory layer to disk.
-    ///
-    /// Returns new layers that replace this one.
-    /// If not dropped, returns a new image layer containing the page versions
-    /// at the `end_lsn`. Can also return a DeltaLayer that includes all the
-    /// WAL records between start and end LSN. (The delta layer is not needed
-    /// when a new relish is created with a single LSN, so that the start and
-    /// end LSN are the same.)
-    pub fn write_to_disk(&self, timeline: &LayeredTimeline) -> Result<LayersOnDisk> {
-        trace!(
-            "write_to_disk {} get_end_lsn is {}",
-            self.filename().display(),
-            self.get_end_lsn()
-        );
-
-        // Grab the lock in read-mode. We hold it over the I/O, but because this
-        // layer is not writeable anymore, no one should be trying to acquire the
-        // write lock on it, so we shouldn't block anyone. There's one exception
-        // though: another thread might have grabbed a reference to this layer
-        // in `get_layer_for_write' just before the checkpointer called
-        // `freeze`, and then `write_to_disk` on it. When the thread gets the
-        // lock, it will see that it's not writeable anymore and retry, but it
-        // would have to wait until we release it. That race condition is very
-        // rare though, so we just accept the potential latency hit for now.
-        let inner = self.inner.read().unwrap();
-        let end_lsn_exclusive = inner.end_lsn.unwrap();
-
-        if inner.dropped {
-            let delta_layer = DeltaLayer::create(
-                self.conf,
-                self.timelineid,
-                self.tenantid,
-                self.seg,
-                self.start_lsn,
-                end_lsn_exclusive,
-                true,
-                inner.page_versions.ordered_page_version_iter(None),
-                inner.segsizes.clone(),
-            )?;
-            trace!(
-                "freeze: created delta layer for dropped segment {} {}-{}",
-                self.seg,
-                self.start_lsn,
-                end_lsn_exclusive
-            );
-            return Ok(LayersOnDisk {
-                delta_layers: vec![delta_layer],
-                image_layers: Vec::new(),
-            });
-        }
-
-        // Since `end_lsn` is inclusive, subtract 1.
-        // We want to make an ImageLayer for the last included LSN,
-        // so the DeltaLayer should exlcude that LSN.
-        let end_lsn_inclusive = Lsn(end_lsn_exclusive.0 - 1);
-
-        let mut page_versions = inner
-            .page_versions
-            .ordered_page_version_iter(Some(end_lsn_inclusive));
-
-        let mut delta_layers = Vec::new();
-
-        if self.start_lsn != end_lsn_inclusive {
-            let (segsizes, _) = inner.segsizes.split_at(&end_lsn_exclusive);
-            // Write the page versions before the cutoff to disk.
-            let delta_layer = DeltaLayer::create(
-                self.conf,
-                self.timelineid,
-                self.tenantid,
-                self.seg,
-                self.start_lsn,
-                end_lsn_inclusive,
-                false,
-                page_versions,
-                segsizes,
-            )?;
-            delta_layers.push(delta_layer);
-            trace!(
-                "freeze: created delta layer {} {}-{}",
-                self.seg,
-                self.start_lsn,
-                end_lsn_inclusive
-            );
-        } else {
-            assert!(page_versions.next().is_none());
-        }
-
-        drop(inner);
-
-        // Write a new base image layer at the cutoff point
-        let image_layer =
-            ImageLayer::create_from_src(self.conf, timeline, self, end_lsn_inclusive)?;
-        trace!(
-            "freeze: created image layer {} at {}",
-            self.seg,
-            end_lsn_inclusive
-        );
-
-        Ok(LayersOnDisk {
-            delta_layers,
-            image_layers: vec![image_layer],
-        })
-    }
-}

pageserver/src/layered_repository/interval_tree.rs

@@ -1,468 +0,0 @@
-///
-/// IntervalTree is data structure for holding intervals. It is generic
-/// to make unit testing possible, but the only real user of it is the layer map,
-///
-/// It's inspired by the "segment tree" or a "statistic tree" as described in
-/// https://en.wikipedia.org/wiki/Segment_tree. However, we use a B-tree to hold
-/// the points instead of a binary tree. This is called an "interval tree" instead
-/// of "segment tree" because the term "segment" is already using Zenith to mean
-/// something else. To add to the confusion, there is another data structure known
-/// as "interval tree" out there (see https://en.wikipedia.org/wiki/Interval_tree),
-/// for storing intervals, but this isn't that.
-///
-/// The basic idea is to have a B-tree of "interesting Points". At each Point,
-/// there is a list of intervals that contain the point. The Points are formed
-/// from the start bounds of each interval; there is a Point for each distinct
-/// start bound.
-///
-/// Operations:
-///
-/// To find intervals that contain a given point, you search the b-tree to find
-/// the nearest Point <= search key. Then you just return the list of intervals.
-///
-/// To insert an interval, find the Point with start key equal to the inserted item.
-/// If the Point doesn't exist yet, create it, by copying all the items from the
-/// previous Point that cover the new Point. Then walk right, inserting the new
-/// interval to all the Points that are contained by the new interval (including the
-/// newly created Point).
-///
-/// To remove an interval, you scan the tree for all the Points that are contained by
-/// the removed interval, and remove it from the list in each Point.
-///
-/// Requirements and assumptions:
-///
-/// - Can store overlapping items
-/// - But there are not many overlapping items
-/// - The interval bounds don't change after it is added to the tree
-/// - Intervals are uniquely identified by pointer equality. You must not be insert the
-///   same interval object twice, and `remove` uses pointer equality to remove the right
-///   interval. It is OK to have two intervals with the same bounds, however.
-///
-use std::collections::BTreeMap;
-use std::fmt::Debug;
-use std::ops::Range;
-use std::sync::Arc;
-
-pub struct IntervalTree<I: ?Sized>
-where
-    I: IntervalItem,
-{
-    points: BTreeMap<I::Key, Point<I>>,
-}
-
-struct Point<I: ?Sized> {
-    /// All intervals that contain this point, in no particular order.
-    ///
-    /// We assume that there aren't a lot of overlappingg intervals, so that this vector
-    /// never grows very large. If that assumption doesn't hold, we could keep this ordered
-    /// by the end bound, to speed up `search`. But as long as there are only a few elements,
-    /// a linear search is OK.
-    elements: Vec<Arc<I>>,
-}
-
-/// Abstraction for an interval that can be stored in the tree
-///
-/// The start bound is inclusive and the end bound is exclusive. End must be greater
-/// than start.
-pub trait IntervalItem {
-    type Key: Ord + Copy + Debug + Sized;
-
-    fn start_key(&self) -> Self::Key;
-    fn end_key(&self) -> Self::Key;
-
-    fn bounds(&self) -> Range<Self::Key> {
-        self.start_key()..self.end_key()
-    }
-}
-
-impl<I: ?Sized> IntervalTree<I>
-where
-    I: IntervalItem,
-{
-    /// Return an element that contains 'key', or precedes it.
-    ///
-    /// If there are multiple candidates, returns the one with the highest 'end' key.
-    pub fn search(&self, key: I::Key) -> Option<Arc<I>> {
-        // Find the greatest point that precedes or is equal to the search key. If there is
-        // none, returns None.
-        let (_, p) = self.points.range(..=key).next_back()?;
-
-        // Find the element with the highest end key at this point
-        let highest_item = p
-            .elements
-            .iter()
-            .reduce(|a, b| {
-                // starting with Rust 1.53, could use `std::cmp::min_by_key` here
-                if a.end_key() > b.end_key() {
-                    a
-                } else {
-                    b
-                }
-            })
-            .unwrap();
-        Some(Arc::clone(highest_item))
-    }
-
-    /// Iterate over all items with start bound >= 'key'
-    pub fn iter_newer(&self, key: I::Key) -> IntervalIter<I> {
-        IntervalIter {
-            point_iter: self.points.range(key..),
-            elem_iter: None,
-        }
-    }
-
-    /// Iterate over all items
-    pub fn iter(&self) -> IntervalIter<I> {
-        IntervalIter {
-            point_iter: self.points.range(..),
-            elem_iter: None,
-        }
-    }
-
-    pub fn insert(&mut self, item: Arc<I>) {
-        let start_key = item.start_key();
-        let end_key = item.end_key();
-        assert!(start_key < end_key);
-        let bounds = start_key..end_key;
-
-        // Find the starting point and walk forward from there
-        let mut found_start_point = false;
-        let iter = self.points.range_mut(bounds);
-        for (point_key, point) in iter {
-            if *point_key == start_key {
-                found_start_point = true;
-                // It is an error to insert the same item to the tree twice.
-                assert!(
-                    !point.elements.iter().any(|x| Arc::ptr_eq(x, &item)),
-                    "interval is already in the tree"
-                );
-            }
-            point.elements.push(Arc::clone(&item));
-        }
-        if !found_start_point {
-            // Create a new Point for the starting point
-
-            // Look at the previous point, and copy over elements that overlap with this
-            // new point
-            let mut new_elements: Vec<Arc<I>> = Vec::new();
-            if let Some((_, prev_point)) = self.points.range(..start_key).next_back() {
-                let overlapping_prev_elements = prev_point
-                    .elements
-                    .iter()
-                    .filter(|x| x.bounds().contains(&start_key))
-                    .cloned();
-
-                new_elements.extend(overlapping_prev_elements);
-            }
-            new_elements.push(item);
-
-            let new_point = Point {
-                elements: new_elements,
-            };
-            self.points.insert(start_key, new_point);
-        }
-    }
-
-    pub fn remove(&mut self, item: &Arc<I>) {
-        // range search points
-        let start_key = item.start_key();
-        let end_key = item.end_key();
-        let bounds = start_key..end_key;
-
-        let mut points_to_remove: Vec<I::Key> = Vec::new();
-        let mut found_start_point = false;
-        for (point_key, point) in self.points.range_mut(bounds) {
-            if *point_key == start_key {
-                found_start_point = true;
-            }
-            let len_before = point.elements.len();
-            point.elements.retain(|other| !Arc::ptr_eq(other, item));
-            let len_after = point.elements.len();
-            assert_eq!(len_after + 1, len_before);
-            if len_after == 0 {
-                points_to_remove.push(*point_key);
-            }
-        }
-        assert!(found_start_point);
-
-        for k in points_to_remove {
-            self.points.remove(&k).unwrap();
-        }
-    }
-}
-
-pub struct IntervalIter<'a, I: ?Sized>
-where
-    I: IntervalItem,
-{
-    point_iter: std::collections::btree_map::Range<'a, I::Key, Point<I>>,
-    elem_iter: Option<(I::Key, std::slice::Iter<'a, Arc<I>>)>,
-}
-
-impl<'a, I> Iterator for IntervalIter<'a, I>
-where
-    I: IntervalItem + ?Sized,
-{
-    type Item = Arc<I>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        // Iterate over all elements in all the points in 'point_iter'. To avoid
-        // returning the same element twice, we only return each element at its
-        // starting point.
-        loop {
-            // Return next remaining element from the current point
-            if let Some((point_key, elem_iter)) = &mut self.elem_iter {
-                for elem in elem_iter {
-                    if elem.start_key() == *point_key {
-                        return Some(Arc::clone(elem));
-                    }
-                }
-            }
-            // No more elements at this point. Move to next point.
-            if let Some((point_key, point)) = self.point_iter.next() {
-                self.elem_iter = Some((*point_key, point.elements.iter()));
-                continue;
-            } else {
-                // No more points, all done
-                return None;
-            }
-        }
-    }
-}
-
-impl<I: ?Sized> Default for IntervalTree<I>
-where
-    I: IntervalItem,
-{
-    fn default() -> Self {
-        IntervalTree {
-            points: BTreeMap::new(),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::fmt;
-
-    #[derive(Debug)]
-    struct MockItem {
-        start_key: u32,
-        end_key: u32,
-        val: String,
-    }
-    impl IntervalItem for MockItem {
-        type Key = u32;
-
-        fn start_key(&self) -> u32 {
-            self.start_key
-        }
-        fn end_key(&self) -> u32 {
-            self.end_key
-        }
-    }
-    impl MockItem {
-        fn new(start_key: u32, end_key: u32) -> Self {
-            MockItem {
-                start_key,
-                end_key,
-                val: format!("{}-{}", start_key, end_key),
-            }
-        }
-        fn new_str(start_key: u32, end_key: u32, val: &str) -> Self {
-            MockItem {
-                start_key,
-                end_key,
-                val: format!("{}-{}: {}", start_key, end_key, val),
-            }
-        }
-    }
-    impl fmt::Display for MockItem {
-        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-            write!(f, "{}", self.val)
-        }
-    }
-    #[rustfmt::skip]
-    fn assert_search(
-        tree: &IntervalTree<MockItem>,
-        key: u32,
-        expected: &[&str],
-    ) -> Option<Arc<MockItem>> {
-        if let Some(v) = tree.search(key) {
-            let vstr = v.to_string();
-
-            assert!(!expected.is_empty(), "search with {} returned {}, expected None", key, v);
-            assert!(
-                expected.contains(&vstr.as_str()),
-                "search with {} returned {}, expected one of: {:?}",
-                key, v, expected,
-            );
-
-            Some(v)
-        } else {
-            assert!(
-                expected.is_empty(),
-                "search with {} returned None, expected one of {:?}",
-                key, expected
-            );
-            None
-        }
-    }
-
-    fn assert_contents(tree: &IntervalTree<MockItem>, expected: &[&str]) {
-        let mut contents: Vec<String> = tree.iter().map(|e| e.to_string()).collect();
-        contents.sort();
-        assert_eq!(contents, expected);
-    }
-
-    fn dump_tree(tree: &IntervalTree<MockItem>) {
-        for (point_key, point) in tree.points.iter() {
-            print!("{}:", point_key);
-            for e in point.elements.iter() {
-                print!(" {}", e);
-            }
-            println!();
-        }
-    }
-
-    #[test]
-    fn test_interval_tree_simple() {
-        let mut tree: IntervalTree<MockItem> = IntervalTree::default();
-
-        // Simple, non-overlapping ranges.
-        tree.insert(Arc::new(MockItem::new(10, 11)));
-        tree.insert(Arc::new(MockItem::new(11, 12)));
-        tree.insert(Arc::new(MockItem::new(12, 13)));
-        tree.insert(Arc::new(MockItem::new(18, 19)));
-        tree.insert(Arc::new(MockItem::new(17, 18)));
-        tree.insert(Arc::new(MockItem::new(15, 16)));
-
-        assert_search(&tree, 9, &[]);
-        assert_search(&tree, 10, &["10-11"]);
-        assert_search(&tree, 11, &["11-12"]);
-        assert_search(&tree, 12, &["12-13"]);
-        assert_search(&tree, 13, &["12-13"]);
-        assert_search(&tree, 14, &["12-13"]);
-        assert_search(&tree, 15, &["15-16"]);
-        assert_search(&tree, 16, &["15-16"]);
-        assert_search(&tree, 17, &["17-18"]);
-        assert_search(&tree, 18, &["18-19"]);
-        assert_search(&tree, 19, &["18-19"]);
-        assert_search(&tree, 20, &["18-19"]);
-
-        // remove a few entries and search around them again
-        tree.remove(&assert_search(&tree, 10, &["10-11"]).unwrap()); // first entry
-        tree.remove(&assert_search(&tree, 12, &["12-13"]).unwrap()); // entry in the middle
-        tree.remove(&assert_search(&tree, 18, &["18-19"]).unwrap()); // last entry
-        assert_search(&tree, 9, &[]);
-        assert_search(&tree, 10, &[]);
-        assert_search(&tree, 11, &["11-12"]);
-        assert_search(&tree, 12, &["11-12"]);
-        assert_search(&tree, 14, &["11-12"]);
-        assert_search(&tree, 15, &["15-16"]);
-        assert_search(&tree, 17, &["17-18"]);
-        assert_search(&tree, 18, &["17-18"]);
-    }
-
-    #[test]
-    fn test_interval_tree_overlap() {
-        let mut tree: IntervalTree<MockItem> = IntervalTree::default();
-
-        // Overlapping items
-        tree.insert(Arc::new(MockItem::new(22, 24)));
-        tree.insert(Arc::new(MockItem::new(23, 25)));
-        let x24_26 = Arc::new(MockItem::new(24, 26));
-        tree.insert(Arc::clone(&x24_26));
-        let x26_28 = Arc::new(MockItem::new(26, 28));
-        tree.insert(Arc::clone(&x26_28));
-        tree.insert(Arc::new(MockItem::new(25, 27)));
-
-        assert_search(&tree, 22, &["22-24"]);
-        assert_search(&tree, 23, &["22-24", "23-25"]);
-        assert_search(&tree, 24, &["23-25", "24-26"]);
-        assert_search(&tree, 25, &["24-26", "25-27"]);
-        assert_search(&tree, 26, &["25-27", "26-28"]);
-        assert_search(&tree, 27, &["26-28"]);
-        assert_search(&tree, 28, &["26-28"]);
-        assert_search(&tree, 29, &["26-28"]);
-
-        tree.remove(&x24_26);
-        tree.remove(&x26_28);
-        assert_search(&tree, 23, &["22-24", "23-25"]);
-        assert_search(&tree, 24, &["23-25"]);
-        assert_search(&tree, 25, &["25-27"]);
-        assert_search(&tree, 26, &["25-27"]);
-        assert_search(&tree, 27, &["25-27"]);
-        assert_search(&tree, 28, &["25-27"]);
-        assert_search(&tree, 29, &["25-27"]);
-    }
-
-    #[test]
-    fn test_interval_tree_nested() {
-        let mut tree: IntervalTree<MockItem> = IntervalTree::default();
-
-        // Items containing other items
-        tree.insert(Arc::new(MockItem::new(31, 39)));
-        tree.insert(Arc::new(MockItem::new(32, 34)));
-        tree.insert(Arc::new(MockItem::new(33, 35)));
-        tree.insert(Arc::new(MockItem::new(30, 40)));
-
-        assert_search(&tree, 30, &["30-40"]);
-        assert_search(&tree, 31, &["30-40", "31-39"]);
-        assert_search(&tree, 32, &["30-40", "32-34", "31-39"]);
-        assert_search(&tree, 33, &["30-40", "32-34", "33-35", "31-39"]);
-        assert_search(&tree, 34, &["30-40", "33-35", "31-39"]);
-        assert_search(&tree, 35, &["30-40", "31-39"]);
-        assert_search(&tree, 36, &["30-40", "31-39"]);
-        assert_search(&tree, 37, &["30-40", "31-39"]);
-        assert_search(&tree, 38, &["30-40", "31-39"]);
-        assert_search(&tree, 39, &["30-40"]);
-        assert_search(&tree, 40, &["30-40"]);
-        assert_search(&tree, 41, &["30-40"]);
-    }
-
-    #[test]
-    fn test_interval_tree_duplicates() {
-        let mut tree: IntervalTree<MockItem> = IntervalTree::default();
-
-        // Duplicate keys
-        let item_a = Arc::new(MockItem::new_str(55, 56, "a"));
-        tree.insert(Arc::clone(&item_a));
-        let item_b = Arc::new(MockItem::new_str(55, 56, "b"));
-        tree.insert(Arc::clone(&item_b));
-        let item_c = Arc::new(MockItem::new_str(55, 56, "c"));
-        tree.insert(Arc::clone(&item_c));
-        let item_d = Arc::new(MockItem::new_str(54, 56, "d"));
-        tree.insert(Arc::clone(&item_d));
-        let item_e = Arc::new(MockItem::new_str(55, 57, "e"));
-        tree.insert(Arc::clone(&item_e));
-
-        dump_tree(&tree);
-
-        assert_search(
-            &tree,
-            55,
-            &["55-56: a", "55-56: b", "55-56: c", "54-56: d", "55-57: e"],
-        );
-        tree.remove(&item_b);
-        dump_tree(&tree);
-
-        assert_contents(&tree, &["54-56: d", "55-56: a", "55-56: c", "55-57: e"]);
-
-        tree.remove(&item_d);
-        dump_tree(&tree);
-        assert_contents(&tree, &["55-56: a", "55-56: c", "55-57: e"]);
-    }
-
-    #[test]
-    #[should_panic]
-    fn test_interval_tree_insert_twice() {
-        let mut tree: IntervalTree<MockItem> = IntervalTree::default();
-
-        // Inserting the same item twice is not cool
-        let item = Arc::new(MockItem::new(1, 2));
-        tree.insert(Arc::clone(&item));
-        tree.insert(Arc::clone(&item)); // fails assertion
-    }
-}

pageserver/src/layered_repository/layer_map.rs

@@ -1,444 +0,0 @@
-//!
-//! The layer map tracks what layers exist for all the relishes in a timeline.
-//!
-//! When the timeline is first accessed, the server lists of all layer files
-//! in the timelines/<timelineid> directory, and populates this map with
-//! ImageLayer and DeltaLayer structs corresponding to each file. When new WAL
-//! is received, we create InMemoryLayers to hold the incoming records. Now and
-//! then, in the checkpoint() function, the in-memory layers are frozen, forming
-//! new image and delta layers and corresponding files are written to disk.
-//!
-
-use crate::layered_repository::interval_tree::{IntervalItem, IntervalIter, IntervalTree};
-use crate::layered_repository::storage_layer::{Layer, SegmentTag};
-use crate::layered_repository::InMemoryLayer;
-use crate::relish::*;
-use anyhow::Result;
-use lazy_static::lazy_static;
-use std::cmp::Ordering;
-use std::collections::{BinaryHeap, HashMap};
-use std::sync::Arc;
-use zenith_metrics::{register_int_gauge, IntGauge};
-use zenith_utils::lsn::Lsn;
-
-lazy_static! {
-    static ref NUM_INMEMORY_LAYERS: IntGauge =
-        register_int_gauge!("pageserver_inmemory_layers", "Number of layers in memory")
-            .expect("failed to define a metric");
-    static ref NUM_ONDISK_LAYERS: IntGauge =
-        register_int_gauge!("pageserver_ondisk_layers", "Number of layers on-disk")
-            .expect("failed to define a metric");
-}
-
-///
-/// LayerMap tracks what layers exist on a timeline.
-///
-#[derive(Default)]
-pub struct LayerMap {
-    /// All the layers keyed by segment tag
-    segs: HashMap<SegmentTag, SegEntry>,
-
-    /// All in-memory layers, ordered by 'oldest_pending_lsn' and generation
-    /// of each layer. This allows easy access to the in-memory layer that
-    /// contains the oldest WAL record.
-    open_layers: BinaryHeap<OpenLayerEntry>,
-
-    /// Generation number, used to distinguish newly inserted entries in the
-    /// binary heap from older entries during checkpoint.
-    current_generation: u64,
-}
-
-impl LayerMap {
-    ///
-    /// Look up a layer using the given segment tag and LSN. This differs from a
-    /// plain key-value lookup in that if there is any layer that covers the
-    /// given LSN, or precedes the given LSN, it is returned. In other words,
-    /// you don't need to know the exact start LSN of the layer.
-    ///
-    pub fn get(&self, tag: &SegmentTag, lsn: Lsn) -> Option<Arc<dyn Layer>> {
-        let segentry = self.segs.get(tag)?;
-
-        segentry.get(lsn)
-    }
-
-    ///
-    /// Get the open layer for given segment for writing. Or None if no open
-    /// layer exists.
-    ///
-    pub fn get_open(&self, tag: &SegmentTag) -> Option<Arc<InMemoryLayer>> {
-        let segentry = self.segs.get(tag)?;
-
-        segentry.open.as_ref().map(Arc::clone)
-    }
-
-    ///
-    /// Insert an open in-memory layer
-    ///
-    pub fn insert_open(&mut self, layer: Arc<InMemoryLayer>) {
-        let segentry = self.segs.entry(layer.get_seg_tag()).or_default();
-
-        segentry.update_open(Arc::clone(&layer));
-
-        let oldest_pending_lsn = layer.get_oldest_pending_lsn();
-
-        // After a crash and restart, 'oldest_pending_lsn' of the oldest in-memory
-        // layer becomes the WAL streaming starting point, so it better not point
-        // in the middle of a WAL record.
-        assert!(oldest_pending_lsn.is_aligned());
-
-        // Also add it to the binary heap
-        let open_layer_entry = OpenLayerEntry {
-            oldest_pending_lsn: layer.get_oldest_pending_lsn(),
-            layer,
-            generation: self.current_generation,
-        };
-        self.open_layers.push(open_layer_entry);
-
-        NUM_INMEMORY_LAYERS.inc();
-    }
-
-    /// Remove the oldest in-memory layer
-    pub fn pop_oldest_open(&mut self) {
-        // Pop it from the binary heap
-        let oldest_entry = self.open_layers.pop().unwrap();
-        let segtag = oldest_entry.layer.get_seg_tag();
-
-        // Also remove it from the SegEntry of this segment
-        let mut segentry = self.segs.get_mut(&segtag).unwrap();
-        if Arc::ptr_eq(segentry.open.as_ref().unwrap(), &oldest_entry.layer) {
-            segentry.open = None;
-        } else {
-            // We could have already updated segentry.open for
-            // dropped (non-writeable) layer. This is fine.
-            assert!(!oldest_entry.layer.is_writeable());
-            assert!(oldest_entry.layer.is_dropped());
-        }
-
-        NUM_INMEMORY_LAYERS.dec();
-    }
-
-    ///
-    /// Insert an on-disk layer
-    ///
-    pub fn insert_historic(&mut self, layer: Arc<dyn Layer>) {
-        let segentry = self.segs.entry(layer.get_seg_tag()).or_default();
-        segentry.insert_historic(layer);
-
-        NUM_ONDISK_LAYERS.inc();
-    }
-
-    ///
-    /// Remove an on-disk layer from the map.
-    ///
-    /// This should be called when the corresponding file on disk has been deleted.
-    ///
-    pub fn remove_historic(&mut self, layer: Arc<dyn Layer>) {
-        let tag = layer.get_seg_tag();
-
-        if let Some(segentry) = self.segs.get_mut(&tag) {
-            segentry.historic.remove(&layer);
-        }
-        NUM_ONDISK_LAYERS.dec();
-    }
-
-    // List relations along with a flag that marks if they exist at the given lsn.
-    // spcnode 0 and dbnode 0 have special meanings and mean all tabespaces/databases.
-    // Pass Tag if we're only interested in some relations.
-    pub fn list_relishes(&self, tag: Option<RelTag>, lsn: Lsn) -> Result<HashMap<RelishTag, bool>> {
-        let mut rels: HashMap<RelishTag, bool> = HashMap::new();
-
-        for (seg, segentry) in self.segs.iter() {
-            match seg.rel {
-                RelishTag::Relation(reltag) => {
-                    if let Some(request_rel) = tag {
-                        if (request_rel.spcnode == 0 || reltag.spcnode == request_rel.spcnode)
-                            && (request_rel.dbnode == 0 || reltag.dbnode == request_rel.dbnode)
-                        {
-                            if let Some(exists) = segentry.exists_at_lsn(lsn)? {
-                                rels.insert(seg.rel, exists);
-                            }
-                        }
-                    }
-                }
-                _ => {
-                    if tag == None {
-                        if let Some(exists) = segentry.exists_at_lsn(lsn)? {
-                            rels.insert(seg.rel, exists);
-                        }
-                    }
-                }
-            }
-        }
-        Ok(rels)
-    }
-
-    /// Is there a newer image layer for given segment?
-    ///
-    /// This is used for garbage collection, to determine if an old layer can
-    /// be deleted.
-    pub fn newer_image_layer_exists(&self, seg: SegmentTag, lsn: Lsn) -> bool {
-        if let Some(segentry) = self.segs.get(&seg) {
-            segentry.newer_image_layer_exists(lsn)
-        } else {
-            false
-        }
-    }
-
-    /// Is there any layer for given segment that is alive at the lsn?
-    ///
-    /// This is a public wrapper for SegEntry fucntion,
-    /// used for garbage collection, to determine if some alive layer
-    /// exists at the lsn. If so, we shouldn't delete a newer dropped layer
-    /// to avoid incorrectly making it visible.
-    pub fn layer_exists_at_lsn(&self, seg: SegmentTag, lsn: Lsn) -> Result<bool> {
-        Ok(if let Some(segentry) = self.segs.get(&seg) {
-            segentry.exists_at_lsn(lsn)?.unwrap_or(false)
-        } else {
-            false
-        })
-    }
-
-    /// Return the oldest in-memory layer, along with its generation number.
-    pub fn peek_oldest_open(&self) -> Option<(Arc<InMemoryLayer>, u64)> {
-        self.open_layers
-            .peek()
-            .map(|oldest_entry| (Arc::clone(&oldest_entry.layer), oldest_entry.generation))
-    }
-
-    /// Increment the generation number used to stamp open in-memory layers. Layers
-    /// added with `insert_open` after this call will be associated with the new
-    /// generation. Returns the new generation number.
-    pub fn increment_generation(&mut self) -> u64 {
-        self.current_generation += 1;
-        self.current_generation
-    }
-
-    pub fn iter_historic_layers(&self) -> HistoricLayerIter {
-        HistoricLayerIter {
-            seg_iter: self.segs.iter(),
-            iter: None,
-        }
-    }
-
-    /// debugging function to print out the contents of the layer map
-    #[allow(unused)]
-    pub fn dump(&self) -> Result<()> {
-        println!("Begin dump LayerMap");
-        for (seg, segentry) in self.segs.iter() {
-            if let Some(open) = &segentry.open {
-                open.dump()?;
-            }
-
-            for layer in segentry.historic.iter() {
-                layer.dump()?;
-            }
-        }
-        println!("End dump LayerMap");
-        Ok(())
-    }
-}
-
-impl IntervalItem for dyn Layer {
-    type Key = Lsn;
-
-    fn start_key(&self) -> Lsn {
-        self.get_start_lsn()
-    }
-    fn end_key(&self) -> Lsn {
-        self.get_end_lsn()
-    }
-}
-
-///
-/// Per-segment entry in the LayerMap::segs hash map. Holds all the layers
-/// associated with the segment.
-///
-/// The last layer that is open for writes is always an InMemoryLayer,
-/// and is kept in a separate field, because there can be only one for
-/// each segment. The older layers, stored on disk, are kept in an
-/// IntervalTree.
-#[derive(Default)]
-struct SegEntry {
-    open: Option<Arc<InMemoryLayer>>,
-    historic: IntervalTree<dyn Layer>,
-}
-
-impl SegEntry {
-    /// Does the segment exist at given LSN?
-    /// Return None if object is not found in this SegEntry.
-    fn exists_at_lsn(&self, lsn: Lsn) -> Result<Option<bool>> {
-        if let Some(layer) = self.get(lsn) {
-            Ok(Some(layer.get_seg_exists(lsn)?))
-        } else {
-            Ok(None)
-        }
-    }
-
-    pub fn get(&self, lsn: Lsn) -> Option<Arc<dyn Layer>> {
-        if let Some(open) = &self.open {
-            if open.get_start_lsn() <= lsn {
-                let x: Arc<dyn Layer> = Arc::clone(open) as _;
-                return Some(x);
-            }
-        }
-
-        self.historic.search(lsn)
-    }
-
-    pub fn newer_image_layer_exists(&self, lsn: Lsn) -> bool {
-        // We only check on-disk layers, because
-        // in-memory layers are not durable
-
-        self.historic
-            .iter_newer(lsn)
-            .any(|layer| !layer.is_incremental())
-    }
-
-    // Set new open layer for a SegEntry.
-    // It's ok to rewrite previous open layer,
-    // but only if it is not writeable anymore.
-    pub fn update_open(&mut self, layer: Arc<InMemoryLayer>) {
-        if let Some(prev_open) = &self.open {
-            assert!(!prev_open.is_writeable());
-        }
-        self.open = Some(layer);
-    }
-
-    pub fn insert_historic(&mut self, layer: Arc<dyn Layer>) {
-        self.historic.insert(layer);
-    }
-}
-
-/// Entry held in LayerMap::open_layers, with boilerplate comparison routines
-/// to implement a min-heap ordered by 'oldest_pending_lsn' and 'generation'
-///
-/// The generation number associated with each entry can be used to distinguish
-/// recently-added entries (i.e after last call to increment_generation()) from older
-/// entries with the same 'oldest_pending_lsn'.
-struct OpenLayerEntry {
-    pub oldest_pending_lsn: Lsn, // copy of layer.get_oldest_pending_lsn()
-    pub generation: u64,
-    pub layer: Arc<InMemoryLayer>,
-}
-impl Ord for OpenLayerEntry {
-    fn cmp(&self, other: &Self) -> Ordering {
-        // BinaryHeap is a max-heap, and we want a min-heap. Reverse the ordering here
-        // to get that. Entries with identical oldest_pending_lsn are ordered by generation
-        other
-            .oldest_pending_lsn
-            .cmp(&self.oldest_pending_lsn)
-            .then_with(|| other.generation.cmp(&self.generation))
-    }
-}
-impl PartialOrd for OpenLayerEntry {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-impl PartialEq for OpenLayerEntry {
-    fn eq(&self, other: &Self) -> bool {
-        self.cmp(other) == Ordering::Equal
-    }
-}
-impl Eq for OpenLayerEntry {}
-
-/// Iterator returned by LayerMap::iter_historic_layers()
-pub struct HistoricLayerIter<'a> {
-    seg_iter: std::collections::hash_map::Iter<'a, SegmentTag, SegEntry>,
-    iter: Option<IntervalIter<'a, dyn Layer>>,
-}
-
-impl<'a> Iterator for HistoricLayerIter<'a> {
-    type Item = Arc<dyn Layer>;
-
-    fn next(&mut self) -> std::option::Option<<Self as std::iter::Iterator>::Item> {
-        loop {
-            if let Some(x) = &mut self.iter {
-                if let Some(x) = x.next() {
-                    return Some(Arc::clone(&x));
-                }
-            }
-            if let Some((_tag, segentry)) = self.seg_iter.next() {
-                self.iter = Some(segentry.historic.iter());
-                continue;
-            } else {
-                return None;
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::PageServerConf;
-    use std::str::FromStr;
-    use zenith_utils::zid::{ZTenantId, ZTimelineId};
-
-    /// Arbitrary relation tag, for testing.
-    const TESTREL_A: RelishTag = RelishTag::Relation(RelTag {
-        spcnode: 0,
-        dbnode: 111,
-        relnode: 1000,
-        forknum: 0,
-    });
-
-    /// Construct a dummy InMemoryLayer for testing
-    fn dummy_inmem_layer(
-        conf: &'static PageServerConf,
-        segno: u32,
-        start_lsn: Lsn,
-        oldest_pending_lsn: Lsn,
-    ) -> Arc<InMemoryLayer> {
-        Arc::new(
-            InMemoryLayer::create(
-                conf,
-                ZTimelineId::from_str("00000000000000000000000000000000").unwrap(),
-                ZTenantId::from_str("00000000000000000000000000000000").unwrap(),
-                SegmentTag {
-                    rel: TESTREL_A,
-                    segno,
-                },
-                start_lsn,
-                oldest_pending_lsn,
-            )
-            .unwrap(),
-        )
-    }
-
-    #[test]
-    fn test_open_layers() -> Result<()> {
-        let conf = PageServerConf::dummy_conf(PageServerConf::test_repo_dir("dummy_inmem_layer"));
-        let conf = Box::leak(Box::new(conf));
-
-        let mut layers = LayerMap::default();
-
-        let gen1 = layers.increment_generation();
-        layers.insert_open(dummy_inmem_layer(conf, 0, Lsn(0x100), Lsn(0x100)));
-        layers.insert_open(dummy_inmem_layer(conf, 1, Lsn(0x100), Lsn(0x200)));
-        layers.insert_open(dummy_inmem_layer(conf, 2, Lsn(0x100), Lsn(0x120)));
-        layers.insert_open(dummy_inmem_layer(conf, 3, Lsn(0x100), Lsn(0x110)));
-
-        let gen2 = layers.increment_generation();
-        layers.insert_open(dummy_inmem_layer(conf, 4, Lsn(0x100), Lsn(0x110)));
-        layers.insert_open(dummy_inmem_layer(conf, 5, Lsn(0x100), Lsn(0x100)));
-
-        // A helper function (closure) to pop the next oldest open entry from the layer map,
-        // and assert that it is what we'd expect
-        let mut assert_pop_layer = |expected_segno: u32, expected_generation: u64| {
-            let (l, generation) = layers.peek_oldest_open().unwrap();
-            assert!(l.get_seg_tag().segno == expected_segno);
-            assert!(generation == expected_generation);
-            layers.pop_oldest_open();
-        };
-
-        assert_pop_layer(0, gen1); // 0x100
-        assert_pop_layer(5, gen2); // 0x100
-        assert_pop_layer(3, gen1); // 0x110
-        assert_pop_layer(4, gen2); // 0x110
-        assert_pop_layer(2, gen1); // 0x120
-        assert_pop_layer(1, gen1); // 0x200
-
-        Ok(())
-    }
-}

pageserver/src/layered_repository/page_versions.rs

@@ -1,150 +0,0 @@
-use std::{collections::HashMap, ops::RangeBounds, slice};
-
-use zenith_utils::{lsn::Lsn, vec_map::VecMap};
-
-use super::storage_layer::PageVersion;
-
-const EMPTY_SLICE: &[(Lsn, PageVersion)] = &[];
-
-#[derive(Debug, Default)]
-pub struct PageVersions(HashMap<u32, VecMap<Lsn, PageVersion>>);
-
-impl PageVersions {
-    pub fn append_or_update_last(
-        &mut self,
-        blknum: u32,
-        lsn: Lsn,
-        page_version: PageVersion,
-    ) -> Option<PageVersion> {
-        let map = self.0.entry(blknum).or_insert_with(VecMap::default);
-        map.append_or_update_last(lsn, page_version).unwrap()
-    }
-
-    /// Get all [`PageVersion`]s in a block
-    pub fn get_block_slice(&self, blknum: u32) -> &[(Lsn, PageVersion)] {
-        self.0
-            .get(&blknum)
-            .map(VecMap::as_slice)
-            .unwrap_or(EMPTY_SLICE)
-    }
-
-    /// Get a range of [`PageVersions`] in a block
-    pub fn get_block_lsn_range<R: RangeBounds<Lsn>>(
-        &self,
-        blknum: u32,
-        range: R,
-    ) -> &[(Lsn, PageVersion)] {
-        self.0
-            .get(&blknum)
-            .map(|vec_map| vec_map.slice_range(range))
-            .unwrap_or(EMPTY_SLICE)
-    }
-
-    /// Iterate through [`PageVersion`]s in (block, lsn) order.
-    /// If a [`cutoff_lsn`] is set, only show versions with `lsn < cutoff_lsn`
-    pub fn ordered_page_version_iter(&self, cutoff_lsn: Option<Lsn>) -> OrderedPageVersionIter<'_> {
-        let mut ordered_blocks: Vec<u32> = self.0.keys().cloned().collect();
-        ordered_blocks.sort_unstable();
-
-        let slice = ordered_blocks
-            .first()
-            .map(|&blknum| self.get_block_slice(blknum))
-            .unwrap_or(EMPTY_SLICE);
-
-        OrderedPageVersionIter {
-            page_versions: self,
-            ordered_blocks,
-            cur_block_idx: 0,
-            cutoff_lsn,
-            cur_slice_iter: slice.iter(),
-        }
-    }
-}
-
-pub struct OrderedPageVersionIter<'a> {
-    page_versions: &'a PageVersions,
-
-    ordered_blocks: Vec<u32>,
-    cur_block_idx: usize,
-
-    cutoff_lsn: Option<Lsn>,
-
-    cur_slice_iter: slice::Iter<'a, (Lsn, PageVersion)>,
-}
-
-impl OrderedPageVersionIter<'_> {
-    fn is_lsn_before_cutoff(&self, lsn: &Lsn) -> bool {
-        if let Some(cutoff_lsn) = self.cutoff_lsn.as_ref() {
-            lsn < cutoff_lsn
-        } else {
-            true
-        }
-    }
-}
-
-impl<'a> Iterator for OrderedPageVersionIter<'a> {
-    type Item = (u32, Lsn, &'a PageVersion);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        loop {
-            if let Some((lsn, page_version)) = self.cur_slice_iter.next() {
-                if self.is_lsn_before_cutoff(lsn) {
-                    let blknum = self.ordered_blocks[self.cur_block_idx];
-                    return Some((blknum, *lsn, page_version));
-                }
-            }
-
-            let next_block_idx = self.cur_block_idx + 1;
-            let blknum: u32 = *self.ordered_blocks.get(next_block_idx)?;
-            self.cur_block_idx = next_block_idx;
-            self.cur_slice_iter = self.page_versions.get_block_slice(blknum).iter();
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    const EMPTY_PAGE_VERSION: PageVersion = PageVersion {
-        page_image: None,
-        record: None,
-    };
-
-    #[test]
-    fn test_ordered_iter() {
-        let mut page_versions = PageVersions::default();
-        const BLOCKS: u32 = 1000;
-        const LSNS: u64 = 50;
-
-        for blknum in 0..BLOCKS {
-            for lsn in 0..LSNS {
-                let old = page_versions.append_or_update_last(blknum, Lsn(lsn), EMPTY_PAGE_VERSION);
-                assert!(old.is_none());
-            }
-        }
-
-        let mut iter = page_versions.ordered_page_version_iter(None);
-        for blknum in 0..BLOCKS {
-            for lsn in 0..LSNS {
-                let (actual_blknum, actual_lsn, _pv) = iter.next().unwrap();
-                assert_eq!(actual_blknum, blknum);
-                assert_eq!(Lsn(lsn), actual_lsn);
-            }
-        }
-        assert!(iter.next().is_none());
-        assert!(iter.next().is_none()); // should be robust against excessive next() calls
-
-        const CUTOFF_LSN: Lsn = Lsn(30);
-        let mut iter = page_versions.ordered_page_version_iter(Some(CUTOFF_LSN));
-        for blknum in 0..BLOCKS {
-            for lsn in 0..CUTOFF_LSN.0 {
-                let (actual_blknum, actual_lsn, _pv) = iter.next().unwrap();
-                assert_eq!(actual_blknum, blknum);
-                assert_eq!(Lsn(lsn), actual_lsn);
-            }
-        }
-        assert!(iter.next().is_none());
-        assert!(iter.next().is_none()); // should be robust against excessive next() calls
-    }
-}

pageserver/src/layered_repository/storage_layer.rs

@@ -1,174 +0,0 @@
-//!
-//! Common traits and structs for layers
-//!
-
-use crate::relish::RelishTag;
-use crate::repository::WALRecord;
-use crate::ZTimelineId;
-use anyhow::Result;
-use bytes::Bytes;
-use serde::{Deserialize, Serialize};
-use std::fmt;
-use std::path::PathBuf;
-
-use zenith_utils::lsn::Lsn;
-
-// Size of one segment in pages (10 MB)
-pub const RELISH_SEG_SIZE: u32 = 10 * 1024 * 1024 / 8192;
-
-///
-/// Each relish stored in the repository is divided into fixed-sized "segments",
-/// with 10 MB of key-space, or 1280 8k pages each.
-///
-#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy, Serialize, Deserialize)]
-pub struct SegmentTag {
-    pub rel: RelishTag,
-    pub segno: u32,
-}
-
-impl fmt::Display for SegmentTag {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}.{}", self.rel, self.segno)
-    }
-}
-
-impl SegmentTag {
-    pub const fn from_blknum(rel: RelishTag, blknum: u32) -> SegmentTag {
-        SegmentTag {
-            rel,
-            segno: blknum / RELISH_SEG_SIZE,
-        }
-    }
-
-    pub fn blknum_in_seg(&self, blknum: u32) -> bool {
-        blknum / RELISH_SEG_SIZE == self.segno
-    }
-}
-
-///
-/// Represents a version of a page at a specific LSN. The LSN is the key of the
-/// entry in the 'page_versions' hash, it is not duplicated here.
-///
-/// A page version can be stored as a full page image, or as WAL record that needs
-/// to be applied over the previous page version to reconstruct this version.
-///
-/// It's also possible to have both a WAL record and a page image in the same
-/// PageVersion. That happens if page version is originally stored as a WAL record
-/// but it is later reconstructed by a GetPage@LSN request by performing WAL
-/// redo. The get_page_at_lsn() code will store the reconstructed pag image next to
-/// the WAL record in that case. TODO: That's pretty accidental, not the result
-/// of any grand design. If we want to keep reconstructed page versions around, we
-/// probably should have a separate buffer cache so that we could control the
-/// replacement policy globally. Or if we keep a reconstructed page image, we
-/// could throw away the WAL record.
-///
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct PageVersion {
-    /// an 8kb page image
-    pub page_image: Option<Bytes>,
-    /// WAL record to get from previous page version to this one.
-    pub record: Option<WALRecord>,
-}
-
-///
-/// Data needed to reconstruct a page version
-///
-/// 'page_img' is the old base image of the page to start the WAL replay with.
-/// It can be None, if the first WAL record initializes the page (will_init)
-/// 'records' contains the records to apply over the base image.
-///
-pub struct PageReconstructData {
-    pub records: Vec<(Lsn, WALRecord)>,
-    pub page_img: Option<Bytes>,
-}
-
-/// Return value from Layer::get_page_reconstruct_data
-pub enum PageReconstructResult {
-    /// Got all the data needed to reconstruct the requested page
-    Complete,
-    /// This layer didn't contain all the required data, the caller should look up
-    /// the predecessor layer at the returned LSN and collect more data from there.
-    Continue(Lsn),
-    /// This layer didn't contain data needed to reconstruct the page version at
-    /// the returned LSN. This is usually considered an error, but might be OK
-    /// in some circumstances.
-    Missing(Lsn),
-}
-
-///
-/// A Layer corresponds to one RELISH_SEG_SIZE slice of a relish in a range of LSNs.
-/// There are two kinds of layers, in-memory and on-disk layers. In-memory
-/// layers are used to ingest incoming WAL, and provide fast access
-/// to the recent page versions. On-disk layers are stored as files on disk, and
-/// are immutable. This trait presents the common functionality of
-/// in-memory and on-disk layers.
-///
-pub trait Layer: Send + Sync {
-    /// Identify the timeline this relish belongs to
-    fn get_timeline_id(&self) -> ZTimelineId;
-
-    /// Identify the relish segment
-    fn get_seg_tag(&self) -> SegmentTag;
-
-    /// Inclusive start bound of the LSN range that this layer holds
-    fn get_start_lsn(&self) -> Lsn;
-
-    /// Exclusive end bound of the LSN range that this layer holds.
-    ///
-    /// - For an open in-memory layer, this is MAX_LSN.
-    /// - For a frozen in-memory layer or a delta layer, this is a valid end bound.
-    /// - An image layer represents snapshot at one LSN, so end_lsn is always the snapshot LSN + 1
-    fn get_end_lsn(&self) -> Lsn;
-
-    /// Is the segment represented by this layer dropped by PostgreSQL?
-    fn is_dropped(&self) -> bool;
-
-    /// Filename used to store this layer on disk. (Even in-memory layers
-    /// implement this, to print a handy unique identifier for the layer for
-    /// log messages, even though they're never not on disk.)
-    fn filename(&self) -> PathBuf;
-
-    ///
-    /// Return data needed to reconstruct given page at LSN.
-    ///
-    /// It is up to the caller to collect more data from previous layer and
-    /// perform WAL redo, if necessary.
-    ///
-    /// Note that the 'blknum' is the offset of the page from the beginning
-    /// of the *relish*, not the beginning of the segment. The requested
-    /// 'blknum' must be covered by this segment.
-    ///
-    /// See PageReconstructResult for possible return values. The collected data
-    /// is appended to reconstruct_data; the caller should pass an empty struct
-    /// on first call. If this returns PageReconstructResult::Continue, look up
-    /// the predecessor layer and call again with the same 'reconstruct_data'
-    /// to collect more data.
-    fn get_page_reconstruct_data(
-        &self,
-        blknum: u32,
-        lsn: Lsn,
-        reconstruct_data: &mut PageReconstructData,
-    ) -> Result<PageReconstructResult>;
-
-    /// Return size of the segment at given LSN. (Only for blocky relations.)
-    fn get_seg_size(&self, lsn: Lsn) -> Result<u32>;
-
-    /// Does the segment exist at given LSN? Or was it dropped before it.
-    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool>;
-
-    /// Does this layer only contain some data for the segment (incremental),
-    /// or does it contain a version of every page? This is important to know
-    /// for garbage collecting old layers: an incremental layer depends on
-    /// the previous non-incremental layer.
-    fn is_incremental(&self) -> bool;
-
-    /// Release memory used by this layer. There is no corresponding 'load'
-    /// function, that's done implicitly when you call one of the get-functions.
-    fn unload(&self) -> Result<()>;
-
-    /// Permanently remove this layer from disk.
-    fn delete(&self) -> Result<()>;
-
-    /// Dump summary of the contents of the layer to stdout
-    fn dump(&self) -> Result<()>;
-}

pageserver/src/lib.rs

@@ -1,73 +1,37 @@
-use zenith_utils::postgres_backend::AuthType;
-use zenith_utils::zid::{ZTenantId, ZTimelineId};
+use serde::{Deserialize, Serialize};
 
+use std::fmt;
 use std::path::PathBuf;
+use std::str::FromStr;
 use std::time::Duration;
 
-use lazy_static::lazy_static;
-use zenith_metrics::{register_int_gauge_vec, IntGaugeVec};
-
 pub mod basebackup;
 pub mod branches;
-pub mod http;
-pub mod layered_repository;
+pub mod inmem_storage;
+pub mod object_key;
+pub mod object_repository;
+pub mod object_store;
+pub mod page_cache;
 pub mod page_service;
-pub mod relish;
-pub mod relish_storage;
 pub mod repository;
 pub mod restore_local_repo;
-pub mod tenant_mgr;
+pub mod rocksdb_storage;
+pub mod tui;
+pub mod tui_event;
+mod tui_logger;
 pub mod waldecoder;
 pub mod walreceiver;
 pub mod walredo;
 
-pub mod defaults {
-    use const_format::formatcp;
-    use std::time::Duration;
-
-    pub const DEFAULT_PG_LISTEN_PORT: u16 = 64000;
-    pub const DEFAULT_PG_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_PG_LISTEN_PORT}");
-    pub const DEFAULT_HTTP_LISTEN_PORT: u16 = 9898;
-    pub const DEFAULT_HTTP_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_HTTP_LISTEN_PORT}");
-
-    // FIXME: This current value is very low. I would imagine something like 1 GB or 10 GB
-    // would be more appropriate. But a low value forces the code to be exercised more,
-    // which is good for now to trigger bugs.
-    pub const DEFAULT_CHECKPOINT_DISTANCE: u64 = 256 * 1024 * 1024;
-    pub const DEFAULT_CHECKPOINT_PERIOD: Duration = Duration::from_secs(1);
-
-    pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
-    pub const DEFAULT_GC_PERIOD: Duration = Duration::from_secs(100);
-
-    pub const DEFAULT_SUPERUSER: &str = "zenith_admin";
-    pub const DEFAULT_RELISH_STORAGE_MAX_CONCURRENT_SYNC_LIMITS: usize = 100;
-}
-
-lazy_static! {
-    static ref LIVE_CONNECTIONS_COUNT: IntGaugeVec = register_int_gauge_vec!(
-        "pageserver_live_connections_count",
-        "Number of live network connections",
-        &["pageserver_connection_kind"]
-    )
-    .expect("failed to define a metric");
-}
-
-pub const LOG_FILE_NAME: &str = "pageserver.log";
-
 #[derive(Debug, Clone)]
 pub struct PageServerConf {
     pub daemonize: bool,
-    pub listen_pg_addr: String,
-    pub listen_http_addr: String,
-    // Flush out an inmemory layer, if it's holding WAL older than this
-    // This puts a backstop on how much WAL needs to be re-digested if the
-    // page server crashes.
-    pub checkpoint_distance: u64,
-    pub checkpoint_period: Duration,
-
+    pub interactive: bool,
+    pub materialize: bool,
+    pub listen_addr: String,
     pub gc_horizon: u64,
     pub gc_period: Duration,
-    pub superuser: String,
+    pub wal_redoers: usize,
 
     // Repository directory, relative to current working directory.
     // Normally, the page server changes the current working directory
@@ -78,11 +42,6 @@ pub struct PageServerConf {
     pub workdir: PathBuf,
 
     pub pg_distrib_dir: PathBuf,
-
-    pub auth_type: AuthType,
-
-    pub auth_validation_public_key_path: Option<PathBuf>,
-    pub relish_storage_config: Option<RelishStorageConfig>,
 }
 
 impl PageServerConf {
@@ -90,40 +49,24 @@ impl PageServerConf {
     // Repository paths, relative to workdir.
     //
 
-    fn tenants_path(&self) -> PathBuf {
-        self.workdir.join("tenants")
+    fn tag_path(&self, name: &str) -> PathBuf {
+        self.workdir.join("refs").join("tags").join(name)
     }
 
-    fn tenant_path(&self, tenantid: &ZTenantId) -> PathBuf {
-        self.tenants_path().join(tenantid.to_string())
+    fn branch_path(&self, name: &str) -> PathBuf {
+        self.workdir.join("refs").join("branches").join(name)
     }
 
-    fn tags_path(&self, tenantid: &ZTenantId) -> PathBuf {
-        self.tenant_path(tenantid).join("refs").join("tags")
+    fn timeline_path(&self, timelineid: ZTimelineId) -> PathBuf {
+        self.workdir.join("timelines").join(timelineid.to_string())
     }
 
-    fn tag_path(&self, tag_name: &str, tenantid: &ZTenantId) -> PathBuf {
-        self.tags_path(tenantid).join(tag_name)
+    fn snapshots_path(&self, timelineid: ZTimelineId) -> PathBuf {
+        self.timeline_path(timelineid).join("snapshots")
     }
 
-    fn branches_path(&self, tenantid: &ZTenantId) -> PathBuf {
-        self.tenant_path(tenantid).join("refs").join("branches")
-    }
-
-    fn branch_path(&self, branch_name: &str, tenantid: &ZTenantId) -> PathBuf {
-        self.branches_path(tenantid).join(branch_name)
-    }
-
-    fn timelines_path(&self, tenantid: &ZTenantId) -> PathBuf {
-        self.tenant_path(tenantid).join("timelines")
-    }
-
-    fn timeline_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
-        self.timelines_path(tenantid).join(timelineid.to_string())
-    }
-
-    fn ancestor_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
-        self.timeline_path(timelineid, tenantid).join("ancestor")
+    fn ancestor_path(&self, timelineid: ZTimelineId) -> PathBuf {
+        self.timeline_path(timelineid).join("ancestor")
     }
 
     //
@@ -137,73 +80,65 @@ impl PageServerConf {
     pub fn pg_lib_dir(&self) -> PathBuf {
         self.pg_distrib_dir.join("lib")
     }
+}
 
-    #[cfg(test)]
-    fn test_repo_dir(test_name: &str) -> PathBuf {
-        PathBuf::from(format!("../tmp_check/test_{}", test_name))
-    }
+/// Zenith Timeline ID is a 128-bit random ID.
+///
+/// Zenith timeline IDs are different from PostgreSQL timeline
+/// IDs. They serve a similar purpose though: they differentiate
+/// between different "histories" of the same cluster.  However,
+/// PostgreSQL timeline IDs are a bit cumbersome, because they are only
+/// 32-bits wide, and they must be in ascending order in any given
+/// timeline history.  Those limitations mean that we cannot generate a
+/// new PostgreSQL timeline ID by just generating a random number. And
+/// that in turn is problematic for the "pull/push" workflow, where you
+/// have a local copy of a zenith repository, and you periodically sync
+/// the local changes with a remote server. When you work "detached"
+/// from the remote server, you cannot create a PostgreSQL timeline ID
+/// that's guaranteed to be different from all existing timelines in
+/// the remote server. For example, if two people are having a clone of
+/// the repository on their laptops, and they both create a new branch
+/// with different name. What timeline ID would they assign to their
+/// branches? If they pick the same one, and later try to push the
+/// branches to the same remote server, they will get mixed up.
+///
+/// To avoid those issues, Zenith has its own concept of timelines that
+/// is separate from PostgreSQL timelines, and doesn't have those
+/// limitations. A zenith timeline is identified by a 128-bit ID, which
+/// is usually printed out as a hex string.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub struct ZTimelineId([u8; 16]);
 
-    #[cfg(test)]
-    fn dummy_conf(repo_dir: PathBuf) -> Self {
-        PageServerConf {
-            daemonize: false,
-            checkpoint_distance: defaults::DEFAULT_CHECKPOINT_DISTANCE,
-            checkpoint_period: Duration::from_secs(10),
-            gc_horizon: defaults::DEFAULT_GC_HORIZON,
-            gc_period: Duration::from_secs(10),
-            listen_pg_addr: defaults::DEFAULT_PG_LISTEN_ADDR.to_string(),
-            listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(),
-            superuser: "zenith_admin".to_string(),
-            workdir: repo_dir,
-            pg_distrib_dir: "".into(),
-            auth_type: AuthType::Trust,
-            auth_validation_public_key_path: None,
-            relish_storage_config: None,
-        }
+impl FromStr for ZTimelineId {
+    type Err = hex::FromHexError;
+
+    fn from_str(s: &str) -> Result<ZTimelineId, Self::Err> {
+        let timelineid = hex::decode(s)?;
+
+        let mut buf: [u8; 16] = [0u8; 16];
+        buf.copy_from_slice(timelineid.as_slice());
+        Ok(ZTimelineId(buf))
     }
 }
 
-/// External relish storage configuration, enough for creating a client for that storage.
-#[derive(Debug, Clone)]
-pub struct RelishStorageConfig {
-    /// Limits the number of concurrent sync operations between pageserver and relish storage.
-    pub max_concurrent_sync: usize,
-    /// The storage connection configuration.
-    pub storage: RelishStorageKind,
-}
+impl ZTimelineId {
+    pub fn from(b: [u8; 16]) -> ZTimelineId {
+        ZTimelineId(b)
+    }
 
-/// A kind of a relish storage to connect to, with its connection configuration.
-#[derive(Debug, Clone)]
-pub enum RelishStorageKind {
-    /// Storage based on local file system.
-    /// Specify a root folder to place all stored relish data into.
-    LocalFs(PathBuf),
-    /// AWS S3 based storage, storing all relishes into the root
-    /// of the S3 bucket from the config.
-    AwsS3(S3Config),
-}
+    pub fn get_from_buf(buf: &mut dyn bytes::Buf) -> ZTimelineId {
+        let mut arr = [0u8; 16];
+        buf.copy_to_slice(&mut arr);
+        ZTimelineId::from(arr)
+    }
 
-/// AWS S3 bucket coordinates and access credentials to manage the bucket contents (read and write).
-#[derive(Clone)]
-pub struct S3Config {
-    /// Name of the bucket to connect to.
-    pub bucket_name: String,
-    /// The region where the bucket is located at.
-    pub bucket_region: String,
-    /// "Login" to use when connecting to bucket.
-    /// Can be empty for cases like AWS k8s IAM
-    /// where we can allow certain pods to connect
-    /// to the bucket directly without any credentials.
-    pub access_key_id: Option<String>,
-    /// "Password" to use when connecting to bucket.
-    pub secret_access_key: Option<String>,
-}
-
-impl std::fmt::Debug for S3Config {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("S3Config")
-            .field("bucket_name", &self.bucket_name)
-            .field("bucket_region", &self.bucket_region)
-            .finish()
+    pub fn as_arr(&self) -> [u8; 16] {
+        self.0
+    }
+}
+
+impl fmt::Display for ZTimelineId {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(&hex::encode(self.0))
     }
 }

pageserver/src/object_key.rs

@@ -0,0 +1,84 @@
+use crate::repository::{BufferTag, RelTag};
+use crate::waldecoder::TransactionId;
+use crate::ZTimelineId;
+use serde::{Deserialize, Serialize};
+
+///
+/// ObjectKey is the key type used to identify objects stored in an object
+/// repository. It is shared between object_repository.rs and object_store.rs.
+/// It is mostly opaque to ObjectStore, it just stores and retrieves objects
+/// using the key given by the caller.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub struct ObjectKey {
+    pub timeline: ZTimelineId,
+    pub tag: ObjectTag,
+}
+
+///
+/// Non-relation transaction status files (clog (a.k.a. pg_xact) and pg_multixact)
+/// in Postgres are handled by SLRU (Simple LRU) buffer, hence the name.
+///
+/// These files are global for a postgres instance.
+///
+/// These files are divided into segments, which are divided into pages
+/// of the same BLCKSZ as used for relation files.
+///
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub struct SlruBufferTag {
+    pub blknum: u32,
+}
+
+///
+/// Special type of Postgres files: pg_filenode.map is needed to map
+/// catalog table OIDs to filenode numbers, which define filename.
+///
+/// Each database has a map file for its local mapped catalogs,
+/// and there is a separate map file for shared catalogs.
+///
+/// These files have untypical size of 512 bytes.
+///
+/// See PostgreSQL relmapper.c for details.
+///
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub struct DatabaseTag {
+    pub spcnode: u32,
+    pub dbnode: u32,
+}
+
+///
+/// Non-relation files that keep state for prepared transactions.
+/// Unlike other files these are not divided into pages.
+///
+/// See PostgreSQL twophase.c for details.
+///
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub struct PrepareTag {
+    pub xid: TransactionId,
+}
+
+/// ObjectTag is a part of ObjectKey that is specific to the type of
+/// the stored object.
+///
+/// NB: the order of the enum values is significant!  In particular,
+/// rocksdb_storage.rs assumes that TimelineMetadataTag is first
+///
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub enum ObjectTag {
+    // dummy tag preceeding all other keys
+    FirstTag,
+    TimelineMetadataTag,
+    // Special entry that represents PostgreSQL checkpoint.
+    // We use it to track fields needed to restore controlfile checkpoint.
+    Checkpoint,
+    // Various types of non-relation files.
+    // We need them to bootstrap compute node.
+    ControlFile,
+    Clog(SlruBufferTag),
+    MultiXactMembers(SlruBufferTag),
+    MultiXactOffsets(SlruBufferTag),
+    FileNodeMap(DatabaseTag),
+    TwoPhase(PrepareTag),
+    // put relations at the end of enum to allow efficient iterations through non-rel objects
+    RelationMetadata(RelTag),
+    RelationBuffer(BufferTag),
+}

pageserver/src/object_store.rs

@@ -0,0 +1,88 @@
+//! Low-level key-value storage abstraction.
+//!
+use crate::object_key::*;
+use crate::repository::RelTag;
+use crate::ZTimelineId;
+use anyhow::Result;
+use std::collections::HashSet;
+use std::iter::Iterator;
+use zenith_utils::lsn::Lsn;
+
+///
+/// Low-level storage abstraction.
+///
+/// All the data in the repository is stored in a key-value store. This trait
+/// abstracts the details of the key-value store.
+///
+/// A simple key-value store would support just GET and PUT operations with
+/// a key, but the upper layer needs slightly complicated read operations
+///
+/// The most frequently used function is 'object_versions'. It is used
+/// to look up a page version. It is LSN aware, in that the caller
+/// specifies an LSN, and the function returns all values for that
+/// block with the same or older LSN.
+///
+pub trait ObjectStore: Send + Sync {
+    ///
+    /// Store a value with given key.
+    ///
+    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()>;
+
+    /// Read entry with the exact given key.
+    ///
+    /// This is used for retrieving metadata with special key that doesn't
+    /// correspond to any real relation.
+    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>>;
+
+    /// Read key greater or equal than specified
+    fn get_next_key(&self, key: &ObjectKey) -> Result<Option<ObjectKey>>;
+
+    /// Iterate through all page versions of one object.
+    ///
+    /// Returns all page versions in descending LSN order, along with the LSN
+    /// of each page version.
+    fn object_versions<'a>(
+        &'a self,
+        key: &ObjectKey,
+        lsn: Lsn,
+    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>>;
+
+    /// Iterate through versions of all objects in a timeline.
+    ///
+    /// Returns objects in increasing key-version order.
+    /// Returns all versions up to and including the specified LSN.
+    fn objects<'a>(
+        &'a self,
+        timeline: ZTimelineId,
+        lsn: Lsn,
+    ) -> Result<Box<dyn Iterator<Item = Result<(ObjectTag, Lsn, Vec<u8>)>> + 'a>>;
+
+    /// Iterate through all keys with given tablespace and database ID, and LSN <= 'lsn'.
+    /// Both dbnode and spcnode can be InvalidId (0) which means get all relations in tablespace/cluster
+    ///
+    /// This is used to implement 'create database'
+    fn list_rels(
+        &self,
+        timelineid: ZTimelineId,
+        spcnode: u32,
+        dbnode: u32,
+        lsn: Lsn,
+    ) -> Result<HashSet<RelTag>>;
+
+    /// Iterate through objects tags. If nonrel_only, then only non-relationa data is iterated.
+    ///
+    /// This is used to implement GC and preparing tarball for new node startup
+    /// Returns objects in increasing key-version order.
+    fn list_objects<'a>(
+        &'a self,
+        timelineid: ZTimelineId,
+        nonrel_only: bool,
+        lsn: Lsn,
+    ) -> Result<Box<dyn Iterator<Item = ObjectTag> + 'a>>;
+
+    /// Unlink object (used by GC). This mehod may actually delete object or just mark it for deletion.
+    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()>;
+
+    // Compact storage and remove versions marged for deletion
+    fn compact(&self);
+}

pageserver/src/page_cache.rs

@@ -0,0 +1,37 @@
+//! This module acts as a switchboard to access different repositories managed by this
+//! page server. Currently, a Page Server can only manage one repository, so there
+//! isn't much here. If we implement multi-tenancy, this will probably be changed into
+//! a hash map, keyed by the tenant ID.
+
+use crate::object_repository::ObjectRepository;
+use crate::repository::Repository;
+use crate::rocksdb_storage::RocksObjectStore;
+//use crate::inmem_storage::InmemObjectStore;
+use crate::walredo::PostgresRedoManager;
+use crate::PageServerConf;
+use lazy_static::lazy_static;
+use std::sync::{Arc, Mutex};
+
+lazy_static! {
+    pub static ref REPOSITORY: Mutex<Option<Arc<dyn Repository>>> = Mutex::new(None);
+}
+
+pub fn init(conf: &'static PageServerConf) {
+    let mut m = REPOSITORY.lock().unwrap();
+
+    let obj_store = RocksObjectStore::open(conf).unwrap();
+    //let obj_store = InmemObjectStore::open(conf).unwrap();
+
+    // Set up a WAL redo manager, for applying WAL records.
+    let walredo_mgr = PostgresRedoManager::new(conf);
+
+    // we have already changed current dir to the repository.
+    let repo = ObjectRepository::new(conf, Arc::new(obj_store), Arc::new(walredo_mgr));
+
+    *m = Some(Arc::new(repo));
+}
+
+pub fn get_repository() -> Arc<dyn Repository> {
+    let o = &REPOSITORY.lock().unwrap();
+    Arc::clone(o.as_ref().unwrap())
+}

pageserver/src/relish.rs

@@ -1,231 +0,0 @@
-//!
-//! Zenith stores PostgreSQL relations, and some other files, in the
-//! repository.  The relations (i.e. tables and indexes) take up most
-//! of the space in a typical installation, while the other files are
-//! small. We call each relation and other file that is stored in the
-//! repository a "relish". It comes from "rel"-ish, as in "kind of a
-//! rel", because it covers relations as well as other things that are
-//! not relations, but are treated similarly for the purposes of the
-//! storage layer.
-//!
-//! This source file contains the definition of the RelishTag struct,
-//! which uniquely identifies a relish.
-//!
-//! Relishes come in two flavors: blocky and non-blocky. Relations and
-//! SLRUs are blocky, that is, they are divided into 8k blocks, and
-//! the repository tracks their size. Other relishes are non-blocky:
-//! the content of the whole relish is stored as one blob. Block
-//! number must be passed as 0 for all operations on a non-blocky
-//! relish. The one "block" that you store in a non-blocky relish can
-//! have arbitrary size, but they are expected to be small, or you
-//! will have performance issues.
-//!
-//! All relishes are versioned by LSN in the repository.
-//!
-
-use serde::{Deserialize, Serialize};
-use std::fmt;
-
-use postgres_ffi::relfile_utils::forknumber_to_name;
-use postgres_ffi::{Oid, TransactionId};
-
-///
-/// RelishTag identifies one relish.
-///
-#[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
-pub enum RelishTag {
-    // Relations correspond to PostgreSQL relation forks. Each
-    // PostgreSQL relation fork is considered a separate relish.
-    Relation(RelTag),
-
-    // SLRUs include pg_clog, pg_multixact/members, and
-    // pg_multixact/offsets. There are other SLRUs in PostgreSQL, but
-    // they don't need to be stored permanently (e.g. pg_subtrans),
-    // or we do not support them in zenith yet (pg_commit_ts).
-    //
-    // These are currently never requested directly by the compute
-    // nodes, although in principle that would be possible. However,
-    // when a new compute node is created, these are included in the
-    // tarball that we send to the compute node to initialize the
-    // PostgreSQL data directory.
-    //
-    // Each SLRU segment in PostgreSQL is considered a separate
-    // relish. For example, pg_clog/0000, pg_clog/0001, and so forth.
-    //
-    // SLRU segments are divided into blocks, like relations.
-    Slru { slru: SlruKind, segno: u32 },
-
-    // Miscellaneous other files that need to be included in the
-    // tarball at compute node creation. These are non-blocky, and are
-    // expected to be small.
-
-    //
-    // FileNodeMap represents PostgreSQL's 'pg_filenode.map'
-    // files. They are needed to map catalog table OIDs to filenode
-    // numbers. Usually the mapping is done by looking up a relation's
-    // 'relfilenode' field in the 'pg_class' system table, but that
-    // doesn't work for 'pg_class' itself and a few other such system
-    // relations. See PostgreSQL relmapper.c for details.
-    //
-    // Each database has a map file for its local mapped catalogs,
-    // and there is a separate map file for shared catalogs.
-    //
-    // These files are always 512 bytes long (although we don't check
-    // or care about that in the page server).
-    //
-    FileNodeMap { spcnode: Oid, dbnode: Oid },
-
-    //
-    // State files for prepared transactions (e.g pg_twophase/1234)
-    //
-    TwoPhase { xid: TransactionId },
-
-    // The control file, stored in global/pg_control
-    ControlFile,
-
-    // Special entry that represents PostgreSQL checkpoint. It doesn't
-    // correspond to to any physical file in PostgreSQL, but we use it
-    // to track fields needed to restore the checkpoint data in the
-    // control file, when a compute node is created.
-    Checkpoint,
-}
-
-impl RelishTag {
-    pub const fn is_blocky(&self) -> bool {
-        match self {
-            // These relishes work with blocks
-            RelishTag::Relation(_) | RelishTag::Slru { slru: _, segno: _ } => true,
-
-            // and these don't
-            RelishTag::FileNodeMap {
-                spcnode: _,
-                dbnode: _,
-            }
-            | RelishTag::TwoPhase { xid: _ }
-            | RelishTag::ControlFile
-            | RelishTag::Checkpoint => false,
-        }
-    }
-
-    // Physical relishes represent files and use
-    // RelationSizeEntry to track existing and dropped files.
-    // They can be both blocky and non-blocky.
-    pub const fn is_physical(&self) -> bool {
-        match self {
-            // These relishes represent physical files
-            RelishTag::Relation(_)
-            | RelishTag::Slru { .. }
-            | RelishTag::FileNodeMap { .. }
-            | RelishTag::TwoPhase { .. } => true,
-
-            // and these don't
-            RelishTag::ControlFile | RelishTag::Checkpoint => false,
-        }
-    }
-
-    // convenience function to check if this relish is a normal relation.
-    pub const fn is_relation(&self) -> bool {
-        matches!(self, RelishTag::Relation(_))
-    }
-}
-
-///
-/// Relation data file segment id throughout the Postgres cluster.
-///
-/// Every data file in Postgres is uniquely identified by 4 numbers:
-/// - relation id / node (`relnode`)
-/// - database id (`dbnode`)
-/// - tablespace id (`spcnode`), in short this is a unique id of a separate
-///   directory to store data files.
-/// - forknumber (`forknum`) is used to split different kinds of data of the same relation
-///   between some set of files (`relnode`, `relnode_fsm`, `relnode_vm`).
-///
-/// In native Postgres code `RelFileNode` structure and individual `ForkNumber` value
-/// are used for the same purpose.
-/// [See more related comments here](https:///github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/relfilenode.h#L57).
-///
-#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy, Serialize, Deserialize)]
-pub struct RelTag {
-    pub forknum: u8,
-    pub spcnode: Oid,
-    pub dbnode: Oid,
-    pub relnode: Oid,
-}
-
-/// Display RelTag in the same format that's used in most PostgreSQL debug messages:
-///
-/// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
-///
-impl fmt::Display for RelTag {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        if let Some(forkname) = forknumber_to_name(self.forknum) {
-            write!(
-                f,
-                "{}/{}/{}_{}",
-                self.spcnode, self.dbnode, self.relnode, forkname
-            )
-        } else {
-            write!(f, "{}/{}/{}", self.spcnode, self.dbnode, self.relnode)
-        }
-    }
-}
-
-/// Display RelTag in the same format that's used in most PostgreSQL debug messages:
-///
-/// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
-///
-impl fmt::Display for RelishTag {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            RelishTag::Relation(rel) => rel.fmt(f),
-            RelishTag::Slru { slru, segno } => {
-                // e.g. pg_clog/0001
-                write!(f, "{}/{:04X}", slru.to_str(), segno)
-            }
-            RelishTag::FileNodeMap { spcnode, dbnode } => {
-                write!(f, "relmapper file for spc {} db {}", spcnode, dbnode)
-            }
-            RelishTag::TwoPhase { xid } => {
-                write!(f, "pg_twophase/{:08X}", xid)
-            }
-            RelishTag::ControlFile => {
-                write!(f, "control file")
-            }
-            RelishTag::Checkpoint => {
-                write!(f, "checkpoint")
-            }
-        }
-    }
-}
-
-///
-/// Non-relation transaction status files (clog (a.k.a. pg_xact) and
-/// pg_multixact) in Postgres are handled by SLRU (Simple LRU) buffer,
-/// hence the name.
-///
-/// These files are global for a postgres instance.
-///
-/// These files are divided into segments, which are divided into
-/// pages of the same BLCKSZ as used for relation files.
-///
-#[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
-pub enum SlruKind {
-    Clog,
-    MultiXactMembers,
-    MultiXactOffsets,
-}
-
-impl SlruKind {
-    pub fn to_str(&self) -> &'static str {
-        match self {
-            Self::Clog => "pg_xact",
-            Self::MultiXactMembers => "pg_multixact/members",
-            Self::MultiXactOffsets => "pg_multixact/offsets",
-        }
-    }
-}
-
-pub const FIRST_NONREL_RELISH_TAG: RelishTag = RelishTag::Slru {
-    slru: SlruKind::Clog,
-    segno: 0,
-};

pageserver/src/relish_storage.rs

@@ -1,87 +0,0 @@
-//! Abstractions for the page server to store its relish layer data in the external storage.
-//!
-//! Main purpose of this module subtree is to provide a set of abstractions to manage the storage state
-//! in a way, optimal for page server.
-//!
-//! The abstractions hide multiple custom external storage API implementations,
-//! such as AWS S3, local filesystem, etc., located in the submodules.
-
-mod local_fs;
-mod rust_s3;
-/// A queue-based storage with the background machinery behind it to synchronize
-/// local page server layer files with external storage.
-mod synced_storage;
-
-use std::{path::Path, thread};
-
-use anyhow::Context;
-
-pub use self::synced_storage::schedule_timeline_upload;
-use self::{local_fs::LocalFs, rust_s3::RustS3};
-use crate::{PageServerConf, RelishStorageKind};
-
-pub fn run_storage_sync_thread(
-    config: &'static PageServerConf,
-) -> anyhow::Result<Option<thread::JoinHandle<anyhow::Result<()>>>> {
-    match &config.relish_storage_config {
-        Some(relish_storage_config) => {
-            let max_concurrent_sync = relish_storage_config.max_concurrent_sync;
-            match &relish_storage_config.storage {
-                RelishStorageKind::LocalFs(root) => synced_storage::run_storage_sync_thread(
-                    config,
-                    LocalFs::new(root.clone())?,
-                    max_concurrent_sync,
-                ),
-                RelishStorageKind::AwsS3(s3_config) => synced_storage::run_storage_sync_thread(
-                    config,
-                    RustS3::new(s3_config)?,
-                    max_concurrent_sync,
-                ),
-            }
-        }
-        None => Ok(None),
-    }
-}
-
-/// Storage (potentially remote) API to manage its state.
-#[async_trait::async_trait]
-pub trait RelishStorage: Send + Sync {
-    type RelishStoragePath;
-
-    fn derive_destination(
-        page_server_workdir: &Path,
-        relish_local_path: &Path,
-    ) -> anyhow::Result<Self::RelishStoragePath>;
-
-    async fn list_relishes(&self) -> anyhow::Result<Vec<Self::RelishStoragePath>>;
-
-    async fn download_relish<W: 'static + std::io::Write + Send>(
-        &self,
-        from: &Self::RelishStoragePath,
-        // rust_s3 `get_object_stream` method requires `std::io::BufWriter` for some reason, not the async counterpart
-        // that forces us to consume and return the writer to satisfy the blocking operation async wrapper requirements
-        to: std::io::BufWriter<W>,
-    ) -> anyhow::Result<std::io::BufWriter<W>>;
-
-    async fn delete_relish(&self, path: &Self::RelishStoragePath) -> anyhow::Result<()>;
-
-    async fn upload_relish<R: tokio::io::AsyncRead + std::marker::Unpin + Send>(
-        &self,
-        from: &mut tokio::io::BufReader<R>,
-        to: &Self::RelishStoragePath,
-    ) -> anyhow::Result<()>;
-}
-
-fn strip_workspace_prefix<'a>(
-    page_server_workdir: &'a Path,
-    relish_local_path: &'a Path,
-) -> anyhow::Result<&'a Path> {
-    relish_local_path
-        .strip_prefix(page_server_workdir)
-        .with_context(|| {
-            format!(
-                "Unexpected: relish local path '{}' is not relevant to server workdir",
-                relish_local_path.display(),
-            )
-        })
-}

pageserver/src/relish_storage/local_fs.rs

@@ -1,189 +0,0 @@
-//! Local filesystem relish storage.
-//!
-//! Page server already stores layer data on the server, when freezing it.
-//! This storage serves a way to
-//!
-//! * test things locally simply
-//! * allow to compabre both binary sets
-//! * help validating the relish storage API
-
-use std::{
-    future::Future,
-    io::Write,
-    path::{Path, PathBuf},
-    pin::Pin,
-};
-
-use anyhow::{bail, Context};
-use tokio::{fs, io};
-
-use super::{strip_workspace_prefix, RelishStorage};
-
-pub struct LocalFs {
-    root: PathBuf,
-}
-
-impl LocalFs {
-    /// Atetmpts to create local FS relish storage, also creates the directory provided, if not exists.
-    pub fn new(root: PathBuf) -> anyhow::Result<Self> {
-        if !root.exists() {
-            std::fs::create_dir_all(&root).with_context(|| {
-                format!(
-                    "Failed to create all directories in the given root path {}",
-                    root.display(),
-                )
-            })?;
-        }
-        Ok(Self { root })
-    }
-
-    fn resolve_in_storage(&self, path: &Path) -> anyhow::Result<PathBuf> {
-        if path.is_relative() {
-            Ok(self.root.join(path))
-        } else if path.starts_with(&self.root) {
-            Ok(path.to_path_buf())
-        } else {
-            bail!(
-                "Path '{}' does not belong to the current storage",
-                path.display()
-            )
-        }
-    }
-}
-
-#[async_trait::async_trait]
-impl RelishStorage for LocalFs {
-    type RelishStoragePath = PathBuf;
-
-    fn derive_destination(
-        page_server_workdir: &Path,
-        relish_local_path: &Path,
-    ) -> anyhow::Result<Self::RelishStoragePath> {
-        Ok(strip_workspace_prefix(page_server_workdir, relish_local_path)?.to_path_buf())
-    }
-
-    async fn list_relishes(&self) -> anyhow::Result<Vec<Self::RelishStoragePath>> {
-        Ok(get_all_files(&self.root).await?.into_iter().collect())
-    }
-
-    async fn download_relish<W: 'static + std::io::Write + Send>(
-        &self,
-        from: &Self::RelishStoragePath,
-        mut to: std::io::BufWriter<W>,
-    ) -> anyhow::Result<std::io::BufWriter<W>> {
-        let file_path = self.resolve_in_storage(from)?;
-        if file_path.exists() && file_path.is_file() {
-            let updated_buffer = tokio::task::spawn_blocking(move || {
-                let mut source = std::io::BufReader::new(
-                    std::fs::OpenOptions::new()
-                        .read(true)
-                        .open(&file_path)
-                        .with_context(|| {
-                            format!(
-                                "Failed to open source file '{}' to use in the download",
-                                file_path.display()
-                            )
-                        })?,
-                );
-                std::io::copy(&mut source, &mut to)
-                    .context("Failed to download the relish file")?;
-                to.flush().context("Failed to flush the download buffer")?;
-                Ok::<_, anyhow::Error>(to)
-            })
-            .await
-            .context("Failed to spawn a blocking task")??;
-            Ok(updated_buffer)
-        } else {
-            bail!(
-                "File '{}' either does not exist or is not a file",
-                file_path.display()
-            )
-        }
-    }
-
-    async fn delete_relish(&self, path: &Self::RelishStoragePath) -> anyhow::Result<()> {
-        let file_path = self.resolve_in_storage(path)?;
-        if file_path.exists() && file_path.is_file() {
-            Ok(tokio::fs::remove_file(file_path).await?)
-        } else {
-            bail!(
-                "File '{}' either does not exist or is not a file",
-                file_path.display()
-            )
-        }
-    }
-
-    async fn upload_relish<R: io::AsyncRead + std::marker::Unpin + Send>(
-        &self,
-        from: &mut io::BufReader<R>,
-        to: &Self::RelishStoragePath,
-    ) -> anyhow::Result<()> {
-        let target_file_path = self.resolve_in_storage(to)?;
-        create_target_directory(&target_file_path).await?;
-        let mut destination = io::BufWriter::new(
-            fs::OpenOptions::new()
-                .write(true)
-                .create(true)
-                .open(&target_file_path)
-                .await
-                .with_context(|| {
-                    format!(
-                        "Failed to open target fs destination at '{}'",
-                        target_file_path.display()
-                    )
-                })?,
-        );
-
-        io::copy_buf(from, &mut destination)
-            .await
-            .context("Failed to upload relish to local storage")?;
-        Ok(())
-    }
-}
-
-fn get_all_files<'a, P>(
-    directory_path: P,
-) -> Pin<Box<dyn Future<Output = anyhow::Result<Vec<PathBuf>>> + Send + Sync + 'a>>
-where
-    P: AsRef<Path> + Send + Sync + 'a,
-{
-    Box::pin(async move {
-        let directory_path = directory_path.as_ref();
-        if directory_path.exists() {
-            if directory_path.is_dir() {
-                let mut paths = Vec::new();
-                let mut dir_contents = tokio::fs::read_dir(directory_path).await?;
-                while let Some(dir_entry) = dir_contents.next_entry().await? {
-                    let file_type = dir_entry.file_type().await?;
-                    let entry_path = dir_entry.path();
-                    if file_type.is_symlink() {
-                        log::debug!("{:?} us a symlink, skipping", entry_path)
-                    } else if file_type.is_dir() {
-                        paths.extend(get_all_files(entry_path).await?.into_iter())
-                    } else {
-                        paths.push(dir_entry.path());
-                    }
-                }
-                Ok(paths)
-            } else {
-                bail!("Path '{}' is not a directory", directory_path.display())
-            }
-        } else {
-            Ok(Vec::new())
-        }
-    })
-}
-
-async fn create_target_directory(target_file_path: &Path) -> anyhow::Result<()> {
-    let target_dir = match target_file_path.parent() {
-        Some(parent_dir) => parent_dir,
-        None => bail!(
-            "Relish path '{}' has no parent directory",
-            target_file_path.display()
-        ),
-    };
-    if !target_dir.exists() {
-        tokio::fs::create_dir_all(target_dir).await?;
-    }
-    Ok(())
-}

pageserver/src/relish_storage/rust_s3.rs

@@ -1,149 +0,0 @@
-//! A wrapper around AWS S3 client library `rust_s3` to be used a relish storage.
-
-use std::io::Write;
-use std::path::Path;
-
-use anyhow::Context;
-use s3::{bucket::Bucket, creds::Credentials, region::Region};
-
-use crate::{
-    relish_storage::{strip_workspace_prefix, RelishStorage},
-    S3Config,
-};
-
-const S3_FILE_SEPARATOR: char = '/';
-
-#[derive(Debug)]
-pub struct S3ObjectKey(String);
-
-impl S3ObjectKey {
-    fn key(&self) -> &str {
-        &self.0
-    }
-}
-
-/// AWS S3 relish storage.
-pub struct RustS3 {
-    bucket: Bucket,
-}
-
-impl RustS3 {
-    /// Creates the relish storage, errors if incorrect AWS S3 configuration provided.
-    pub fn new(aws_config: &S3Config) -> anyhow::Result<Self> {
-        let region = aws_config
-            .bucket_region
-            .parse::<Region>()
-            .context("Failed to parse the s3 region from config")?;
-        let credentials = Credentials::new(
-            aws_config.access_key_id.as_deref(),
-            aws_config.secret_access_key.as_deref(),
-            None,
-            None,
-            None,
-        )
-        .context("Failed to create the s3 credentials")?;
-        Ok(Self {
-            bucket: Bucket::new_with_path_style(
-                aws_config.bucket_name.as_str(),
-                region,
-                credentials,
-            )
-            .context("Failed to create the s3 bucket")?,
-        })
-    }
-}
-
-#[async_trait::async_trait]
-impl RelishStorage for RustS3 {
-    type RelishStoragePath = S3ObjectKey;
-
-    fn derive_destination(
-        page_server_workdir: &Path,
-        relish_local_path: &Path,
-    ) -> anyhow::Result<Self::RelishStoragePath> {
-        let relative_path = strip_workspace_prefix(page_server_workdir, relish_local_path)?;
-        let mut key = String::new();
-        for segment in relative_path {
-            key.push(S3_FILE_SEPARATOR);
-            key.push_str(&segment.to_string_lossy());
-        }
-        Ok(S3ObjectKey(key))
-    }
-
-    async fn list_relishes(&self) -> anyhow::Result<Vec<Self::RelishStoragePath>> {
-        let list_response = self
-            .bucket
-            .list(String::new(), None)
-            .await
-            .context("Failed to list s3 objects")?;
-
-        Ok(list_response
-            .into_iter()
-            .flat_map(|response| response.contents)
-            .map(|s3_object| S3ObjectKey(s3_object.key))
-            .collect())
-    }
-
-    async fn download_relish<W: 'static + std::io::Write + Send>(
-        &self,
-        from: &Self::RelishStoragePath,
-        mut to: std::io::BufWriter<W>,
-    ) -> anyhow::Result<std::io::BufWriter<W>> {
-        let code = self
-            .bucket
-            .get_object_stream(from.key(), &mut to)
-            .await
-            .with_context(|| format!("Failed to download s3 object with key {}", from.key()))?;
-        if code != 200 {
-            Err(anyhow::format_err!(
-                "Received non-200 exit code during downloading object from directory, code: {}",
-                code
-            ))
-        } else {
-            tokio::task::spawn_blocking(move || {
-                to.flush().context("Failed to fluch the downoad buffer")?;
-                Ok::<_, anyhow::Error>(to)
-            })
-            .await
-            .context("Failed to joim the download buffer flush task")?
-        }
-    }
-
-    async fn delete_relish(&self, path: &Self::RelishStoragePath) -> anyhow::Result<()> {
-        let (_, code) = self
-            .bucket
-            .delete_object(path.key())
-            .await
-            .with_context(|| format!("Failed to delete s3 object with key {}", path.key()))?;
-        if code != 200 {
-            Err(anyhow::format_err!(
-                "Received non-200 exit code during deleting object with key '{}', code: {}",
-                path.key(),
-                code
-            ))
-        } else {
-            Ok(())
-        }
-    }
-
-    async fn upload_relish<R: tokio::io::AsyncRead + std::marker::Unpin + Send>(
-        &self,
-        from: &mut tokio::io::BufReader<R>,
-        to: &Self::RelishStoragePath,
-    ) -> anyhow::Result<()> {
-        let code = self
-            .bucket
-            .put_object_stream(from, to.key())
-            .await
-            .with_context(|| format!("Failed to create s3 object with key {}", to.key()))?;
-        if code != 200 {
-            Err(anyhow::format_err!(
-                "Received non-200 exit code during creating object with key '{}', code: {}",
-                to.key(),
-                code
-            ))
-        } else {
-            Ok(())
-        }
-    }
-}

pageserver/src/relish_storage/synced_storage.rs

@@ -1,57 +0,0 @@
-use std::time::Duration;
-use std::{collections::BinaryHeap, sync::Mutex, thread};
-
-use crate::tenant_mgr;
-use crate::{relish_storage::RelishStorage, PageServerConf};
-
-lazy_static::lazy_static! {
-    static ref UPLOAD_QUEUE: Mutex<BinaryHeap<SyncTask>> = Mutex::new(BinaryHeap::new());
-}
-
-pub fn schedule_timeline_upload(_local_timeline: ()) {
-    // UPLOAD_QUEUE
-    //     .lock()
-    //     .unwrap()
-    //     .push(SyncTask::Upload(local_timeline))
-}
-
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
-enum SyncTask {}
-
-pub fn run_storage_sync_thread<
-    P: std::fmt::Debug,
-    S: 'static + RelishStorage<RelishStoragePath = P>,
->(
-    config: &'static PageServerConf,
-    relish_storage: S,
-    max_concurrent_sync: usize,
-) -> anyhow::Result<Option<thread::JoinHandle<anyhow::Result<()>>>> {
-    let runtime = tokio::runtime::Builder::new_current_thread()
-        .enable_all()
-        .build()?;
-
-    let handle = thread::Builder::new()
-        .name("Queue based relish storage sync".to_string())
-        .spawn(move || {
-            while !tenant_mgr::shutdown_requested() {
-                let mut queue_accessor = UPLOAD_QUEUE.lock().unwrap();
-                log::debug!("Upload queue length: {}", queue_accessor.len());
-                let next_task = queue_accessor.pop();
-                drop(queue_accessor);
-                match next_task {
-                    Some(task) => runtime.block_on(async {
-                        // suppress warnings
-                        let _ = (config, task, &relish_storage, max_concurrent_sync);
-                        todo!("omitted for brevity")
-                    }),
-                    None => {
-                        thread::sleep(Duration::from_secs(1));
-                        continue;
-                    }
-                }
-            }
-            log::debug!("Queue based relish storage sync thread shut down");
-            Ok(())
-        })?;
-    Ok(Some(handle))
-}

pageserver/src/rocksdb_storage.rs

@@ -0,0 +1,443 @@
+//!
+//! An implementation of the ObjectStore interface, backed by RocksDB
+//!
+use crate::object_key::*;
+use crate::object_store::ObjectStore;
+use crate::repository::RelTag;
+use crate::PageServerConf;
+use crate::ZTimelineId;
+use anyhow::{bail, Result};
+use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
+use std::sync::{Arc, Mutex};
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct StorageKey {
+    obj_key: ObjectKey,
+    lsn: Lsn,
+}
+
+impl StorageKey {
+    /// The first key for a given timeline
+    fn timeline_start(timeline: ZTimelineId) -> Self {
+        Self {
+            obj_key: ObjectKey {
+                timeline,
+                tag: ObjectTag::TimelineMetadataTag,
+            },
+            lsn: Lsn(0),
+        }
+    }
+}
+
+///
+/// RocksDB very inefficiently delete random record. Instead of it we have to use merge
+/// filter, which allows to throw away records at LSM merge phase.
+/// Unfortunately, it is hard (if ever possible)  to determine whether version can be removed
+/// at merge time. Version ca be removed if:
+/// 1. It is above PITR horizon (we need to get current LSN and gc_horizon from config)
+/// 2. Page is reconstructed at horizon (all WAL records above horizon are applied and can be removed)
+///
+/// So we have GC process which reconstructs pages at horizon and mark deteriorated WAL record
+/// for deletion. To mark object for deletion we can either set some flag in object itself.
+/// But it is complicated with new object value format, because RocksDB storage knows nothing about
+/// this format. Also updating whole record just to set one bit seems to be inefficient in any case.
+/// This is why we keep keys of marked for deletion versions in HashSet in memory.
+/// When LSM merge filter found key in this map, it removes it from the set preventing memory overflow.
+///
+struct GarbageCollector {
+    garbage: Mutex<HashSet<Vec<u8>>>,
+}
+
+impl GarbageCollector {
+    fn new() -> GarbageCollector {
+        GarbageCollector {
+            garbage: Mutex::new(HashSet::new()),
+        }
+    }
+
+    /// Called by GC to mark version as delete
+    fn mark_for_deletion(&self, key: &[u8]) {
+        let mut garbage = self.garbage.lock().unwrap();
+        garbage.insert(key.to_vec());
+    }
+
+    /// Called by LSM merge filter. If it finds key in the set, then
+    /// it doesn't merge it and removes from this set.
+    fn was_deleted(&self, key: &[u8]) -> bool {
+        let key = key.to_vec();
+        let mut garbage = self.garbage.lock().unwrap();
+        garbage.remove(&key)
+    }
+}
+
+pub struct RocksObjectStore {
+    _conf: &'static PageServerConf,
+
+    // RocksDB handle
+    db: rocksdb::DB,
+    gc: Arc<GarbageCollector>,
+}
+
+impl ObjectStore for RocksObjectStore {
+    fn get(&self, key: &ObjectKey, lsn: Lsn) -> Result<Vec<u8>> {
+        let val = self.db.get(StorageKey::ser(&StorageKey {
+            obj_key: key.clone(),
+            lsn,
+        })?)?;
+        if let Some(val) = val {
+            Ok(val)
+        } else {
+            bail!("could not find page {:?}", key);
+        }
+    }
+
+    fn get_next_key(&self, key: &ObjectKey) -> Result<Option<ObjectKey>> {
+        let mut iter = self.db.raw_iterator();
+        let search_key = StorageKey {
+            obj_key: key.clone(),
+            lsn: Lsn(0),
+        };
+        iter.seek(search_key.ser()?);
+        if !iter.valid() {
+            Ok(None)
+        } else {
+            let key = StorageKey::des(iter.key().unwrap())?;
+            Ok(Some(key.obj_key.clone()))
+        }
+    }
+
+    fn put(&self, key: &ObjectKey, lsn: Lsn, value: &[u8]) -> Result<()> {
+        self.db.put(
+            StorageKey::ser(&StorageKey {
+                obj_key: key.clone(),
+                lsn,
+            })?,
+            value,
+        )?;
+        Ok(())
+    }
+
+    fn unlink(&self, key: &ObjectKey, lsn: Lsn) -> Result<()> {
+        self.gc.mark_for_deletion(&StorageKey::ser(&StorageKey {
+            obj_key: key.clone(),
+            lsn,
+        })?);
+        Ok(())
+    }
+
+    /// Iterate through page versions of given page, starting from the given LSN.
+    /// The versions are walked in descending LSN order.
+    fn object_versions<'a>(
+        &'a self,
+        key: &ObjectKey,
+        lsn: Lsn,
+    ) -> Result<Box<dyn Iterator<Item = (Lsn, Vec<u8>)> + 'a>> {
+        let iter = RocksObjectVersionIter::new(&self.db, key, lsn)?;
+        Ok(Box::new(iter))
+    }
+
+    /// Iterate through all timeline objects
+    fn list_objects<'a>(
+        &'a self,
+        timeline: ZTimelineId,
+        nonrel_only: bool,
+        lsn: Lsn,
+    ) -> Result<Box<dyn Iterator<Item = ObjectTag> + 'a>> {
+        let iter = RocksObjectIter::new(&self.db, timeline, nonrel_only, lsn)?;
+        Ok(Box::new(iter))
+    }
+
+    /// Get a list of all distinct relations in given tablespace and database.
+    ///
+    /// TODO: This implementation is very inefficient, it scans
+    /// through all entries in the given database. In practice, this
+    /// is used for CREATE DATABASE, and usually the template database is small.
+    /// But if it's not, this will be slow.
+    fn list_rels(
+        &self,
+        timelineid: ZTimelineId,
+        spcnode: u32,
+        dbnode: u32,
+        lsn: Lsn,
+    ) -> Result<HashSet<RelTag>> {
+        // FIXME: This scans everything. Very slow
+
+        let mut rels: HashSet<RelTag> = HashSet::new();
+
+        let mut search_rel_tag = RelTag {
+            spcnode,
+            dbnode,
+            relnode: 0,
+            forknum: 0u8,
+        };
+        let mut iter = self.db.raw_iterator();
+        loop {
+            let search_key = StorageKey {
+                obj_key: ObjectKey {
+                    timeline: timelineid,
+                    tag: ObjectTag::RelationMetadata(search_rel_tag),
+                },
+                lsn: Lsn(0),
+            };
+            iter.seek(search_key.ser()?);
+            if !iter.valid() {
+                break;
+            }
+            let key = StorageKey::des(iter.key().unwrap())?;
+
+            if let ObjectTag::RelationMetadata(rel_tag) = key.obj_key.tag {
+                if spcnode != 0 && rel_tag.spcnode != spcnode
+                    || dbnode != 0 && rel_tag.dbnode != dbnode
+                {
+                    break;
+                }
+                if key.lsn <= lsn {
+                    // visible in this snapshot
+                    rels.insert(rel_tag);
+                }
+                search_rel_tag = rel_tag;
+                // skip to next relation
+                // FIXME: What if relnode is u32::MAX ?
+                search_rel_tag.relnode += 1;
+            } else {
+                // no more relation metadata entries
+                break;
+            }
+        }
+
+        Ok(rels)
+    }
+
+    /// Iterate through versions of all objects in a timeline.
+    ///
+    /// Returns objects in increasing key-version order.
+    /// Returns all versions up to and including the specified LSN.
+    fn objects<'a>(
+        &'a self,
+        timeline: ZTimelineId,
+        lsn: Lsn,
+    ) -> Result<Box<dyn Iterator<Item = Result<(ObjectTag, Lsn, Vec<u8>)>> + 'a>> {
+        let start_key = StorageKey::timeline_start(timeline);
+        let start_key_bytes = StorageKey::ser(&start_key)?;
+        let iter = self.db.iterator(rocksdb::IteratorMode::From(
+            &start_key_bytes,
+            rocksdb::Direction::Forward,
+        ));
+
+        Ok(Box::new(RocksObjects {
+            iter,
+            timeline,
+            lsn,
+        }))
+    }
+
+    fn compact(&self) {
+        self.db.compact_range::<&[u8], &[u8]>(None, None);
+    }
+}
+
+impl RocksObjectStore {
+    /// Open a RocksDB database.
+    pub fn open(conf: &'static PageServerConf) -> Result<RocksObjectStore> {
+        let opts = Self::get_rocksdb_opts();
+        let obj_store = Self::new(conf, opts)?;
+        Ok(obj_store)
+    }
+
+    /// Create a new, empty RocksDB database.
+    pub fn create(conf: &'static PageServerConf) -> Result<RocksObjectStore> {
+        let path = conf.workdir.join("rocksdb-storage");
+        std::fs::create_dir(&path)?;
+
+        let mut opts = Self::get_rocksdb_opts();
+        opts.create_if_missing(true);
+        opts.set_error_if_exists(true);
+        let obj_store = Self::new(conf, opts)?;
+        Ok(obj_store)
+    }
+
+    fn new(conf: &'static PageServerConf, mut opts: rocksdb::Options) -> Result<RocksObjectStore> {
+        let path = conf.workdir.join("rocksdb-storage");
+        let gc = Arc::new(GarbageCollector::new());
+        let gc_ref = gc.clone();
+        opts.set_compaction_filter("ttl", move |_level: u32, key: &[u8], _val: &[u8]| {
+            if gc_ref.was_deleted(key) {
+                rocksdb::compaction_filter::Decision::Remove
+            } else {
+                rocksdb::compaction_filter::Decision::Keep
+            }
+        });
+        let db = rocksdb::DB::open(&opts, &path)?;
+        let obj_store = RocksObjectStore {
+            _conf: conf,
+            db,
+            gc,
+        };
+        Ok(obj_store)
+    }
+
+    /// common options used by `open` and `create`
+    fn get_rocksdb_opts() -> rocksdb::Options {
+        let mut opts = rocksdb::Options::default();
+        opts.set_use_fsync(true);
+        opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
+        opts
+    }
+}
+
+///
+/// Iterator for `object_versions`. Returns all page versions of a given block, in
+/// reverse LSN order.
+///
+struct RocksObjectVersionIter<'a> {
+    obj_key: ObjectKey,
+    dbiter: rocksdb::DBRawIterator<'a>,
+    first_call: bool,
+}
+impl<'a> RocksObjectVersionIter<'a> {
+    fn new(
+        db: &'a rocksdb::DB,
+        obj_key: &ObjectKey,
+        lsn: Lsn,
+    ) -> Result<RocksObjectVersionIter<'a>> {
+        let key = StorageKey {
+            obj_key: obj_key.clone(),
+            lsn,
+        };
+        let mut dbiter = db.raw_iterator();
+        dbiter.seek_for_prev(StorageKey::ser(&key)?); // locate last entry
+        Ok(RocksObjectVersionIter {
+            first_call: true,
+            obj_key: obj_key.clone(),
+            dbiter,
+        })
+    }
+}
+impl<'a> Iterator for RocksObjectVersionIter<'a> {
+    type Item = (Lsn, Vec<u8>);
+
+    fn next(&mut self) -> std::option::Option<Self::Item> {
+        if self.first_call {
+            self.first_call = false;
+        } else {
+            self.dbiter.prev(); // walk backwards
+        }
+
+        if !self.dbiter.valid() {
+            return None;
+        }
+        let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
+        if key.obj_key.tag != self.obj_key.tag {
+            return None;
+        }
+        let val = self.dbiter.value().unwrap();
+        let result = val.to_vec();
+
+        Some((key.lsn, result))
+    }
+}
+
+struct RocksObjects<'r> {
+    iter: rocksdb::DBIterator<'r>,
+    timeline: ZTimelineId,
+    lsn: Lsn,
+}
+
+impl<'r> Iterator for RocksObjects<'r> {
+    // TODO consider returning Box<[u8]>
+    type Item = Result<(ObjectTag, Lsn, Vec<u8>)>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.next_result().transpose()
+    }
+}
+
+impl<'r> RocksObjects<'r> {
+    fn next_result(&mut self) -> Result<Option<(ObjectTag, Lsn, Vec<u8>)>> {
+        for (key_bytes, v) in &mut self.iter {
+            let key = StorageKey::des(&key_bytes)?;
+
+            if key.obj_key.timeline != self.timeline {
+                return Ok(None);
+            }
+
+            if key.lsn > self.lsn {
+                // TODO can speed up by seeking iterator
+                continue;
+            }
+
+            return Ok(Some((key.obj_key.tag, key.lsn, v.to_vec())));
+        }
+
+        Ok(None)
+    }
+}
+
+///
+/// Iterator for `list_objects`. Returns all objects preceeding specified LSN
+///
+struct RocksObjectIter<'a> {
+    timeline: ZTimelineId,
+    key: StorageKey,
+    nonrel_only: bool,
+    lsn: Lsn,
+    dbiter: rocksdb::DBRawIterator<'a>,
+}
+impl<'a> RocksObjectIter<'a> {
+    fn new(
+        db: &'a rocksdb::DB,
+        timeline: ZTimelineId,
+        nonrel_only: bool,
+        lsn: Lsn,
+    ) -> Result<RocksObjectIter<'a>> {
+        let key = StorageKey {
+            obj_key: ObjectKey {
+                timeline,
+                tag: ObjectTag::FirstTag,
+            },
+            lsn: Lsn(0),
+        };
+        let dbiter = db.raw_iterator();
+        Ok(RocksObjectIter {
+            key,
+            timeline,
+            nonrel_only,
+            lsn,
+            dbiter,
+        })
+    }
+}
+impl<'a> Iterator for RocksObjectIter<'a> {
+    type Item = ObjectTag;
+
+    fn next(&mut self) -> std::option::Option<Self::Item> {
+        loop {
+            self.dbiter.seek(StorageKey::ser(&self.key).unwrap());
+            if !self.dbiter.valid() {
+                return None;
+            }
+            let key = StorageKey::des(self.dbiter.key().unwrap()).unwrap();
+            if key.obj_key.timeline != self.timeline {
+                // End of this timeline
+                return None;
+            }
+            self.key = key.clone();
+            self.key.lsn = Lsn(u64::MAX); // next seek should skip all versions
+            if key.lsn <= self.lsn {
+                // visible in this snapshot
+                if self.nonrel_only {
+                    match key.obj_key.tag {
+                        ObjectTag::RelationMetadata(_) => return None,
+                        ObjectTag::RelationBuffer(_) => return None,
+                        _ => return Some(key.obj_key.tag),
+                    }
+                } else {
+                    return Some(key.obj_key.tag);
+                }
+            }
+        }
+    }
+}

pageserver/src/tenant_mgr.rs

@@ -1,182 +0,0 @@
-//! This module acts as a switchboard to access different repositories managed by this
-//! page server.
-
-use crate::branches;
-use crate::layered_repository::LayeredRepository;
-use crate::repository::{Repository, Timeline};
-use crate::walredo::PostgresRedoManager;
-use crate::PageServerConf;
-use anyhow::{anyhow, bail, Context, Result};
-use lazy_static::lazy_static;
-use log::{debug, info};
-use std::collections::hash_map::Entry;
-use std::collections::HashMap;
-use std::fs;
-use std::str::FromStr;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, Mutex, MutexGuard};
-use std::thread::JoinHandle;
-use zenith_utils::zid::{ZTenantId, ZTimelineId};
-
-lazy_static! {
-    static ref REPOSITORY: Mutex<HashMap<ZTenantId, Arc<dyn Repository>>> =
-        Mutex::new(HashMap::new());
-}
-
-fn access_repository() -> MutexGuard<'static, HashMap<ZTenantId, Arc<dyn Repository>>> {
-    REPOSITORY.lock().unwrap()
-}
-struct TenantHandleEntry {
-    checkpointer_handle: Option<JoinHandle<()>>,
-    gc_handle: Option<JoinHandle<()>>,
-}
-
-// Logically these handles belong to Repository,
-// but it's just simpler to store them separately
-lazy_static! {
-    static ref TENANT_HANDLES: Mutex<HashMap<ZTenantId, TenantHandleEntry>> =
-        Mutex::new(HashMap::new());
-}
-
-static SHUTDOWN_REQUESTED: AtomicBool = AtomicBool::new(false);
-
-pub fn init(conf: &'static PageServerConf) {
-    let mut m = access_repository();
-    for dir_entry in fs::read_dir(conf.tenants_path()).unwrap() {
-        let tenantid =
-            ZTenantId::from_str(dir_entry.unwrap().file_name().to_str().unwrap()).unwrap();
-        let repo = init_repo(conf, tenantid);
-        info!("initialized storage for tenant: {}", &tenantid);
-        m.insert(tenantid, repo);
-    }
-}
-
-fn init_repo(conf: &'static PageServerConf, tenant_id: ZTenantId) -> Arc<LayeredRepository> {
-    // Set up a WAL redo manager, for applying WAL records.
-    let walredo_mgr = PostgresRedoManager::new(conf, tenant_id);
-
-    // Set up an object repository, for actual data storage.
-    let repo = Arc::new(LayeredRepository::new(
-        conf,
-        Arc::new(walredo_mgr),
-        tenant_id,
-        true,
-    ));
-
-    let checkpointer_handle = LayeredRepository::launch_checkpointer_thread(conf, repo.clone());
-    let gc_handle = LayeredRepository::launch_gc_thread(conf, repo.clone());
-
-    let mut handles = TENANT_HANDLES.lock().unwrap();
-    let h = TenantHandleEntry {
-        checkpointer_handle: Some(checkpointer_handle),
-        gc_handle: Some(gc_handle),
-    };
-
-    handles.insert(tenant_id, h);
-
-    repo
-}
-
-// TODO kb Currently unused function, will later be used when the relish storage downloads a new layer.
-// Relevant PR: https://github.com/zenithdb/zenith/pull/686
-pub fn register_relish_download(
-    conf: &'static PageServerConf,
-    tenant_id: ZTenantId,
-    timeline_id: ZTimelineId,
-) {
-    log::info!(
-        "Registering new download, tenant id {}, timeline id: {}",
-        tenant_id,
-        timeline_id
-    );
-    match access_repository().entry(tenant_id) {
-        Entry::Occupied(o) => init_timeline(o.get().as_ref(), timeline_id),
-        Entry::Vacant(v) => {
-            log::info!("New repo initialized");
-            let new_repo = init_repo(conf, tenant_id);
-            init_timeline(new_repo.as_ref(), timeline_id);
-            v.insert(new_repo);
-        }
-    }
-}
-
-fn init_timeline(repo: &dyn Repository, timeline_id: ZTimelineId) {
-    match repo.get_timeline(timeline_id) {
-        Ok(_timeline) => log::info!("Successfully initialized timeline {}", timeline_id),
-        Err(e) => log::error!("Failed to init timeline {}, reason: {:#}", timeline_id, e),
-    }
-}
-
-// Check this flag in the thread loops to know when to exit
-pub fn shutdown_requested() -> bool {
-    SHUTDOWN_REQUESTED.load(Ordering::Relaxed)
-}
-
-pub fn stop_tenant_threads(tenantid: ZTenantId) {
-    let mut handles = TENANT_HANDLES.lock().unwrap();
-    if let Some(h) = handles.get_mut(&tenantid) {
-        h.checkpointer_handle.take().map(JoinHandle::join);
-        debug!("checkpointer for tenant {} has stopped", tenantid);
-        h.gc_handle.take().map(JoinHandle::join);
-        debug!("gc for tenant {} has stopped", tenantid);
-    }
-}
-
-pub fn shutdown_all_tenants() -> Result<()> {
-    SHUTDOWN_REQUESTED.swap(true, Ordering::Relaxed);
-
-    let tenants = list_tenants()?;
-    for tenantid in tenants {
-        stop_tenant_threads(tenantid);
-        let repo = get_repository_for_tenant(tenantid)?;
-        debug!("shutdown tenant {}", tenantid);
-        repo.shutdown()?;
-    }
-
-    Ok(())
-}
-
-pub fn create_repository_for_tenant(
-    conf: &'static PageServerConf,
-    tenantid: ZTenantId,
-) -> Result<()> {
-    let mut m = access_repository();
-
-    // First check that the tenant doesn't exist already
-    if m.get(&tenantid).is_some() {
-        bail!("tenant {} already exists", tenantid);
-    }
-    let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenantid));
-    let repo = branches::create_repo(conf, tenantid, wal_redo_manager)?;
-
-    m.insert(tenantid, repo);
-
-    Ok(())
-}
-
-pub fn get_repository_for_tenant(tenantid: ZTenantId) -> Result<Arc<dyn Repository>> {
-    access_repository()
-        .get(&tenantid)
-        .map(Arc::clone)
-        .ok_or_else(|| anyhow!("repository not found for tenant name {}", tenantid))
-}
-
-pub fn get_timeline_for_tenant(
-    tenantid: ZTenantId,
-    timelineid: ZTimelineId,
-) -> Result<Arc<dyn Timeline>> {
-    get_repository_for_tenant(tenantid)?
-        .get_timeline(timelineid)
-        .with_context(|| format!("cannot fetch timeline {}", timelineid))
-}
-
-fn list_tenants() -> Result<Vec<ZTenantId>> {
-    let o = &mut REPOSITORY.lock().unwrap();
-
-    o.iter()
-        .map(|tenant| {
-            let (tenantid, _) = tenant;
-            Ok(*tenantid)
-        })
-        .collect()
-}

pageserver/src/tui.rs

@@ -0,0 +1,307 @@
+use crate::tui_event::{Event, Events};
+use crate::tui_logger::TuiLogger;
+use crate::tui_logger::TuiLoggerWidget;
+
+use lazy_static::lazy_static;
+use std::sync::Arc;
+use std::{error::Error, io};
+use termion::{event::Key, input::MouseTerminal, raw::IntoRawMode, screen::AlternateScreen};
+use tui::backend::TermionBackend;
+use tui::buffer::Buffer;
+use tui::layout::{Constraint, Direction, Layout, Rect};
+use tui::style::{Color, Modifier, Style};
+use tui::text::{Span, Spans, Text};
+use tui::widgets::{Block, BorderType, Borders, Paragraph, Widget};
+use tui::Terminal;
+
+use slog::Drain;
+
+lazy_static! {
+    pub static ref PAGESERVICE_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
+    pub static ref WALRECEIVER_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
+    pub static ref WALREDO_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
+    pub static ref CATCHALL_DRAIN: Arc<TuiLogger> = Arc::new(TuiLogger::default());
+}
+
+pub fn init_logging() -> slog_scope::GlobalLoggerGuard {
+    let pageservice_drain =
+        slog::Filter::new(PAGESERVICE_DRAIN.as_ref(), |record: &slog::Record| {
+            if record.level().is_at_least(slog::Level::Debug)
+                && record.module().starts_with("pageserver::page_service")
+            {
+                return true;
+            }
+            false
+        })
+        .fuse();
+
+    let walredo_drain = slog::Filter::new(WALREDO_DRAIN.as_ref(), |record: &slog::Record| {
+        if record.level().is_at_least(slog::Level::Debug)
+            && record.module().starts_with("pageserver::walredo")
+        {
+            return true;
+        }
+        false
+    })
+    .fuse();
+
+    let walreceiver_drain =
+        slog::Filter::new(WALRECEIVER_DRAIN.as_ref(), |record: &slog::Record| {
+            if record.level().is_at_least(slog::Level::Debug)
+                && record.module().starts_with("pageserver::walreceiver")
+            {
+                return true;
+            }
+            false
+        })
+        .fuse();
+
+    let catchall_drain = slog::Filter::new(CATCHALL_DRAIN.as_ref(), |record: &slog::Record| {
+        if record.level().is_at_least(slog::Level::Info) {
+            return true;
+        }
+        if record.level().is_at_least(slog::Level::Debug)
+            && record.module().starts_with("pageserver")
+        {
+            return true;
+        }
+        false
+    })
+    .fuse();
+
+    let drain = pageservice_drain;
+    let drain = slog::Duplicate::new(drain, walreceiver_drain).fuse();
+    let drain = slog::Duplicate::new(drain, walredo_drain).fuse();
+    let drain = slog::Duplicate::new(drain, catchall_drain).fuse();
+    let drain = slog_async::Async::new(drain).chan_size(1000).build().fuse();
+    let drain = slog::Filter::new(drain, |record: &slog::Record| {
+        if record.level().is_at_least(slog::Level::Info) {
+            return true;
+        }
+        if record.level().is_at_least(slog::Level::Debug)
+            && record.module().starts_with("pageserver")
+        {
+            return true;
+        }
+
+        false
+    })
+    .fuse();
+    let logger = slog::Logger::root(drain, slog::o!());
+    slog_scope::set_global_logger(logger)
+}
+
+pub fn ui_main() -> Result<(), Box<dyn Error>> {
+    // Terminal initialization
+    let stdout = io::stdout().into_raw_mode()?;
+    let stdout = MouseTerminal::from(stdout);
+    let stdout = AlternateScreen::from(stdout);
+    let backend = TermionBackend::new(stdout);
+    let mut terminal = Terminal::new(backend)?;
+
+    // Setup event handlers
+    let events = Events::new();
+
+    loop {
+        terminal.draw(|f| {
+            let size = f.size();
+
+            // +----------------+----------------+
+            // |                |                |
+            // |  top_top_left  | top_top_right  |
+            // |                |                |
+            // +----------------+----------------|
+            // |                |                |
+            // |  top_bot_left  | top_left_right |
+            // |                |                |
+            // +----------------+----------------+
+            // |                                 |
+            // |             bottom              |
+            // |                                 |
+            // +---------------------------------+
+            let chunks = Layout::default()
+                .direction(Direction::Vertical)
+                .constraints([Constraint::Percentage(70), Constraint::Percentage(30)].as_ref())
+                .split(size);
+            let top_chunk = chunks[0];
+            let bottom_chunk = chunks[1];
+
+            let top_chunks = Layout::default()
+                .direction(Direction::Horizontal)
+                .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
+                .split(top_chunk);
+            let top_left_chunk = top_chunks[0];
+            let top_right_chunk = top_chunks[1];
+
+            let c = Layout::default()
+                .direction(Direction::Vertical)
+                .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
+                .split(top_left_chunk);
+            let top_top_left_chunk = c[0];
+            let top_bot_left_chunk = c[1];
+
+            let c = Layout::default()
+                .direction(Direction::Vertical)
+                .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
+                .split(top_right_chunk);
+            let top_top_right_chunk = c[0];
+            let top_bot_right_chunk = c[1];
+
+            f.render_widget(
+                LogWidget::new(PAGESERVICE_DRAIN.as_ref(), "Page Service"),
+                top_top_left_chunk,
+            );
+
+            f.render_widget(
+                LogWidget::new(WALREDO_DRAIN.as_ref(), "WAL Redo"),
+                top_bot_left_chunk,
+            );
+
+            f.render_widget(
+                LogWidget::new(WALRECEIVER_DRAIN.as_ref(), "WAL Receiver"),
+                top_top_right_chunk,
+            );
+
+            f.render_widget(MetricsWidget {}, top_bot_right_chunk);
+
+            f.render_widget(
+                LogWidget::new(CATCHALL_DRAIN.as_ref(), "All Log").show_module(true),
+                bottom_chunk,
+            );
+        })?;
+
+        // If ther user presses 'q', quit.
+
+        // silence clippy's suggestion to rewrite this as an if-statement. Match
+        // makes more sense as soon as we get another command than 'q'.
+        #[allow(clippy::single_match)]
+        #[allow(clippy::collapsible_match)]
+        if let Event::Input(key) = events.next()? {
+            match key {
+                Key::Char('q') => {
+                    break;
+                }
+                _ => (),
+            }
+        }
+    }
+
+    terminal.show_cursor().unwrap();
+    terminal.clear().unwrap();
+
+    Ok(())
+}
+
+#[allow(dead_code)]
+struct LogWidget<'a> {
+    logger: &'a TuiLogger,
+    title: &'a str,
+    show_module: bool,
+}
+
+impl<'a> LogWidget<'a> {
+    fn new(logger: &'a TuiLogger, title: &'a str) -> LogWidget<'a> {
+        LogWidget {
+            logger,
+            title,
+            show_module: false,
+        }
+    }
+
+    fn show_module(mut self, b: bool) -> LogWidget<'a> {
+        self.show_module = b;
+        self
+    }
+}
+
+impl<'a> Widget for LogWidget<'a> {
+    fn render(self, area: Rect, buf: &mut Buffer) {
+        let w = TuiLoggerWidget::default(self.logger)
+            .block(
+                Block::default()
+                    .borders(Borders::ALL)
+                    .title(self.title)
+                    .border_type(BorderType::Rounded),
+            )
+            .show_module(true)
+            .style_error(Style::default().fg(Color::Red))
+            .style_warn(Style::default().fg(Color::Yellow))
+            .style_info(Style::default().fg(Color::Green));
+        w.render(area, buf);
+    }
+}
+
+// Render a widget to show some metrics
+struct MetricsWidget {}
+
+fn _get_metric_u64(title: &str, value: u64) -> Spans {
+    Spans::from(vec![
+        Span::styled(format!("{:<20}", title), Style::default()),
+        Span::raw(": "),
+        Span::styled(
+            value.to_string(),
+            Style::default().add_modifier(Modifier::BOLD),
+        ),
+    ])
+}
+
+// This is not used since LSNs were removed from page cache stats.
+// Maybe it will be used in the future?
+fn _get_metric_str<'a>(title: &str, value: &'a str) -> Spans<'a> {
+    Spans::from(vec![
+        Span::styled(format!("{:<20}", title), Style::default()),
+        Span::raw(": "),
+        Span::styled(value, Style::default().add_modifier(Modifier::BOLD)),
+    ])
+}
+
+impl tui::widgets::Widget for MetricsWidget {
+    fn render(self, area: Rect, buf: &mut Buffer) {
+        let block = Block::default()
+            .borders(Borders::ALL)
+            .title("Page Cache Metrics")
+            .border_type(BorderType::Rounded);
+        let inner_area = block.inner(area);
+
+        block.render(area, buf);
+
+        #[allow(unused_mut)]
+        let mut lines: Vec<Spans> = Vec::new();
+
+        // FIXME
+        //let page_cache_stats = crate::page_cache::get_stats();
+
+        // This is not used since LSNs were removed from page cache stats.
+        // Maybe it will be used in the future?
+        /*
+        let lsnrange = format!(
+            "{} - {}",
+            page_cache_stats.first_valid_lsn, page_cache_stats.last_valid_lsn
+        );
+        let last_valid_recordlsn_str = page_cache_stats.last_record_lsn.to_string();
+        lines.push(get_metric_str("Valid LSN range", &lsnrange));
+        lines.push(get_metric_str("Last record LSN", &last_valid_recordlsn_str));
+        */
+        /*
+        lines.push(get_metric_u64(
+            "# of cache entries",
+            page_cache_stats.num_entries,
+        ));
+        lines.push(get_metric_u64(
+            "# of page images",
+            page_cache_stats.num_page_images,
+        ));
+        lines.push(get_metric_u64(
+            "# of WAL records",
+            page_cache_stats.num_wal_records,
+        ));
+        lines.push(get_metric_u64(
+            "# of GetPage@LSN calls",
+            page_cache_stats.num_getpage_requests,
+        ));
+        */
+        let text = Text::from(lines);
+
+        Paragraph::new(text).render(inner_area, buf);
+    }
+}

pageserver/src/tui_event.rs

@@ -0,0 +1,96 @@
+use std::io;
+use std::sync::mpsc;
+use std::sync::{
+    atomic::{AtomicBool, Ordering},
+    Arc,
+};
+use std::thread;
+use std::time::Duration;
+
+use termion::event::Key;
+use termion::input::TermRead;
+
+pub enum Event<I> {
+    Input(I),
+    Tick,
+}
+
+/// A small event handler that wrap termion input and tick events. Each event
+/// type is handled in its own thread and returned to a common `Receiver`
+#[allow(dead_code)]
+pub struct Events {
+    rx: mpsc::Receiver<Event<Key>>,
+    input_handle: thread::JoinHandle<()>,
+    ignore_exit_key: Arc<AtomicBool>,
+    tick_handle: thread::JoinHandle<()>,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct Config {
+    pub exit_key: Key,
+    pub tick_rate: Duration,
+}
+
+impl Default for Config {
+    fn default() -> Config {
+        Config {
+            exit_key: Key::Char('q'),
+            tick_rate: Duration::from_millis(250),
+        }
+    }
+}
+
+impl Events {
+    pub fn new() -> Events {
+        Events::with_config(Config::default())
+    }
+
+    pub fn with_config(config: Config) -> Events {
+        let (tx, rx) = mpsc::channel();
+        let ignore_exit_key = Arc::new(AtomicBool::new(false));
+        let input_handle = {
+            let tx = tx.clone();
+            let ignore_exit_key = ignore_exit_key.clone();
+            thread::spawn(move || {
+                let stdin = io::stdin();
+                for evt in stdin.keys() {
+                    // This will panic if stdin returns EOF.
+                    let key = evt.unwrap();
+                    if let Err(err) = tx.send(Event::Input(key)) {
+                        eprintln!("{}", err);
+                        return;
+                    }
+                    if !ignore_exit_key.load(Ordering::Relaxed) && key == config.exit_key {
+                        return;
+                    }
+                }
+            })
+        };
+        let tick_handle = {
+            thread::spawn(move || loop {
+                if tx.send(Event::Tick).is_err() {
+                    break;
+                }
+                thread::sleep(config.tick_rate);
+            })
+        };
+        Events {
+            rx,
+            input_handle,
+            ignore_exit_key,
+            tick_handle,
+        }
+    }
+
+    pub fn next(&self) -> Result<Event<Key>, mpsc::RecvError> {
+        self.rx.recv()
+    }
+
+    pub fn disable_exit_key(&mut self) {
+        self.ignore_exit_key.store(true, Ordering::Relaxed);
+    }
+
+    pub fn enable_exit_key(&mut self) {
+        self.ignore_exit_key.store(false, Ordering::Relaxed);
+    }
+}

pageserver/src/tui_logger.rs

@@ -0,0 +1,199 @@
+//
+// A TUI Widget that displays log entries
+//
+// This is heavily inspired by gin66's tui_logger crate at https://github.com/gin66/tui-logger,
+// but I wrote this based on the 'slog' module, which simplified things a lot. tui-logger also
+// implemented the slog Drain trait, but it had a model of one global buffer for the records.
+// With this implementation, each TuiLogger is a separate ring buffer and separate slog Drain.
+// Also, I didn't do any of the "hot log" stuff that gin66's implementation had, you can use an
+// AsyncDrain to buffer and handle overflow if desired.
+//
+use chrono::offset::Local;
+use chrono::DateTime;
+use slog::{Drain, Level, OwnedKVList, Record};
+use slog_async::AsyncRecord;
+use std::collections::VecDeque;
+use std::sync::Mutex;
+use std::time::SystemTime;
+use tui::buffer::Buffer;
+use tui::layout::Rect;
+use tui::style::{Modifier, Style};
+use tui::text::{Span, Spans};
+use tui::widgets::{Block, Paragraph, Widget, Wrap};
+
+// Size of the log ring buffer, in # of records
+static BUFFER_SIZE: usize = 1000;
+
+pub struct TuiLogger {
+    events: Mutex<VecDeque<(SystemTime, AsyncRecord)>>,
+}
+
+impl<'a> Default for TuiLogger {
+    fn default() -> TuiLogger {
+        TuiLogger {
+            events: Mutex::new(VecDeque::with_capacity(BUFFER_SIZE)),
+        }
+    }
+}
+
+impl Drain for TuiLogger {
+    type Ok = ();
+    type Err = slog::Error;
+
+    fn log(&self, record: &Record, values: &OwnedKVList) -> Result<Self::Ok, Self::Err> {
+        let mut events = self.events.lock().unwrap();
+
+        let now = SystemTime::now();
+        let asyncrec = AsyncRecord::from(record, values);
+        events.push_front((now, asyncrec));
+
+        if events.len() > BUFFER_SIZE {
+            events.pop_back();
+        }
+
+        Ok(())
+    }
+}
+
+// TuiLoggerWidget renders a TuiLogger ring buffer
+pub struct TuiLoggerWidget<'b> {
+    block: Option<Block<'b>>,
+    /// Base style of the widget
+    style: Style,
+    /// Level based style
+    style_error: Option<Style>,
+    style_warn: Option<Style>,
+    style_debug: Option<Style>,
+    style_trace: Option<Style>,
+    style_info: Option<Style>,
+    show_module: bool,
+    logger: &'b TuiLogger,
+}
+impl<'b> TuiLoggerWidget<'b> {
+    pub fn default(logger: &'b TuiLogger) -> TuiLoggerWidget<'b> {
+        TuiLoggerWidget {
+            block: None,
+            style: Default::default(),
+            style_error: None,
+            style_warn: None,
+            style_debug: None,
+            style_trace: None,
+            style_info: None,
+            show_module: true,
+            logger,
+        }
+    }
+}
+impl<'b> TuiLoggerWidget<'b> {
+    pub fn block(mut self, block: Block<'b>) -> TuiLoggerWidget<'b> {
+        self.block = Some(block);
+        self
+    }
+    #[allow(unused)]
+    pub fn style(mut self, style: Style) -> TuiLoggerWidget<'b> {
+        self.style = style;
+        self
+    }
+    pub fn style_error(mut self, style: Style) -> TuiLoggerWidget<'b> {
+        self.style_error = Some(style);
+        self
+    }
+    pub fn style_warn(mut self, style: Style) -> TuiLoggerWidget<'b> {
+        self.style_warn = Some(style);
+        self
+    }
+    pub fn style_info(mut self, style: Style) -> TuiLoggerWidget<'b> {
+        self.style_info = Some(style);
+        self
+    }
+    #[allow(unused)]
+    pub fn style_trace(mut self, style: Style) -> TuiLoggerWidget<'b> {
+        self.style_trace = Some(style);
+        self
+    }
+    #[allow(unused)]
+    pub fn style_debug(mut self, style: Style) -> TuiLoggerWidget<'b> {
+        self.style_debug = Some(style);
+        self
+    }
+
+    pub fn show_module(mut self, b: bool) -> TuiLoggerWidget<'b> {
+        self.show_module = b;
+        self
+    }
+}
+impl<'b> Widget for TuiLoggerWidget<'b> {
+    fn render(mut self, area: Rect, buf: &mut Buffer) {
+        buf.set_style(area, self.style);
+        let list_area = match self.block.take() {
+            Some(b) => {
+                let inner_area = b.inner(area);
+                b.render(area, buf);
+                inner_area
+            }
+            None => area,
+        };
+        if list_area.width == 0 || list_area.height == 0 {
+            return;
+        }
+
+        let la_height = list_area.height as usize;
+
+        //
+        // Iterate through the records in the buffer. The records are
+        // pushed to the front, so the newest records come first.
+        //
+        let mut lines: Vec<Spans> = Vec::new();
+
+        let style_msg = Style::default().add_modifier(Modifier::BOLD);
+        {
+            let events = self.logger.events.lock().unwrap();
+
+            for evt in events.iter() {
+                let (timestamp, rec) = evt;
+
+                rec.as_record_values(|rec, _kwlist| {
+                    let mut line: Vec<Span> = Vec::new();
+
+                    let datetime: DateTime<Local> = timestamp.clone().into();
+                    let ts = format!("{}", datetime.format("%H:%M:%S%.3f "));
+                    line.push(Span::raw(ts));
+
+                    let (lvl_style, txt, with_loc) = match rec.level() {
+                        Level::Critical => (self.style_error, "CRIT ", true),
+                        Level::Error => (self.style_error, "ERROR", true),
+                        Level::Warning => (self.style_warn, "WARN ", true),
+                        Level::Info => (self.style_info, "INFO ", false),
+                        Level::Debug => (self.style_debug, "DEBUG", true),
+                        Level::Trace => (self.style_trace, "TRACE", true),
+                    };
+                    line.push(Span::styled(txt, lvl_style.unwrap_or_default()));
+
+                    if self.show_module {
+                        line.push(Span::raw(" "));
+                        line.push(Span::raw(rec.module()));
+                    }
+                    if with_loc {
+                        let loc = format!(" {}:{}", rec.file(), rec.line());
+                        line.push(Span::raw(loc));
+                    }
+                    let msg = format!(" {}", rec.msg());
+                    line.push(Span::styled(msg, style_msg));
+
+                    lines.push(Spans::from(line));
+                });
+                if lines.len() == la_height {
+                    break;
+                }
+            }
+        }
+
+        lines.reverse();
+
+        let text = tui::text::Text::from(lines);
+
+        Paragraph::new(text)
+            .wrap(Wrap { trim: true })
+            .render(list_area, buf);
+    }
+}

pageserver/src/waldecoder.rs

@@ -1,21 +1,26 @@
 //!
 //! WAL decoder. For each WAL record, it decodes the record to figure out which data blocks
-//! the record affects, so that they can be stored in repository.
+//! the record affects, to add the records to the page cache.
 //!
 use bytes::{Buf, BufMut, Bytes, BytesMut};
-use crc32c::*;
 use log::*;
 use postgres_ffi::pg_constants;
 use postgres_ffi::xlog_utils::*;
 use postgres_ffi::XLogLongPageHeaderData;
 use postgres_ffi::XLogPageHeaderData;
 use postgres_ffi::XLogRecord;
-use postgres_ffi::{BlockNumber, OffsetNumber};
-use postgres_ffi::{MultiXactId, MultiXactOffset, MultiXactStatus, Oid, TransactionId};
 use std::cmp::min;
 use thiserror::Error;
 use zenith_utils::lsn::Lsn;
 
+pub type Oid = u32;
+pub type TransactionId = u32;
+pub type BlockNumber = u32;
+pub type OffsetNumber = u16;
+pub type MultiXactId = TransactionId;
+pub type MultiXactOffset = u32;
+pub type MultiXactStatus = u32;
+
 #[allow(dead_code)]
 pub struct WalStreamDecoder {
     lsn: Lsn,
@@ -54,11 +59,6 @@ impl WalStreamDecoder {
         }
     }
 
-    // The latest LSN position fed to the decoder.
-    pub fn available(&self) -> Lsn {
-        self.lsn + self.inputbuf.remaining() as u64
-    }
-
     pub fn feed_bytes(&mut self, buf: &[u8]) {
         self.inputbuf.extend_from_slice(buf);
     }
@@ -72,10 +72,6 @@ impl WalStreamDecoder {
     ///     Err(WalDecodeError): an error occured while decoding, meaning the input was invalid.
     ///
     pub fn poll_decode(&mut self) -> Result<Option<(Lsn, Bytes)>, WalDecodeError> {
-        let recordbuf;
-
-        // Run state machine that validates page headers, and reassembles records
-        // that cross page boundaries.
         loop {
             // parse and verify page boundaries as we go
             if self.lsn.segment_offset(pg_constants::WAL_SEGMENT_SIZE) == 0 {
@@ -124,41 +120,29 @@ impl WalStreamDecoder {
                 self.lsn += self.padlen as u64;
                 self.padlen = 0;
             } else if self.contlen == 0 {
-                assert!(self.recordbuf.is_empty());
-
                 // need to have at least the xl_tot_len field
+
                 if self.inputbuf.remaining() < 4 {
                     return Ok(None);
                 }
 
-                // peek xl_tot_len at the beginning of the record.
-                // FIXME: assumes little-endian
+                // read xl_tot_len FIXME: assumes little-endian
                 self.startlsn = self.lsn;
-                let xl_tot_len = (&self.inputbuf[0..4]).get_u32_le();
+                let xl_tot_len = self.inputbuf.get_u32_le();
                 if (xl_tot_len as usize) < XLOG_SIZE_OF_XLOG_RECORD {
                     return Err(WalDecodeError {
                         msg: format!("invalid xl_tot_len {}", xl_tot_len),
                         lsn: self.lsn,
                     });
                 }
+                self.lsn += 4;
 
-                // Fast path for the common case that the whole record fits on the page.
-                let pageleft = self.lsn.remaining_in_block() as u32;
-                if self.inputbuf.remaining() >= xl_tot_len as usize && xl_tot_len <= pageleft {
-                    // Take the record from the 'inputbuf', and validate it.
-                    recordbuf = self.inputbuf.copy_to_bytes(xl_tot_len as usize);
-                    self.lsn += xl_tot_len as u64;
-                    break;
-                } else {
-                    // Need to assemble the record from pieces. Remember the size of the
-                    // record, and loop back. On next iteration, we will reach the 'else'
-                    // branch below, and copy the part of the record that was on this page
-                    // to 'recordbuf'.  Subsequent iterations will skip page headers, and
-                    // append the continuations from the next pages to 'recordbuf'.
-                    self.recordbuf.reserve(xl_tot_len as usize);
-                    self.contlen = xl_tot_len;
-                    continue;
-                }
+                self.recordbuf.clear();
+                self.recordbuf.reserve(xl_tot_len as usize);
+                self.recordbuf.put_u32_le(xl_tot_len);
+
+                self.contlen = xl_tot_len - 4;
+                continue;
             } else {
                 // we're continuing a record, possibly from previous page.
                 let pageleft = self.lsn.remaining_in_block() as u32;
@@ -175,47 +159,38 @@ impl WalStreamDecoder {
                 self.contlen -= n as u32;
 
                 if self.contlen == 0 {
-                    // The record is now complete.
-                    recordbuf = std::mem::replace(&mut self.recordbuf, BytesMut::new()).freeze();
-                    break;
+                    let recordbuf = std::mem::replace(&mut self.recordbuf, BytesMut::new());
+
+                    let recordbuf = recordbuf.freeze();
+                    let mut buf = recordbuf.clone();
+
+                    // XLOG_SWITCH records are special. If we see one, we need to skip
+                    // to the next WAL segment.
+                    let xlogrec = XLogRecord::from_bytes(&mut buf);
+                    if xlogrec.is_xlog_switch_record() {
+                        trace!("saw xlog switch record at {}", self.lsn);
+                        self.padlen =
+                            self.lsn.calc_padding(pg_constants::WAL_SEGMENT_SIZE as u64) as u32;
+                    } else {
+                        // Pad to an 8-byte boundary
+                        self.padlen = self.lsn.calc_padding(8u32) as u32;
+                    }
+
+                    let result = (self.lsn, recordbuf);
+                    return Ok(Some(result));
                 }
                 continue;
             }
         }
+        // check record boundaries
 
-        // We now have a record in the 'recordbuf' local variable.
-        let xlogrec = XLogRecord::from_slice(&recordbuf[0..XLOG_SIZE_OF_XLOG_RECORD]);
+        // deal with continuation records
 
-        let mut crc = 0;
-        crc = crc32c_append(crc, &recordbuf[XLOG_RECORD_CRC_OFFS + 4..]);
-        crc = crc32c_append(crc, &recordbuf[0..XLOG_RECORD_CRC_OFFS]);
-        if crc != xlogrec.xl_crc {
-            return Err(WalDecodeError {
-                msg: "WAL record crc mismatch".into(),
-                lsn: self.lsn,
-            });
-        }
-
-        // XLOG_SWITCH records are special. If we see one, we need to skip
-        // to the next WAL segment.
-        if xlogrec.is_xlog_switch_record() {
-            trace!("saw xlog switch record at {}", self.lsn);
-            self.padlen = self.lsn.calc_padding(pg_constants::WAL_SEGMENT_SIZE as u64) as u32;
-        } else {
-            // Pad to an 8-byte boundary
-            self.padlen = self.lsn.calc_padding(8u32) as u32;
-        }
-
-        // Always align resulting LSN on 0x8 boundary -- that is important for getPage()
-        // and WalReceiver integration. Since this code is used both for WalReceiver and
-        // initial WAL import let's force alignment right here.
-        let result = (self.lsn.align(), recordbuf);
-        Ok(Some(result))
+        // deal with xlog_switch records
     }
 }
 
 #[allow(dead_code)]
-#[derive(Default)]
 pub struct DecodedBkpBlock {
     /* Is this block ref in use? */
     //in_use: bool,
@@ -248,7 +223,25 @@ pub struct DecodedBkpBlock {
 
 impl DecodedBkpBlock {
     pub fn new() -> DecodedBkpBlock {
-        Default::default()
+        DecodedBkpBlock {
+            rnode_spcnode: 0,
+            rnode_dbnode: 0,
+            rnode_relnode: 0,
+            forknum: 0,
+            blkno: 0,
+
+            flags: 0,
+            has_image: false,
+            apply_image: false,
+            will_init: false,
+            hole_offset: 0,
+            hole_length: 0,
+            bimg_len: 0,
+            bimg_info: 0,
+
+            has_data: false,
+            data_len: 0,
+        }
     }
 }
 
@@ -330,31 +323,6 @@ impl XlCreateDatabase {
     }
 }
 
-#[repr(C)]
-#[derive(Debug)]
-pub struct XlDropDatabase {
-    pub db_id: Oid,
-    pub n_tablespaces: Oid, /* number of tablespace IDs */
-    pub tablespace_ids: Vec<Oid>,
-}
-
-impl XlDropDatabase {
-    pub fn decode(buf: &mut Bytes) -> XlDropDatabase {
-        let mut rec = XlDropDatabase {
-            db_id: buf.get_u32_le(),
-            n_tablespaces: buf.get_u32_le(),
-            tablespace_ids: Vec::<Oid>::new(),
-        };
-
-        for _i in 0..rec.n_tablespaces {
-            let id = buf.get_u32_le();
-            rec.tablespace_ids.push(id);
-        }
-
-        rec
-    }
-}
-
 #[repr(C)]
 #[derive(Debug)]
 pub struct XlHeapInsert {
@@ -375,7 +343,6 @@ impl XlHeapInsert {
 #[derive(Debug)]
 pub struct XlHeapMultiInsert {
     pub flags: u8,
-    pub _padding: u8,
     pub ntuples: u16,
 }
 
@@ -383,7 +350,6 @@ impl XlHeapMultiInsert {
     pub fn decode(buf: &mut Bytes) -> XlHeapMultiInsert {
         XlHeapMultiInsert {
             flags: buf.get_u8(),
-            _padding: buf.get_u8(),
             ntuples: buf.get_u16_le(),
         }
     }
@@ -394,8 +360,6 @@ impl XlHeapMultiInsert {
 pub struct XlHeapDelete {
     pub xmax: TransactionId,
     pub offnum: OffsetNumber,
-    pub _padding: u16,
-    pub t_cid: u32,
     pub infobits_set: u8,
     pub flags: u8,
 }
@@ -405,8 +369,6 @@ impl XlHeapDelete {
         XlHeapDelete {
             xmax: buf.get_u32_le(),
             offnum: buf.get_u16_le(),
-            _padding: buf.get_u16_le(),
-            t_cid: buf.get_u32_le(),
             infobits_set: buf.get_u8(),
             flags: buf.get_u8(),
         }
@@ -420,7 +382,6 @@ pub struct XlHeapUpdate {
     pub old_offnum: OffsetNumber,
     pub old_infobits_set: u8,
     pub flags: u8,
-    pub t_cid: u32,
     pub new_xmax: TransactionId,
     pub new_offnum: OffsetNumber,
 }
@@ -432,7 +393,6 @@ impl XlHeapUpdate {
             old_offnum: buf.get_u16_le(),
             old_infobits_set: buf.get_u8(),
             flags: buf.get_u8(),
-            t_cid: buf.get_u32(),
             new_xmax: buf.get_u32_le(),
             new_offnum: buf.get_u16_le(),
         }
@@ -536,24 +496,6 @@ impl XlXactParsedRecord {
     }
 }
 
-#[repr(C)]
-#[derive(Debug)]
-pub struct XlClogTruncate {
-    pub pageno: u32,
-    pub oldest_xid: TransactionId,
-    pub oldest_xid_db: Oid,
-}
-
-impl XlClogTruncate {
-    pub fn decode(buf: &mut Bytes) -> XlClogTruncate {
-        XlClogTruncate {
-            pageno: buf.get_u32_le(),
-            oldest_xid: buf.get_u32_le(),
-            oldest_xid_db: buf.get_u32_le(),
-        }
-    }
-}
-
 #[repr(C)]
 #[derive(Debug)]
 pub struct MultiXactMember {
@@ -876,7 +818,6 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
         let blkno = blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32;
         if info == pg_constants::XLOG_HEAP_INSERT {
             let xlrec = XlHeapInsert::decode(&mut buf);
-            assert_eq!(0, buf.remaining());
             if (xlrec.flags
                 & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
                     | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
@@ -892,7 +833,6 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
             }
         } else if info == pg_constants::XLOG_HEAP_DELETE {
             let xlrec = XlHeapDelete::decode(&mut buf);
-            assert_eq!(0, buf.remaining());
             if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 {
                 let mut blk = DecodedBkpBlock::new();
                 blk.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
@@ -906,9 +846,6 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
             || info == pg_constants::XLOG_HEAP_HOT_UPDATE
         {
             let xlrec = XlHeapUpdate::decode(&mut buf);
-            // the size of tuple data is inferred from the size of the record.
-            // we can't validate the remaining number of bytes without parsing
-            // the tuple data.
             if (xlrec.flags & pg_constants::XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0 {
                 let mut blk = DecodedBkpBlock::new();
                 blk.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
@@ -934,15 +871,6 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
         let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
         if info == pg_constants::XLOG_HEAP2_MULTI_INSERT {
             let xlrec = XlHeapMultiInsert::decode(&mut buf);
-
-            let offset_array_len = if xlogrec.xl_info & pg_constants::XLOG_HEAP_INIT_PAGE > 0 {
-                // the offsets array is omitted if XLOG_HEAP_INIT_PAGE is set
-                0
-            } else {
-                std::mem::size_of::<u16>() * xlrec.ntuples as usize
-            };
-            assert_eq!(offset_array_len, buf.remaining());
-
             if (xlrec.flags
                 & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
                     | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
@@ -969,71 +897,3 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
         main_data_offset,
     }
 }
-
-///
-/// Build a human-readable string to describe a WAL record
-///
-/// For debugging purposes
-pub fn describe_wal_record(record: &Bytes) -> String {
-    // TODO: It would be nice to use the PostgreSQL rmgrdesc infrastructure for this.
-    // Maybe use the postgres wal redo process, the same used for replaying WAL records?
-    // Or could we compile the rmgrdesc routines into the dump_layer_file() binary directly,
-    // without worrying about security?
-    //
-    // But for now, we have a hand-written code for a few common WAL record types here.
-
-    let mut buf = record.clone();
-
-    // 1. Parse XLogRecord struct
-
-    // FIXME: assume little-endian here
-    let xlogrec = XLogRecord::from_bytes(&mut buf);
-
-    let unknown_str: String;
-
-    let result: &str = match xlogrec.xl_rmid {
-        pg_constants::RM_HEAP2_ID => {
-            let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
-            match info {
-                pg_constants::XLOG_HEAP2_MULTI_INSERT => "HEAP2 MULTI_INSERT",
-                pg_constants::XLOG_HEAP2_VISIBLE => "HEAP2 VISIBLE",
-                _ => {
-                    unknown_str = format!("HEAP2 UNKNOWN_0x{:02x}", info);
-                    &unknown_str
-                }
-            }
-        }
-        pg_constants::RM_HEAP_ID => {
-            let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
-            match info {
-                pg_constants::XLOG_HEAP_INSERT => "HEAP INSERT",
-                pg_constants::XLOG_HEAP_DELETE => "HEAP DELETE",
-                pg_constants::XLOG_HEAP_UPDATE => "HEAP UPDATE",
-                pg_constants::XLOG_HEAP_HOT_UPDATE => "HEAP HOT_UPDATE",
-                _ => {
-                    unknown_str = format!("HEAP2 UNKNOWN_0x{:02x}", info);
-                    &unknown_str
-                }
-            }
-        }
-        pg_constants::RM_XLOG_ID => {
-            let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
-            match info {
-                pg_constants::XLOG_FPI => "XLOG FPI",
-                pg_constants::XLOG_FPI_FOR_HINT => "XLOG FPI_FOR_HINT",
-                _ => {
-                    unknown_str = format!("XLOG UNKNOWN_0x{:02x}", info);
-                    &unknown_str
-                }
-            }
-        }
-        rmid => {
-            let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
-
-            unknown_str = format!("UNKNOWN_RM_{} INFO_0x{:02x}", rmid, info);
-            &unknown_str
-        }
-    };
-
-    String::from(result)
-}

pageserver/src/walreceiver.rs

@@ -1,43 +1,43 @@
 //!
-//! WAL receiver connects to the WAL safekeeper service, streams WAL,
-//! decodes records and saves them in the repository for the correct
-//! timeline.
+//! WAL receiver connects to the WAL safekeeper service,
+//! streams WAL, decodes records and saves them in page cache.
 //!
 //! We keep one WAL receiver active per timeline.
 
-use crate::relish::*;
+use crate::object_key::*;
+use crate::page_cache;
 use crate::restore_local_repo;
-use crate::tenant_mgr;
 use crate::waldecoder::*;
 use crate::PageServerConf;
-use anyhow::{bail, Error, Result};
+use crate::ZTimelineId;
+use anyhow::{Error, Result};
 use lazy_static::lazy_static;
+use log::*;
 use postgres::fallible_iterator::FallibleIterator;
 use postgres::replication::ReplicationIter;
 use postgres::{Client, NoTls, SimpleQueryMessage, SimpleQueryRow};
+use postgres_ffi::xlog_utils::*;
 use postgres_ffi::*;
 use postgres_protocol::message::backend::ReplicationMessage;
 use postgres_types::PgLsn;
-use std::cell::Cell;
+use std::cmp::{max, min};
 use std::collections::HashMap;
+use std::fs;
+use std::fs::{File, OpenOptions};
+use std::io::{Seek, SeekFrom, Write};
+use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Mutex;
 use std::thread;
 use std::thread::sleep;
-use std::thread::JoinHandle;
-use std::thread_local;
 use std::time::{Duration, SystemTime};
-use tracing::*;
 use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::ZTenantId;
-use zenith_utils::zid::ZTimelineId;
 
 //
 // We keep one WAL Receiver active per timeline.
 //
 struct WalReceiverEntry {
     wal_producer_connstr: String,
-    wal_receiver_handle: Option<JoinHandle<()>>,
 }
 
 lazy_static! {
@@ -45,32 +45,11 @@ lazy_static! {
         Mutex::new(HashMap::new());
 }
 
-thread_local! {
-    // Boolean that is true only for WAL receiver threads
-    //
-    // This is used in `wait_lsn` to guard against usage that might lead to a deadlock.
-    pub(crate) static IS_WAL_RECEIVER: Cell<bool> = Cell::new(false);
-}
-
-// Wait for walreceiver to stop
-// Now it stops when pageserver shutdown is requested.
-// In future we can make this more granular and send shutdown signals
-// per tenant/timeline to cancel inactive walreceivers.
-// TODO deal with blocking pg connections
-pub fn stop_wal_receiver(timelineid: ZTimelineId) {
-    let mut receivers = WAL_RECEIVERS.lock().unwrap();
-    if let Some(r) = receivers.get_mut(&timelineid) {
-        r.wal_receiver_handle.take();
-        // r.wal_receiver_handle.take().map(JoinHandle::join);
-    }
-}
-
 // Launch a new WAL receiver, or tell one that's running about change in connection string
 pub fn launch_wal_receiver(
     conf: &'static PageServerConf,
     timelineid: ZTimelineId,
     wal_producer_connstr: &str,
-    tenantid: ZTenantId,
 ) {
     let mut receivers = WAL_RECEIVERS.lock().unwrap();
 
@@ -79,19 +58,18 @@ pub fn launch_wal_receiver(
             receiver.wal_producer_connstr = wal_producer_connstr.into();
         }
         None => {
-            let wal_receiver_handle = thread::Builder::new()
-                .name("WAL receiver thread".into())
-                .spawn(move || {
-                    IS_WAL_RECEIVER.with(|c| c.set(true));
-                    thread_main(conf, timelineid, tenantid);
-                })
-                .unwrap();
-
             let receiver = WalReceiverEntry {
                 wal_producer_connstr: wal_producer_connstr.into(),
-                wal_receiver_handle: Some(wal_receiver_handle),
             };
             receivers.insert(timelineid, receiver);
+
+            // Also launch a new thread to handle this connection
+            let _walreceiver_thread = thread::Builder::new()
+                .name("WAL receiver thread".into())
+                .spawn(move || {
+                    thread_main(conf, timelineid);
+                })
+                .unwrap();
         }
     };
 }
@@ -110,19 +88,21 @@ fn get_wal_producer_connstr(timelineid: ZTimelineId) -> String {
 //
 // This is the entry point for the WAL receiver thread.
 //
-fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId, tenantid: ZTenantId) {
-    let _enter = info_span!("WAL receiver", timeline = %timelineid, tenant = %tenantid).entered();
-    info!("WAL receiver thread started");
+fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId) {
+    info!(
+        "WAL receiver thread started for timeline : '{}'",
+        timelineid
+    );
 
     //
     // Make a connection to the WAL safekeeper, or directly to the primary PostgreSQL server,
     // and start streaming WAL from it. If the connection is lost, keep retrying.
     //
-    while !tenant_mgr::shutdown_requested() {
+    loop {
         // Look up the current WAL producer address
         let wal_producer_connstr = get_wal_producer_connstr(timelineid);
 
-        let res = walreceiver_main(conf, timelineid, &wal_producer_connstr, tenantid);
+        let res = walreceiver_main(conf, timelineid, &wal_producer_connstr);
 
         if let Err(e) = res {
             info!(
@@ -132,14 +112,12 @@ fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId, tenantid:
             sleep(Duration::from_secs(1));
         }
     }
-    debug!("WAL streaming shut down");
 }
 
 fn walreceiver_main(
     _conf: &PageServerConf,
     timelineid: ZTimelineId,
     wal_producer_connstr: &str,
-    tenantid: ZTenantId,
 ) -> Result<(), Error> {
     // Connect to the database in replication mode.
     info!("connecting to {:?}", wal_producer_connstr);
@@ -151,40 +129,37 @@ fn walreceiver_main(
     let mut rclient = Client::connect(&connect_cfg, NoTls)?;
     info!("connected!");
 
-    // Immediately increment the gauge, then create a job to decrement it on thread exit.
-    // One of the pros of `defer!` is that this will *most probably*
-    // get called, even in presence of panics.
-    let gauge = crate::LIVE_CONNECTIONS_COUNT.with_label_values(&["wal_receiver"]);
-    gauge.inc();
-    scopeguard::defer! {
-        gauge.dec();
-    }
-
     let identify = identify_system(&mut rclient)?;
     info!("{:?}", identify);
     let end_of_wal = Lsn::from(u64::from(identify.xlogpos));
     let mut caught_up = false;
 
-    let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)?;
+    let repository = page_cache::get_repository();
+    let timeline = repository.get_timeline(timelineid).unwrap();
 
     //
     // Start streaming the WAL, from where we left off previously.
     //
     // If we had previously received WAL up to some point in the middle of a WAL record, we
-    // better start from the end of last full WAL record, not in the middle of one.
+    // better start from the end of last full WAL record, not in the middle of one. Hence,
+    // use 'last_record_lsn' rather than 'last_valid_lsn' here.
     let mut last_rec_lsn = timeline.get_last_record_lsn();
     let mut startpoint = last_rec_lsn;
 
     if startpoint == Lsn(0) {
-        bail!("No previous WAL position");
+        error!("No previous WAL position");
     }
 
     // There might be some padding after the last full record, skip it.
+    //
+    // FIXME: It probably would be better to always start streaming from the beginning
+    // of the page, or the segment, so that we could check the page/segment headers
+    // too. Just for the sake of paranoia.
     startpoint += startpoint.calc_padding(8u32);
 
-    info!(
-        "last_record_lsn {} starting replication from {}, server is at {}...",
-        last_rec_lsn, startpoint, end_of_wal
+    debug!(
+        "last_record_lsn {} starting replication from {} for timeline {}, server is at {}...",
+        last_rec_lsn, startpoint, timelineid, end_of_wal
     );
 
     let query = format!("START_REPLICATION PHYSICAL {}", startpoint);
@@ -194,7 +169,8 @@ fn walreceiver_main(
 
     let mut waldecoder = WalStreamDecoder::new(startpoint);
 
-    let checkpoint_bytes = timeline.get_page_at_lsn(RelishTag::Checkpoint, 0, startpoint)?;
+    let checkpoint_bytes =
+        timeline.get_page_at_lsn_nowait(ObjectTag::Checkpoint, startpoint, false)?;
     let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
     trace!("CheckPoint.nextXid = {}", checkpoint.nextXid.value);
 
@@ -206,49 +182,73 @@ fn walreceiver_main(
                 let data = xlog_data.data();
                 let startlsn = Lsn::from(xlog_data.wal_start());
                 let endlsn = startlsn + data.len() as u64;
+                let prev_last_rec_lsn = last_rec_lsn;
+
+                write_wal_file(startlsn, timelineid, pg_constants::WAL_SEGMENT_SIZE, data)?;
 
                 trace!("received XLogData between {} and {}", startlsn, endlsn);
 
                 waldecoder.feed_bytes(data);
 
                 while let Some((lsn, recdata)) = waldecoder.poll_decode()? {
-                    let _enter = info_span!("processing record", lsn = %lsn).entered();
-
-                    // It is important to deal with the aligned records as lsn in getPage@LSN is
-                    // aligned and can be several bytes bigger. Without this alignment we are
-                    // at risk of hittind a deadlock.
-                    assert!(lsn.is_aligned());
-
-                    let writer = timeline.writer();
-
-                    let mut checkpoint_modified = false;
-
+                    // Save old checkpoint value to compare with it after decoding WAL record
+                    let old_checkpoint_bytes = checkpoint.encode();
                     let decoded = decode_wal_record(recdata.clone());
                     restore_local_repo::save_decoded_record(
                         &mut checkpoint,
-                        &mut checkpoint_modified,
-                        writer.as_ref(),
+                        &*timeline,
                         &decoded,
                         recdata,
                         lsn,
                     )?;
+                    last_rec_lsn = lsn;
 
+                    let new_checkpoint_bytes = checkpoint.encode();
                     // Check if checkpoint data was updated by save_decoded_record
-                    if checkpoint_modified {
-                        let new_checkpoint_bytes = checkpoint.encode();
-
-                        writer.put_page_image(
-                            RelishTag::Checkpoint,
-                            0,
+                    if new_checkpoint_bytes != old_checkpoint_bytes {
+                        timeline.put_page_image(
+                            ObjectTag::Checkpoint,
                             lsn,
                             new_checkpoint_bytes,
+                            false,
                         )?;
                     }
+                }
 
-                    // Now that this record has been fully handled, including updating the
-                    // checkpoint data, let the repository know that it is up-to-date to this LSN
-                    writer.advance_last_record_lsn(lsn);
-                    last_rec_lsn = lsn;
+                // Update the last_valid LSN value in the page cache one more time. We updated
+                // it in the loop above, between each WAL record, but we might have received
+                // a partial record after the last completed record. Our page cache's value
+                // better reflect that, because GetPage@LSN requests might also point in the
+                // middle of a record, if the request LSN was taken from the server's current
+                // flush ptr.
+                timeline.advance_last_valid_lsn(endlsn);
+
+                // Somewhat arbitrarily, if we have at least 10 complete wal segments (16 MB each),
+                // "checkpoint" the repository to flush all the changes from WAL we've processed
+                // so far to disk. After this, we don't need the original WAL anymore, and it
+                // can be removed. This is probably too aggressive for production, but it's useful
+                // to expose bugs now.
+                //
+                // TODO: We don't actually dare to remove the WAL. It's useful for debugging,
+                // and we might it for logical decoding other things in the future. Although
+                // we should also be able to fetch it back from the WAL safekeepers or S3 if
+                // needed.
+                if prev_last_rec_lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE)
+                    != last_rec_lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE)
+                {
+                    info!("switched segment {} to {}", prev_last_rec_lsn, last_rec_lsn);
+                    let (oldest_segno, newest_segno) = find_wal_file_range(
+                        timelineid,
+                        pg_constants::WAL_SEGMENT_SIZE,
+                        last_rec_lsn,
+                    )?;
+
+                    if newest_segno - oldest_segno >= 10 {
+                        timeline.checkpoint()?;
+
+                        // TODO: This is where we could remove WAL older than last_rec_lsn.
+                        //remove_wal_files(timelineid, pg_constants::WAL_SEGMENT_SIZE, last_rec_lsn)?;
+                    }
                 }
 
                 if !caught_up && endlsn >= end_of_wal {
@@ -272,7 +272,7 @@ fn walreceiver_main(
                 );
 
                 if reply_requested {
-                    Some(last_rec_lsn)
+                    Some(timeline.get_last_valid_lsn())
                 } else {
                     None
                 }
@@ -292,24 +292,55 @@ fn walreceiver_main(
 
             physical_stream.standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY)?;
         }
-
-        if tenant_mgr::shutdown_requested() {
-            debug!("stop walreceiver because pageserver shutdown is requested");
-            break;
-        }
     }
     Ok(())
 }
 
+fn find_wal_file_range(
+    timeline: ZTimelineId,
+    wal_seg_size: usize,
+    written_upto: Lsn,
+) -> Result<(u64, u64)> {
+    let written_upto_segno = written_upto.segment_number(wal_seg_size);
+
+    let mut oldest_segno = written_upto_segno;
+    let mut newest_segno = written_upto_segno;
+    // Scan the wal directory, and count how many WAL filed we could remove
+    let wal_dir = PathBuf::from(format!("timelines/{}/wal", timeline));
+    for entry in fs::read_dir(wal_dir)? {
+        let entry = entry?;
+        let path = entry.path();
+
+        if path.is_dir() {
+            continue;
+        }
+
+        let filename = path.file_name().unwrap().to_str().unwrap();
+
+        if IsXLogFileName(filename) {
+            let (segno, _tli) = XLogFromFileName(filename, wal_seg_size);
+
+            if segno > written_upto_segno {
+                // that's strange.
+                warn!("there is a WAL file from future at {}", path.display());
+                continue;
+            }
+
+            oldest_segno = min(oldest_segno, segno);
+            newest_segno = max(newest_segno, segno);
+        }
+    }
+    // FIXME: would be good to assert that there are no gaps in the WAL files
+
+    Ok((oldest_segno, newest_segno))
+}
+
 /// Data returned from the postgres `IDENTIFY_SYSTEM` command
 ///
 /// See the [postgres docs] for more details.
 ///
 /// [postgres docs]: https://www.postgresql.org/docs/current/protocol-replication.html
 #[derive(Debug)]
-// As of nightly 2021-09-11, fields that are only read by the type's `Debug` impl still count as
-// unused. Relevant issue: https://github.com/rust-lang/rust/issues/88900
-#[allow(dead_code)]
 pub struct IdentifySystem {
     systemid: u64,
     timeline: u32,
@@ -350,3 +381,96 @@ pub fn identify_system(client: &mut Client) -> Result<IdentifySystem, Error> {
         Err(IdentifyError.into())
     }
 }
+
+fn write_wal_file(
+    startpos: Lsn,
+    timeline: ZTimelineId,
+    wal_seg_size: usize,
+    buf: &[u8],
+) -> anyhow::Result<()> {
+    let mut bytes_left: usize = buf.len();
+    let mut bytes_written: usize = 0;
+    let mut partial;
+    let mut start_pos = startpos;
+    const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ];
+
+    let wal_dir = PathBuf::from(format!("timelines/{}/wal", timeline));
+
+    /* Extract WAL location for this block */
+    let mut xlogoff = start_pos.segment_offset(wal_seg_size);
+
+    while bytes_left != 0 {
+        let bytes_to_write;
+
+        /*
+         * If crossing a WAL boundary, only write up until we reach wal
+         * segment size.
+         */
+        if xlogoff + bytes_left > wal_seg_size {
+            bytes_to_write = wal_seg_size - xlogoff;
+        } else {
+            bytes_to_write = bytes_left;
+        }
+
+        /* Open file */
+        let segno = start_pos.segment_number(wal_seg_size);
+        let wal_file_name = XLogFileName(
+            1, // FIXME: always use Postgres timeline 1
+            segno,
+            wal_seg_size,
+        );
+        let wal_file_path = wal_dir.join(wal_file_name.clone());
+        let wal_file_partial_path = wal_dir.join(wal_file_name.clone() + ".partial");
+
+        {
+            let mut wal_file: File;
+            /* Try to open already completed segment */
+            if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_path) {
+                wal_file = file;
+                partial = false;
+            } else if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_partial_path) {
+                /* Try to open existed partial file */
+                wal_file = file;
+                partial = true;
+            } else {
+                /* Create and fill new partial file */
+                partial = true;
+                match OpenOptions::new()
+                    .create(true)
+                    .write(true)
+                    .open(&wal_file_partial_path)
+                {
+                    Ok(mut file) => {
+                        for _ in 0..(wal_seg_size / XLOG_BLCKSZ) {
+                            file.write_all(&ZERO_BLOCK)?;
+                        }
+                        wal_file = file;
+                    }
+                    Err(e) => {
+                        error!("Failed to open log file {:?}: {}", &wal_file_path, e);
+                        return Err(e.into());
+                    }
+                }
+            }
+            wal_file.seek(SeekFrom::Start(xlogoff as u64))?;
+            wal_file.write_all(&buf[bytes_written..(bytes_written + bytes_to_write)])?;
+
+            // FIXME: Flush the file
+            //wal_file.sync_all()?;
+        }
+        /* Write was successful, advance our position */
+        bytes_written += bytes_to_write;
+        bytes_left -= bytes_to_write;
+        start_pos += bytes_to_write as u64;
+        xlogoff += bytes_to_write;
+
+        /* Did we reach the end of a WAL segment? */
+        if start_pos.segment_offset(wal_seg_size) == 0 {
+            xlogoff = 0;
+            if partial {
+                fs::rename(&wal_file_partial_path, &wal_file_path)?;
+            }
+        }
+    }
+    Ok(())
+}

pageserver/src/walredo.rs

@@ -1,7 +1,6 @@
 //!
 //! WAL redo. This service runs PostgreSQL in a special wal_redo mode
-//! to apply given WAL records over an old page image and return new
-//! page image.
+//! to apply given WAL records over an old page image and return new page image.
 //!
 //! We rely on Postgres to perform WAL redo for us. We launch a
 //! postgres process in special "wal redo" mode that's similar to
@@ -13,22 +12,21 @@
 //! See src/backend/tcop/zenith_wal_redo.c for the other side of
 //! this communication.
 //!
-//! The Postgres process is assumed to be secure against malicious WAL
-//! records. It achieves it by dropping privileges before replaying
-//! any WAL records, so that even if an attacker hijacks the Postgres
-//! process, he cannot escape out of it.
+//! TODO: Even though the postgres code runs in a separate process,
+//! it's not a secure sandbox.
 //!
 use byteorder::{ByteOrder, LittleEndian};
 use bytes::{Buf, BufMut, Bytes, BytesMut};
-use lazy_static::lazy_static;
 use log::*;
-use serde::Serialize;
+use std::cell::RefCell;
 use std::fs;
 use std::fs::OpenOptions;
 use std::io::prelude::*;
 use std::io::Error;
 use std::path::PathBuf;
 use std::process::Stdio;
+use std::sync::mpsc;
+use std::sync::Arc;
 use std::sync::Mutex;
 use std::time::Duration;
 use std::time::Instant;
@@ -36,35 +34,19 @@ use tokio::io::AsyncBufReadExt;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::process::{ChildStdin, ChildStdout, Command};
 use tokio::time::timeout;
-use zenith_metrics::{register_histogram, register_int_counter, Histogram, IntCounter};
 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::ZTenantId;
 
-use crate::relish::*;
+use crate::object_key::*;
+use crate::repository::BufferTag;
 use crate::repository::WALRecord;
-use crate::waldecoder::XlMultiXactCreate;
 use crate::waldecoder::XlXactParsedRecord;
+use crate::waldecoder::{MultiXactId, XlMultiXactCreate};
 use crate::PageServerConf;
-use postgres_ffi::nonrelfile_utils::mx_offset_to_flags_bitshift;
-use postgres_ffi::nonrelfile_utils::mx_offset_to_flags_offset;
-use postgres_ffi::nonrelfile_utils::mx_offset_to_member_offset;
 use postgres_ffi::nonrelfile_utils::transaction_id_set_status;
 use postgres_ffi::pg_constants;
 use postgres_ffi::XLogRecord;
 
-///
-/// `RelTag` + block number (`blknum`) gives us a unique id of the page in the cluster.
-///
-/// In Postgres `BufferTag` structure is used for exactly the same purpose.
-/// [See more related comments here](https://github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/buf_internals.h#L91).
-///
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Serialize)]
-pub struct BufferTag {
-    pub rel: RelTag,
-    pub blknum: u32,
-}
-
 ///
 /// WAL Redo Manager is responsible for replaying WAL records.
 ///
@@ -78,11 +60,10 @@ pub trait WalRedoManager: Send + Sync {
     /// the reords.
     fn request_redo(
         &self,
-        rel: RelishTag,
-        blknum: u32,
+        tag: ObjectTag,
         lsn: Lsn,
         base_img: Option<Bytes>,
-        records: Vec<(Lsn, WALRecord)>,
+        records: Vec<WALRecord>,
     ) -> Result<Bytes, WalRedoError>;
 }
 
@@ -95,11 +76,10 @@ pub struct DummyRedoManager {}
 impl crate::walredo::WalRedoManager for DummyRedoManager {
     fn request_redo(
         &self,
-        _rel: RelishTag,
-        _blknum: u32,
+        _tag: ObjectTag,
         _lsn: Lsn,
         _base_img: Option<Bytes>,
-        _records: Vec<(Lsn, WALRecord)>,
+        _records: Vec<WALRecord>,
     ) -> Result<Bytes, WalRedoError> {
         Err(WalRedoError::InvalidState)
     }
@@ -107,50 +87,34 @@ impl crate::walredo::WalRedoManager for DummyRedoManager {
 
 static TIMEOUT: Duration = Duration::from_secs(20);
 
-// Metrics collected on WAL redo operations
-//
-// We collect the time spent in actual WAL redo ('redo'), and time waiting
-// for access to the postgres process ('wait') since there is only one for
-// each tenant.
-lazy_static! {
-    static ref WAL_REDO_TIME: Histogram =
-        register_histogram!("pageserver_wal_redo_time", "Time spent on WAL redo")
-            .expect("failed to define a metric");
-    static ref WAL_REDO_WAIT_TIME: Histogram = register_histogram!(
-        "pageserver_wal_redo_wait_time",
-        "Time spent waiting for access to the WAL redo process"
-    )
-    .expect("failed to define a metric");
-    static ref WAL_REDO_RECORD_COUNTER: IntCounter = register_int_counter!(
-        "pageserver_wal_records_replayed",
-        "Number of WAL records replayed"
-    )
-    .unwrap();
-}
-
 ///
-/// This is the real implementation that uses a Postgres process to
-/// perform WAL replay. Only one thread can use the processs at a time,
-/// that is controlled by the Mutex. In the future, we might want to
-/// launch a pool of processes to allow concurrent replay of multiple
-/// records.
+/// The implementation consists of two parts: PostgresRedoManager, and
+/// PostgresRedoManagerInternal. PostgresRedoManager is the public struct
+/// that can be used to send redo requests to the manager.
+/// PostgresRedoManagerInternal is used by the manager thread itself.
 ///
 pub struct PostgresRedoManager {
-    tenantid: ZTenantId,
+    request_tx: Mutex<mpsc::Sender<WalRedoRequest>>,
+}
+
+struct PostgresRedoManagerInternal {
     conf: &'static PageServerConf,
 
-    runtime: tokio::runtime::Runtime,
-    process: Mutex<Option<PostgresRedoProcess>>,
+    request_rx: mpsc::Receiver<WalRedoRequest>,
+}
+
+#[derive(Debug)]
+struct WalRedoRequestData {
+    tag: ObjectTag,
+    lsn: Lsn,
+    base_img: Option<Bytes>,
+    records: Vec<WALRecord>,
 }
 
 #[derive(Debug)]
 struct WalRedoRequest {
-    rel: RelishTag,
-    blknum: u32,
-    lsn: Lsn,
-
-    base_img: Option<Bytes>,
-    records: Vec<(Lsn, WALRecord)>,
+    data: WalRedoRequestData,
+    response_channel: mpsc::Sender<Result<Bytes, WalRedoError>>,
 }
 
 /// An error happened in WAL redo
@@ -166,6 +130,37 @@ pub enum WalRedoError {
 ///
 /// Public interface of WAL redo manager
 ///
+impl PostgresRedoManager {
+    ///
+    /// Create a new PostgresRedoManager.
+    ///
+    /// This launches a new thread to handle the requests.
+    pub fn new(conf: &'static PageServerConf) -> PostgresRedoManager {
+        let (tx, rx) = mpsc::channel();
+
+        //
+        // Launch the WAL redo thread
+        //
+        // Get mutable references to the values that we need to pass to the
+        // thread.
+        let request_rx = rx;
+
+        // Currently, the join handle is not saved anywhere and we
+        // won't try restart the thread if it dies.
+        let _walredo_thread = std::thread::Builder::new()
+            .name("WAL redo thread".into())
+            .spawn(move || {
+                let mut internal = PostgresRedoManagerInternal { conf, request_rx };
+                internal.wal_redo_main();
+            })
+            .unwrap();
+
+        PostgresRedoManager {
+            request_tx: Mutex::new(tx),
+        }
+    }
+}
+
 impl WalRedoManager for PostgresRedoManager {
     ///
     /// Request the WAL redo manager to apply some WAL records
@@ -175,55 +170,63 @@ impl WalRedoManager for PostgresRedoManager {
     ///
     fn request_redo(
         &self,
-        rel: RelishTag,
-        blknum: u32,
+        tag: ObjectTag,
         lsn: Lsn,
         base_img: Option<Bytes>,
-        records: Vec<(Lsn, WALRecord)>,
+        records: Vec<WALRecord>,
     ) -> Result<Bytes, WalRedoError> {
-        let start_time;
-        let lock_time;
-        let end_time;
+        // Create a channel where to receive the response
+        let (tx, rx) = mpsc::channel::<Result<Bytes, WalRedoError>>();
 
         let request = WalRedoRequest {
-            rel,
-            blknum,
-            lsn,
-            base_img,
-            records,
+            data: WalRedoRequestData {
+                tag,
+                lsn,
+                base_img,
+                records,
+            },
+            response_channel: tx,
         };
 
-        start_time = Instant::now();
-        let result = {
-            let mut process_guard = self.process.lock().unwrap();
-            lock_time = Instant::now();
+        self.request_tx
+            .lock()
+            .unwrap()
+            .send(request)
+            .expect("could not send WAL redo request");
 
-            // launch the WAL redo process on first use
-            if process_guard.is_none() {
-                let p = self
-                    .runtime
-                    .block_on(PostgresRedoProcess::launch(self.conf, &self.tenantid))?;
-                *process_guard = Some(p);
-            }
-            let process = process_guard.as_mut().unwrap();
-
-            self.runtime
-                .block_on(self.handle_apply_request(process, &request))
-        };
-        end_time = Instant::now();
-
-        WAL_REDO_WAIT_TIME.observe(lock_time.duration_since(start_time).as_secs_f64());
-        WAL_REDO_TIME.observe(end_time.duration_since(lock_time).as_secs_f64());
-
-        result
+        rx.recv()
+            .expect("could not receive response to WAL redo request")
     }
 }
 
-impl PostgresRedoManager {
-    ///
-    /// Create a new PostgresRedoManager.
-    ///
-    pub fn new(conf: &'static PageServerConf, tenantid: ZTenantId) -> PostgresRedoManager {
+fn mx_offset_to_flags_offset(xid: MultiXactId) -> usize {
+    ((xid / pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP as u32) as u16
+        % pg_constants::MULTIXACT_MEMBERGROUPS_PER_PAGE
+        * pg_constants::MULTIXACT_MEMBERGROUP_SIZE) as usize
+}
+
+fn mx_offset_to_flags_bitshift(xid: MultiXactId) -> u16 {
+    (xid as u16) % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP
+        * pg_constants::MXACT_MEMBER_BITS_PER_XACT
+}
+
+/* Location (byte offset within page) of TransactionId of given member */
+fn mx_offset_to_member_offset(xid: MultiXactId) -> usize {
+    mx_offset_to_flags_offset(xid)
+        + (pg_constants::MULTIXACT_FLAGBYTES_PER_GROUP
+            + (xid as u16 % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP) * 4) as usize
+}
+
+///
+/// WAL redo thread
+///
+impl PostgresRedoManagerInternal {
+    //
+    // Main entry point for the WAL applicator thread.
+    //
+    fn wal_redo_main(&mut self) {
+        info!("WAL redo thread started");
+
         // We block on waiting for requests on the walredo request channel, but
         // use async I/O to communicate with the child process. Initialize the
         // runtime for the async part.
@@ -232,25 +235,66 @@ impl PostgresRedoManager {
             .build()
             .unwrap();
 
-        // The actual process is launched lazily, on first request.
-        PostgresRedoManager {
-            runtime,
-            tenantid,
-            conf,
-            process: Mutex::new(None),
+        let processes: Vec<PostgresRedoProcess>;
+
+        info!("launching WAL redo postgres process");
+
+        let wal_redoers = self.conf.wal_redoers;
+        processes = (0..wal_redoers)
+            .map(|i| {
+                runtime
+                    .block_on(PostgresRedoProcess::launch(self.conf, i))
+                    .unwrap()
+            })
+            .collect();
+
+        // Loop forever, handling requests as they come.
+        loop {
+            let mut requests: Vec<WalRedoRequest> = Vec::new();
+            requests.push(
+                self.request_rx
+                    .recv()
+                    .expect("WAL redo request channel was closed"),
+            );
+            loop {
+                let req = self.request_rx.try_recv();
+                match req {
+                    Ok(req) => requests.push(req),
+                    Err(_) => break,
+                }
+            }
+            let request_data = requests.iter().map(|req| &req.data);
+            let mut rr = 0; // round robin
+            let results = runtime.block_on(async {
+                let futures = request_data.map(|req| {
+                    rr += 1;
+                    self.handle_apply_request(&processes[rr % wal_redoers], &req)
+                });
+                let mut results: Vec<Result<Bytes, WalRedoError>> = Vec::new();
+                for future in futures {
+                    results.push(future.await);
+                }
+                results
+            });
+            for (result, request) in results.into_iter().zip(requests.iter()) {
+                let result_ok = result.is_ok();
+
+                // Send the result to the requester
+                let _ = request.response_channel.send(result);
+
+                if !result_ok {
+                    error!("wal-redo-postgres failed to apply request {:?}", request);
+                }
+            }
         }
     }
 
-    ///
-    /// Process one request for WAL redo.
-    ///
     async fn handle_apply_request(
         &self,
-        process: &mut PostgresRedoProcess,
-        request: &WalRedoRequest,
+        process: &PostgresRedoProcess,
+        request: &WalRedoRequestData,
     ) -> Result<Bytes, WalRedoError> {
-        let rel = request.rel;
-        let blknum = request.blknum;
+        let tag = request.tag;
         let lsn = request.lsn;
         let base_img = request.base_img.clone();
         let records = &request.records;
@@ -260,13 +304,11 @@ impl PostgresRedoManager {
         let start = Instant::now();
 
         let apply_result: Result<Bytes, Error>;
-        if let RelishTag::Relation(rel) = rel {
+        if let ObjectTag::RelationBuffer(buf_tag) = tag {
             // Relational WAL records are applied using wal-redo-postgres
-            let buf_tag = BufferTag { rel, blknum };
             apply_result = process.apply_wal_records(buf_tag, base_img, records).await;
         } else {
-            // Non-relational WAL records are handled here, with custom code that has the
-            // same effects as the corresponding Postgres WAL redo function.
+            // Non-relational WAL records we apply ourselves.
             const ZERO_PAGE: [u8; 8192] = [0u8; 8192];
             let mut page = BytesMut::new();
             if let Some(fpi) = base_img {
@@ -277,11 +319,9 @@ impl PostgresRedoManager {
                 page.extend_from_slice(&ZERO_PAGE);
             }
             // Apply all collected WAL records
-            for (_lsn, record) in records {
+            for record in records {
                 let mut buf = record.rec.clone();
 
-                WAL_REDO_RECORD_COUNTER.inc();
-
                 // 1. Parse XLogRecord struct
                 // FIXME: refactor to avoid code duplication.
                 let xlogrec = XLogRecord::from_bytes(&mut buf);
@@ -294,18 +334,25 @@ impl PostgresRedoManager {
                     buf.advance(skip);
                 }
 
-                if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
+                if xlogrec.xl_rmid == pg_constants::RM_CLOG_ID {
+                    let info = xlogrec.xl_info & !pg_constants::XLR_INFO_MASK;
+                    if info == pg_constants::CLOG_ZEROPAGE {
+                        // The only operation we need to implement is CLOG_ZEROPAGE
+                        page.copy_from_slice(&ZERO_PAGE);
+                    }
+                } else if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
                     // Transaction manager stuff
-                    let rec_segno = match rel {
-                        RelishTag::Slru { slru, segno } => {
-                            assert!(
-                                slru == SlruKind::Clog,
-                                "Not valid XACT relish tag {:?}",
-                                rel
-                            );
-                            segno
+                    let info = xlogrec.xl_info & pg_constants::XLOG_XACT_OPMASK;
+                    let tag_blknum = match tag {
+                        ObjectTag::Clog(slru) => slru.blknum,
+                        ObjectTag::TwoPhase(_) => {
+                            assert!(info == pg_constants::XLOG_XACT_PREPARE);
+                            trace!("Apply prepare {} record", xlogrec.xl_xid);
+                            page.clear();
+                            page.extend_from_slice(&buf[..]);
+                            continue;
                         }
-                        _ => panic!("Not valid XACT relish tag {:?}", rel),
+                        _ => panic!("Not valid XACT object tag {:?}", tag),
                     };
                     let parsed_xact =
                         XlXactParsedRecord::decode(&mut buf, xlogrec.xl_xid, xlogrec.xl_info);
@@ -318,14 +365,12 @@ impl PostgresRedoManager {
                             &mut page,
                         );
                         for subxact in &parsed_xact.subxacts {
-                            let pageno = *subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
-                            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-                            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+                            let blkno = *subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
                             // only update xids on the requested page
-                            if rec_segno == segno && blknum == rpageno {
+                            if tag_blknum == blkno {
                                 transaction_id_set_status(
                                     *subxact,
-                                    pg_constants::TRANSACTION_STATUS_COMMITTED,
+                                    pg_constants::TRANSACTION_STATUS_SUB_COMMITTED,
                                     &mut page,
                                 );
                             }
@@ -339,11 +384,9 @@ impl PostgresRedoManager {
                             &mut page,
                         );
                         for subxact in &parsed_xact.subxacts {
-                            let pageno = *subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
-                            let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-                            let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+                            let blkno = *subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
                             // only update xids on the requested page
-                            if rec_segno == segno && blknum == rpageno {
+                            if tag_blknum == blkno {
                                 transaction_id_set_status(
                                     *subxact,
                                     pg_constants::TRANSACTION_STATUS_ABORTED,
@@ -353,57 +396,54 @@ impl PostgresRedoManager {
                         }
                     }
                 } else if xlogrec.xl_rmid == pg_constants::RM_MULTIXACT_ID {
-                    // Multixact operations
+                    // Multiexact operations
                     let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
-                    if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
+                    if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE
+                        || info == pg_constants::XLOG_MULTIXACT_ZERO_MEM_PAGE
+                    {
+                        // Just need to zero page
+                        page.copy_from_slice(&ZERO_PAGE);
+                    } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
                         let xlrec = XlMultiXactCreate::decode(&mut buf);
-                        if let RelishTag::Slru {
-                            slru,
-                            segno: rec_segno,
-                        } = rel
-                        {
-                            if slru == SlruKind::MultiXactMembers {
-                                for i in 0..xlrec.nmembers {
-                                    let pageno =
-                                        i / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-                                    let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-                                    let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-                                    if segno == rec_segno && rpageno == blknum {
-                                        // update only target block
-                                        let offset = xlrec.moff + i;
-                                        let memberoff = mx_offset_to_member_offset(offset);
-                                        let flagsoff = mx_offset_to_flags_offset(offset);
-                                        let bshift = mx_offset_to_flags_bitshift(offset);
-                                        let mut flagsval =
-                                            LittleEndian::read_u32(&page[flagsoff..flagsoff + 4]);
-                                        flagsval &= !(((1
-                                            << pg_constants::MXACT_MEMBER_BITS_PER_XACT)
-                                            - 1)
+                        if let ObjectTag::MultiXactMembers(slru) = tag {
+                            for i in 0..xlrec.nmembers {
+                                let blkno = i / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
+                                if blkno == slru.blknum {
+                                    // update only target block
+                                    let offset = xlrec.moff + i;
+                                    let memberoff = mx_offset_to_member_offset(offset);
+                                    let flagsoff = mx_offset_to_flags_offset(offset);
+                                    let bshift = mx_offset_to_flags_bitshift(offset);
+                                    let mut flagsval =
+                                        LittleEndian::read_u32(&page[flagsoff..flagsoff + 4]);
+                                    flagsval &=
+                                        !(((1 << pg_constants::MXACT_MEMBER_BITS_PER_XACT) - 1)
                                             << bshift);
-                                        flagsval |= xlrec.members[i as usize].status << bshift;
-                                        LittleEndian::write_u32(
-                                            &mut page[flagsoff..flagsoff + 4],
-                                            flagsval,
-                                        );
-                                        LittleEndian::write_u32(
-                                            &mut page[memberoff..memberoff + 4],
-                                            xlrec.members[i as usize].xid,
-                                        );
-                                    }
+                                    flagsval |= xlrec.members[i as usize].status << bshift;
+                                    LittleEndian::write_u32(
+                                        &mut page[flagsoff..flagsoff + 4],
+                                        flagsval,
+                                    );
+                                    LittleEndian::write_u32(
+                                        &mut page[memberoff..memberoff + 4],
+                                        xlrec.members[i as usize].xid,
+                                    );
                                 }
-                            } else {
-                                // Multixact offsets SLRU
-                                let offs = (xlrec.mid
-                                    % pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32
-                                    * 4) as usize;
-                                LittleEndian::write_u32(&mut page[offs..offs + 4], xlrec.moff);
                             }
                         } else {
-                            panic!();
+                            // Multixact offsets SLRU
+                            let offs = (xlrec.mid % pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32
+                                * 4) as usize;
+                            LittleEndian::write_u32(&mut page[offs..offs + 4], xlrec.moff);
                         }
                     } else {
                         panic!();
                     }
+                } else if xlogrec.xl_rmid == pg_constants::RM_RELMAP_ID {
+                    // Ralation map file has size 512 bytes
+                    page.clear();
+                    page.extend_from_slice(&buf[12..]); // skip xl_relmap_update
+                    assert!(page.len() == 512); // size of pg_filenode.map
                 }
             }
 
@@ -422,7 +462,7 @@ impl PostgresRedoManager {
         );
 
         if let Err(e) = apply_result {
-            error!("could not apply WAL records: {:#}", e);
+            error!("could not apply WAL records: {}", e);
             result = Err(WalRedoError::IoError(e));
         } else {
             let img = apply_result.unwrap();
@@ -435,32 +475,26 @@ impl PostgresRedoManager {
     }
 }
 
-///
-/// Handle to the Postgres WAL redo process
-///
 struct PostgresRedoProcess {
-    stdin: ChildStdin,
-    stdout: ChildStdout,
+    stdin: Arc<RefCell<ChildStdin>>,
+    stdout: Arc<RefCell<ChildStdout>>,
 }
 
 impl PostgresRedoProcess {
     //
     // Start postgres binary in special WAL redo mode.
     //
-    async fn launch(
-        conf: &PageServerConf,
-        tenantid: &ZTenantId,
-    ) -> Result<PostgresRedoProcess, Error> {
+    async fn launch(conf: &PageServerConf, id: usize) -> Result<PostgresRedoProcess, Error> {
         // FIXME: We need a dummy Postgres cluster to run the process in. Currently, we
         // just create one with constant name. That fails if you try to launch more than
         // one WAL redo manager concurrently.
-        let datadir = conf.tenant_path(tenantid).join("wal-redo-datadir");
+        let datadir = conf.workdir.join(format!("wal-redo-datadir-{}", id));
 
         // Create empty data directory for wal-redo postgres, deleting old one first.
         if datadir.exists() {
             info!("directory {:?} exists, removing", &datadir);
             if let Err(e) = fs::remove_dir_all(&datadir) {
-                error!("could not remove old wal-redo-datadir: {:#}", e);
+                error!("could not remove old wal-redo-datadir: {:?}", e);
             }
         }
         info!("running initdb in {:?}", datadir.display());
@@ -533,7 +567,10 @@ impl PostgresRedoProcess {
         };
         tokio::spawn(f_stderr);
 
-        Ok(PostgresRedoProcess { stdin, stdout })
+        Ok(PostgresRedoProcess {
+            stdin: Arc::new(RefCell::new(stdin)),
+            stdout: Arc::new(RefCell::new(stdout)),
+        })
     }
 
     //
@@ -541,14 +578,13 @@ impl PostgresRedoProcess {
     // new page image.
     //
     async fn apply_wal_records(
-        &mut self,
+        &self,
         tag: BufferTag,
         base_img: Option<Bytes>,
-        records: &[(Lsn, WALRecord)],
+        records: &[WALRecord],
     ) -> Result<Bytes, std::io::Error> {
-        let stdout = &mut self.stdout;
-        // Buffer the writes to avoid a lot of small syscalls.
-        let mut stdin = tokio::io::BufWriter::new(&mut self.stdin);
+        let mut stdin = self.stdin.borrow_mut();
+        let mut stdout = self.stdout.borrow_mut();
 
         // We do three things simultaneously: send the old base image and WAL records to
         // the child process's stdin, read the result from child's stdout, and forward any logging
@@ -565,16 +601,20 @@ impl PostgresRedoProcess {
                 stdin.write_all(&build_begin_redo_for_block_msg(tag)),
             )
             .await??;
-            if let Some(img) = base_img {
-                timeout(TIMEOUT, stdin.write_all(&build_push_page_msg(tag, &img))).await??;
+            if base_img.is_some() {
+                timeout(
+                    TIMEOUT,
+                    stdin.write_all(&build_push_page_msg(tag, base_img.unwrap())),
+                )
+                .await??;
             }
 
             // Send WAL records.
-            for (lsn, rec) in records.iter() {
-                WAL_REDO_RECORD_COUNTER.inc();
+            for rec in records.iter() {
+                let r = rec.clone();
 
                 stdin
-                    .write_all(&build_apply_record_msg(*lsn, &rec.rec))
+                    .write_all(&build_apply_record_msg(r.lsn, r.rec))
                     .await?;
 
                 //debug!("sent WAL record to wal redo postgres process ({:X}/{:X}",
@@ -611,41 +651,58 @@ impl PostgresRedoProcess {
 // process. See vendor/postgres/src/backend/tcop/zenith_wal_redo.c for
 // explanation of the protocol.
 
-fn build_begin_redo_for_block_msg(tag: BufferTag) -> Vec<u8> {
+fn build_begin_redo_for_block_msg(tag: BufferTag) -> Bytes {
     let len = 4 + 1 + 4 * 4;
-    let mut buf = Vec::with_capacity(1 + len);
+    let mut buf = BytesMut::with_capacity(1 + len);
 
     buf.put_u8(b'B');
     buf.put_u32(len as u32);
 
-    tag.ser_into(&mut buf)
+    // FIXME: this is a temporary hack that should go away when we refactor
+    // the postgres protocol serialization + handlers.
+    //
+    // BytesMut is a dynamic growable buffer, used a lot in tokio code but
+    // not in the std library. To write to a BytesMut from a serde serializer,
+    // we need to either:
+    // - pre-allocate the required buffer space. This is annoying because we
+    //   shouldn't care what the exact serialized size is-- that's the
+    //   serializer's job.
+    // - Or, we need to create a temporary "writer" (which implements the
+    //   `Write` trait). It's a bit awkward, because the writer consumes the
+    //   underlying BytesMut, and we need to extract it later with
+    //   `into_inner`.
+    let mut writer = buf.writer();
+    tag.ser_into(&mut writer)
         .expect("serialize BufferTag should always succeed");
+    let buf = writer.into_inner();
 
     debug_assert!(buf.len() == 1 + len);
 
-    buf
+    buf.freeze()
 }
 
-fn build_push_page_msg(tag: BufferTag, base_img: &[u8]) -> Vec<u8> {
+fn build_push_page_msg(tag: BufferTag, base_img: Bytes) -> Bytes {
     assert!(base_img.len() == 8192);
 
     let len = 4 + 1 + 4 * 4 + base_img.len();
-    let mut buf = Vec::with_capacity(1 + len);
+    let mut buf = BytesMut::with_capacity(1 + len);
 
     buf.put_u8(b'P');
     buf.put_u32(len as u32);
-    tag.ser_into(&mut buf)
+    let mut writer = buf.writer();
+    tag.ser_into(&mut writer)
         .expect("serialize BufferTag should always succeed");
+    let mut buf = writer.into_inner();
     buf.put(base_img);
 
     debug_assert!(buf.len() == 1 + len);
 
-    buf
+    buf.freeze()
 }
 
-fn build_apply_record_msg(endlsn: Lsn, rec: &[u8]) -> Vec<u8> {
+fn build_apply_record_msg(endlsn: Lsn, rec: Bytes) -> Bytes {
     let len = 4 + 8 + rec.len();
-    let mut buf: Vec<u8> = Vec::with_capacity(1 + len);
+    let mut buf = BytesMut::with_capacity(1 + len);
 
     buf.put_u8(b'A');
     buf.put_u32(len as u32);
@@ -654,19 +711,21 @@ fn build_apply_record_msg(endlsn: Lsn, rec: &[u8]) -> Vec<u8> {
 
     debug_assert!(buf.len() == 1 + len);
 
-    buf
+    buf.freeze()
 }
 
-fn build_get_page_msg(tag: BufferTag) -> Vec<u8> {
+fn build_get_page_msg(tag: BufferTag) -> Bytes {
     let len = 4 + 1 + 4 * 4;
-    let mut buf = Vec::with_capacity(1 + len);
+    let mut buf = BytesMut::with_capacity(1 + len);
 
     buf.put_u8(b'G');
     buf.put_u32(len as u32);
-    tag.ser_into(&mut buf)
+    let mut writer = buf.writer();
+    tag.ser_into(&mut writer)
         .expect("serialize BufferTag should always succeed");
+    let buf = writer.into_inner();
 
     debug_assert!(buf.len() == 1 + len);
 
-    buf
+    buf.freeze()
 }

postgres_ffi/Cargo.toml

@@ -19,9 +19,7 @@ lazy_static = "1.4"
 log = "0.4.14"
 memoffset = "0.6.2"
 thiserror = "1.0"
-serde = { version = "1.0", features = ["derive"] }
 workspace_hack = { path = "../workspace_hack" }
-zenith_utils = { path = "../zenith_utils" }
 
 [build-dependencies]
-bindgen = "0.59.1"
+bindgen = "0.57"

postgres_ffi/README

@@ -13,6 +13,11 @@ in each major PostgreSQL version. Currently, this module is based on
 PostgreSQL v14, but in the future we will probably need a separate
 copy for each PostgreSQL version.
 
+To interact with the C structs, there is some unsafe code in this
+module. Do not copy-paste that to the rest of the codebase! Keep the
+amount of unsafe code to a minimum, and limited to this module only,
+and only where it's truly needed.
+
 TODO: Currently, there is also some code that deals with WAL records
 in pageserver/src/waldecoder.rs.  That should be moved into this
 module. The rest of the codebase should not have intimate knowledge of

postgres_ffi/build.rs

@@ -3,44 +3,6 @@ extern crate bindgen;
 use std::env;
 use std::path::PathBuf;
 
-use bindgen::callbacks::ParseCallbacks;
-
-#[derive(Debug)]
-struct PostgresFfiCallbacks;
-
-impl ParseCallbacks for PostgresFfiCallbacks {
-    fn include_file(&self, filename: &str) {
-        // This does the equivalent of passing bindgen::CargoCallbacks
-        // to the builder .parse_callbacks() method.
-        let cargo_callbacks = bindgen::CargoCallbacks;
-        cargo_callbacks.include_file(filename)
-    }
-
-    // Add any custom #[derive] attributes to the data structures that bindgen
-    // creates.
-    fn add_derives(&self, name: &str) -> Vec<String> {
-        // This is the list of data structures that we want to serialize/deserialize.
-        let serde_list = [
-            "XLogRecord",
-            "XLogPageHeaderData",
-            "XLogLongPageHeaderData",
-            "CheckPoint",
-            "FullTransactionId",
-            "ControlFileData",
-        ];
-
-        if serde_list.contains(&name) {
-            vec![
-                "Default".into(), // Default allows us to easily fill the padding fields with 0.
-                "Serialize".into(),
-                "Deserialize".into(),
-            ]
-        } else {
-            vec![]
-        }
-    }
-}
-
 fn main() {
     // Tell cargo to invalidate the built crate whenever the wrapper changes
     println!("cargo:rerun-if-changed=pg_control_ffi.h");
@@ -57,28 +19,20 @@ fn main() {
         // Tell cargo to invalidate the built crate whenever any of the
         // included header files changed.
         //
-        .parse_callbacks(Box::new(PostgresFfiCallbacks))
+        .parse_callbacks(Box::new(bindgen::CargoCallbacks))
         //
         // These are the types and constants that we want to generate bindings for
         //
-        .allowlist_type("BlockNumber")
-        .allowlist_type("OffsetNumber")
-        .allowlist_type("MultiXactId")
-        .allowlist_type("MultiXactOffset")
-        .allowlist_type("MultiXactStatus")
-        .allowlist_type("ControlFileData")
-        .allowlist_type("CheckPoint")
-        .allowlist_type("FullTransactionId")
-        .allowlist_type("XLogRecord")
-        .allowlist_type("XLogPageHeaderData")
-        .allowlist_type("XLogLongPageHeaderData")
-        .allowlist_var("XLOG_PAGE_MAGIC")
-        .allowlist_var("PG_CONTROL_FILE_SIZE")
-        .allowlist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
-        .allowlist_type("DBState")
-        // Because structs are used for serialization, tell bindgen to emit
-        // explicit padding fields.
-        .explicit_padding(true)
+        .whitelist_type("ControlFileData")
+        .whitelist_type("CheckPoint")
+        .whitelist_type("FullTransactionId")
+        .whitelist_type("XLogRecord")
+        .whitelist_type("XLogPageHeaderData")
+        .whitelist_type("XLogLongPageHeaderData")
+        .whitelist_var("XLOG_PAGE_MAGIC")
+        .whitelist_var("PG_CONTROL_FILE_SIZE")
+        .whitelist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
+        .whitelist_type("DBState")
         //
         // Path the server include dir. It is in tmp_install/include/server, if you did
         // "configure --prefix=<path to tmp_install>". But if you used "configure --prefix=/",

postgres_ffi/pg_control_ffi.h

@@ -8,6 +8,3 @@
 #include "catalog/pg_control.h"
 #include "access/xlog_internal.h"
 
-#include "storage/block.h"
-#include "storage/off.h"
-#include "access/multixact.h"

postgres_ffi/samples/pg_hba.conf

@@ -1,98 +0,0 @@
-# PostgreSQL Client Authentication Configuration File
-# ===================================================
-#
-# Refer to the "Client Authentication" section in the PostgreSQL
-# documentation for a complete description of this file.  A short
-# synopsis follows.
-#
-# This file controls: which hosts are allowed to connect, how clients
-# are authenticated, which PostgreSQL user names they can use, which
-# databases they can access.  Records take one of these forms:
-#
-# local         DATABASE  USER  METHOD  [OPTIONS]
-# host          DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
-# hostssl       DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
-# hostnossl     DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
-# hostgssenc    DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
-# hostnogssenc  DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
-#
-# (The uppercase items must be replaced by actual values.)
-#
-# The first field is the connection type:
-# - "local" is a Unix-domain socket
-# - "host" is a TCP/IP socket (encrypted or not)
-# - "hostssl" is a TCP/IP socket that is SSL-encrypted
-# - "hostnossl" is a TCP/IP socket that is not SSL-encrypted
-# - "hostgssenc" is a TCP/IP socket that is GSSAPI-encrypted
-# - "hostnogssenc" is a TCP/IP socket that is not GSSAPI-encrypted
-#
-# DATABASE can be "all", "sameuser", "samerole", "replication", a
-# database name, or a comma-separated list thereof. The "all"
-# keyword does not match "replication". Access to replication
-# must be enabled in a separate record (see example below).
-#
-# USER can be "all", a user name, a group name prefixed with "+", or a
-# comma-separated list thereof.  In both the DATABASE and USER fields
-# you can also write a file name prefixed with "@" to include names
-# from a separate file.
-#
-# ADDRESS specifies the set of hosts the record matches.  It can be a
-# host name, or it is made up of an IP address and a CIDR mask that is
-# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that
-# specifies the number of significant bits in the mask.  A host name
-# that starts with a dot (.) matches a suffix of the actual host name.
-# Alternatively, you can write an IP address and netmask in separate
-# columns to specify the set of hosts.  Instead of a CIDR-address, you
-# can write "samehost" to match any of the server's own IP addresses,
-# or "samenet" to match any address in any subnet that the server is
-# directly connected to.
-#
-# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256",
-# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert".
-# Note that "password" sends passwords in clear text; "md5" or
-# "scram-sha-256" are preferred since they send encrypted passwords.
-#
-# OPTIONS are a set of options for the authentication in the format
-# NAME=VALUE.  The available options depend on the different
-# authentication methods -- refer to the "Client Authentication"
-# section in the documentation for a list of which options are
-# available for which authentication methods.
-#
-# Database and user names containing spaces, commas, quotes and other
-# special characters must be quoted.  Quoting one of the keywords
-# "all", "sameuser", "samerole" or "replication" makes the name lose
-# its special character, and just match a database or username with
-# that name.
-#
-# This file is read on server startup and when the server receives a
-# SIGHUP signal.  If you edit the file on a running system, you have to
-# SIGHUP the server for the changes to take effect, run "pg_ctl reload",
-# or execute "SELECT pg_reload_conf()".
-#
-# Put your actual configuration here
-# ----------------------------------
-#
-# If you want to allow non-local connections, you need to add more
-# "host" records.  In that case you will also need to make PostgreSQL
-# listen on a non-local interface via the listen_addresses
-# configuration parameter, or via the -i or -h command line switches.
-
-# CAUTION: Configuring the system for local "trust" authentication
-# allows any local user to connect as any PostgreSQL user, including
-# the database superuser.  If you do not trust all your local users,
-# use another authentication method.
-
-
-# TYPE  DATABASE        USER            ADDRESS                 METHOD
-
-# "local" is for Unix domain socket connections only
-local   all             all                                     trust
-# IPv4 local connections:
-host    all             all             127.0.0.1/32            trust
-# IPv6 local connections:
-host    all             all             ::1/128                 trust
-# Allow replication connections from localhost, by a user with the
-# replication privilege.
-local   replication     all                                     trust
-host    replication     all             127.0.0.1/32            trust
-host    replication     all             ::1/128                 trust

postgres_ffi/src/controlfile_utils.rs

@@ -11,7 +11,7 @@
 //! data directory is compatible with a postgres binary. That includes
 //! a version number, configuration options that can be set at
 //! compilation time like the block size, and the platform's alignment
-//! and endianness information. (The PostgreSQL on-disk file format is
+//! and endianess information. (The PostgreSQL on-disk file format is
 //! not portable across platforms.)
 //!
 //! The control file is stored in the PostgreSQL data directory, as
@@ -43,8 +43,6 @@ impl ControlFileData {
     /// Interpret a slice of bytes as a Postgres control file.
     ///
     pub fn decode(buf: &[u8]) -> Result<ControlFileData> {
-        use zenith_utils::bin_ser::LeSer;
-
         // Check that the slice has the expected size. The control file is
         // padded with zeros up to a 512 byte sector size, so accept a
         // larger size too, so that the caller can just the whole file
@@ -57,8 +55,26 @@ impl ControlFileData {
         let OFFSETOF_CRC = Self::pg_control_crc_offset();
         let expectedcrc = crc32c::crc32c(&buf[0..OFFSETOF_CRC]);
 
-        // Use serde to deserialize the input as a ControlFileData struct.
-        let controlfile = ControlFileData::des_prefix(buf)?;
+        // Convert the slice into an array of the right size, and use `transmute` to
+        // reinterpret the raw bytes as a ControlFileData struct.
+        //
+        // NB: Ideally we would use 'zerocopy::FromBytes' for this, but bindgen doesn't
+        // derive FromBytes for us. The safety of this depends on the same constraints
+        // as for FromBytes, namely, all of its fields must implement FromBytes. That
+        // includes the primitive integer types, like `u8`, `u16`, `u32`, `u64` and their
+        // signed variants. But `bool` is not safe, because the contents of the high bits
+        // in a rust bool are undefined. In practice, PostgreSQL uses 1 to represent
+        // true and 0 for false, which is compatible with Rust bool, but let's try not to
+        // depend on it.
+        //
+        // FIXME: ControlFileData does contain 'bool's at the moment.
+        //
+        // See https://github.com/zenithdb/zenith/issues/207 for discussion on the safety
+        // of this.
+        let mut b: [u8; SIZEOF_CONTROLDATA] = [0u8; SIZEOF_CONTROLDATA];
+        b.copy_from_slice(&buf[0..SIZEOF_CONTROLDATA]);
+        let controlfile: ControlFileData =
+            unsafe { std::mem::transmute::<[u8; SIZEOF_CONTROLDATA], ControlFileData>(b) };
 
         // Check the CRC
         if expectedcrc != controlfile.crc {
@@ -77,10 +93,21 @@ impl ControlFileData {
     ///
     /// The CRC is recomputed to match the contents of the fields.
     pub fn encode(&self) -> Bytes {
-        use zenith_utils::bin_ser::LeSer;
-
-        // Serialize into a new buffer.
-        let b = self.ser().unwrap();
+        //
+        // Use `transmute` to reinterpret struct as raw bytes.
+        //
+        // FIXME: This triggers undefined behavior, because the contents
+        // of the padding bytes are undefined, and this leaks those
+        // undefined bytes into the resulting array. The Rust code won't
+        // care what's in those bytes, and PostgreSQL doesn't care
+        // either. HOWEVER, it is a potential security issue, because the
+        // bytes can contain arbitrary pieces of memory from the page
+        // server. In the worst case, that could be private keys or
+        // another tenant's data.
+        //
+        // See https://github.com/zenithdb/zenith/issues/207 for discussion.
+        let b: [u8; SIZEOF_CONTROLDATA] =
+            unsafe { std::mem::transmute::<ControlFileData, [u8; SIZEOF_CONTROLDATA]>(*self) };
 
         // Recompute the CRC
         let OFFSETOF_CRC = Self::pg_control_crc_offset();

postgres_ffi/src/lib.rs

@@ -1,12 +1,6 @@
 #![allow(non_upper_case_globals)]
 #![allow(non_camel_case_types)]
 #![allow(non_snake_case)]
-// suppress warnings on rust 1.53 due to bindgen unit tests.
-// https://github.com/rust-lang/rust-bindgen/issues/1651
-#![allow(deref_nullptr)]
-
-use serde::{Deserialize, Serialize};
-
 include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
 
 pub mod controlfile_utils;
@@ -14,23 +8,3 @@ pub mod nonrelfile_utils;
 pub mod pg_constants;
 pub mod relfile_utils;
 pub mod xlog_utils;
-
-//  See TransactionIdIsNormal in transam.h
-pub const fn transaction_id_is_normal(id: TransactionId) -> bool {
-    id > pg_constants::FIRST_NORMAL_TRANSACTION_ID
-}
-
-// See TransactionIdPrecedes in transam.c
-pub const fn transaction_id_precedes(id1: TransactionId, id2: TransactionId) -> bool {
-    /*
-     * If either ID is a permanent XID then we can just do unsigned
-     * comparison.  If both are normal, do a modulo-2^32 comparison.
-     */
-
-    if !(transaction_id_is_normal(id1)) || !transaction_id_is_normal(id2) {
-        return id1 < id2;
-    }
-
-    let diff = id1.wrapping_sub(id2) as i32;
-    diff < 0
-}

postgres_ffi/src/nonrelfile_utils.rs

@@ -1,12 +1,10 @@
 //!
 //! Common utilities for dealing with PostgreSQL non-relation files.
 //!
-use crate::{pg_constants, transaction_id_precedes};
+use crate::pg_constants;
 use bytes::BytesMut;
 use log::*;
 
-use crate::MultiXactId;
-
 pub fn transaction_id_set_status(xid: u32, status: u8, page: &mut BytesMut) {
     trace!(
         "handle_apply_request for RM_XACT_ID-{} (1-commit, 2-abort, 3-sub_commit)",
@@ -32,51 +30,3 @@ pub fn transaction_id_get_status(xid: u32, page: &[u8]) -> u8 {
 
     ((page[byteno] >> bshift) & pg_constants::CLOG_XACT_BITMASK) as u8
 }
-
-// See CLOGPagePrecedes in clog.c
-pub const fn clogpage_precedes(page1: u32, page2: u32) -> bool {
-    let mut xid1 = page1 * pg_constants::CLOG_XACTS_PER_PAGE;
-    xid1 += pg_constants::FIRST_NORMAL_TRANSACTION_ID + 1;
-    let mut xid2 = page2 * pg_constants::CLOG_XACTS_PER_PAGE;
-    xid2 += pg_constants::FIRST_NORMAL_TRANSACTION_ID + 1;
-
-    transaction_id_precedes(xid1, xid2)
-        && transaction_id_precedes(xid1, xid2 + pg_constants::CLOG_XACTS_PER_PAGE - 1)
-}
-
-// See SlruMayDeleteSegment() in slru.c
-pub fn slru_may_delete_clogsegment(segpage: u32, cutoff_page: u32) -> bool {
-    let seg_last_page = segpage + pg_constants::SLRU_PAGES_PER_SEGMENT - 1;
-
-    assert_eq!(segpage % pg_constants::SLRU_PAGES_PER_SEGMENT, 0);
-
-    clogpage_precedes(segpage, cutoff_page) && clogpage_precedes(seg_last_page, cutoff_page)
-}
-
-// Multixact utils
-
-pub fn mx_offset_to_flags_offset(xid: MultiXactId) -> usize {
-    ((xid / pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP as u32) as u16
-        % pg_constants::MULTIXACT_MEMBERGROUPS_PER_PAGE
-        * pg_constants::MULTIXACT_MEMBERGROUP_SIZE) as usize
-}
-
-pub fn mx_offset_to_flags_bitshift(xid: MultiXactId) -> u16 {
-    (xid as u16) % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP
-        * pg_constants::MXACT_MEMBER_BITS_PER_XACT
-}
-
-/* Location (byte offset within page) of TransactionId of given member */
-pub fn mx_offset_to_member_offset(xid: MultiXactId) -> usize {
-    mx_offset_to_flags_offset(xid)
-        + (pg_constants::MULTIXACT_FLAGBYTES_PER_GROUP
-            + (xid as u16 % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP) * 4) as usize
-}
-
-fn mx_offset_to_member_page(xid: u32) -> u32 {
-    xid / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32
-}
-
-pub fn mx_offset_to_member_segment(xid: u32) -> i32 {
-    (mx_offset_to_member_page(xid) / pg_constants::SLRU_PAGES_PER_SEGMENT) as i32
-}

postgres_ffi/src/pg_constants.rs

@@ -46,6 +46,7 @@ pub const SIZE_OF_PAGE_HEADER: u16 = 24;
 pub const BITS_PER_HEAPBLOCK: u16 = 2;
 pub const HEAPBLOCKS_PER_PAGE: u16 = (BLCKSZ - SIZE_OF_PAGE_HEADER) * 8 / BITS_PER_HEAPBLOCK;
 
+pub const TRANSACTION_STATUS_IN_PROGRESS: u8 = 0x00;
 pub const TRANSACTION_STATUS_COMMITTED: u8 = 0x01;
 pub const TRANSACTION_STATUS_ABORTED: u8 = 0x02;
 pub const TRANSACTION_STATUS_SUB_COMMITTED: u8 = 0x03;
@@ -87,14 +88,11 @@ pub const XACT_XINFO_HAS_TWOPHASE: u32 = 1u32 << 4;
 pub const XLOG_NEXTOID: u8 = 0x30;
 pub const XLOG_SWITCH: u8 = 0x40;
 pub const XLOG_SMGR_TRUNCATE: u8 = 0x20;
-pub const XLOG_FPI_FOR_HINT: u8 = 0xA0;
-pub const XLOG_FPI: u8 = 0xB0;
 pub const DB_SHUTDOWNED: u32 = 1;
 
 // From multixact.h
 pub const FIRST_MULTIXACT_ID: u32 = 1;
 pub const MAX_MULTIXACT_ID: u32 = 0xFFFFFFFF;
-pub const MAX_MULTIXACT_OFFSET: u32 = 0xFFFFFFFF;
 
 pub const XLOG_MULTIXACT_ZERO_OFF_PAGE: u8 = 0x00;
 pub const XLOG_MULTIXACT_ZERO_MEM_PAGE: u8 = 0x10;
@@ -119,7 +117,6 @@ pub const XLOG_HEAP_INSERT: u8 = 0x00;
 pub const XLOG_HEAP_DELETE: u8 = 0x10;
 pub const XLOG_HEAP_UPDATE: u8 = 0x20;
 pub const XLOG_HEAP_HOT_UPDATE: u8 = 0x40;
-pub const XLOG_HEAP_INIT_PAGE: u8 = 0x80;
 pub const XLOG_HEAP2_VISIBLE: u8 = 0x40;
 pub const XLOG_HEAP2_MULTI_INSERT: u8 = 0x50;
 pub const XLH_INSERT_ALL_FROZEN_SET: u8 = (1 << 5) as u8;
@@ -188,41 +185,3 @@ pub const XLOG_BLCKSZ: usize = 8192;
 pub const XLOG_CHECKPOINT_SHUTDOWN: u8 = 0x00;
 pub const XLOG_CHECKPOINT_ONLINE: u8 = 0x10;
 pub const XLP_LONG_HEADER: u16 = 0x0002;
-
-pub const PG_MAJORVERSION: &str = "14";
-
-// List of subdirectories inside pgdata.
-// Copied from src/bin/initdb/initdb.c
-pub const PGDATA_SUBDIRS: [&str; 22] = [
-    "global",
-    "pg_wal/archive_status",
-    "pg_commit_ts",
-    "pg_dynshmem",
-    "pg_notify",
-    "pg_serial",
-    "pg_snapshots",
-    "pg_subtrans",
-    "pg_twophase",
-    "pg_multixact",
-    "pg_multixact/members",
-    "pg_multixact/offsets",
-    "base",
-    "base/1",
-    "pg_replslot",
-    "pg_tblspc",
-    "pg_stat",
-    "pg_stat_tmp",
-    "pg_xact",
-    "pg_logical",
-    "pg_logical/snapshots",
-    "pg_logical/mappings",
-];
-
-// Don't include postgresql.conf as it is inconvenient on node start:
-// we need postgresql.conf before basebackup to synchronize safekeepers
-// so no point in overwriting it during backup restore. Rest of the files
-// here are not needed before backup so it is okay to edit them after.
-pub const PGDATA_SPECIAL_FILES: [&str; 3] =
-    ["pg_hba.conf", "pg_ident.conf", "postgresql.auto.conf"];
-
-pub static PG_HBA: &str = include_str!("../samples/pg_hba.conf");

postgres_ffi/src/xlog_utils.rs

@@ -16,20 +16,16 @@ use crate::XLogPageHeaderData;
 use crate::XLogRecord;
 use crate::XLOG_PAGE_MAGIC;
 
-use anyhow::{bail, Result};
 use byteorder::{ByteOrder, LittleEndian};
 use bytes::{Buf, Bytes};
 use bytes::{BufMut, BytesMut};
 use crc32c::*;
 use log::*;
-use std::cmp::max;
 use std::cmp::min;
 use std::fs::{self, File};
 use std::io::prelude::*;
-use std::io::SeekFrom;
 use std::path::{Path, PathBuf};
 use std::time::SystemTime;
-use zenith_utils::lsn::Lsn;
 
 pub const XLOG_FNAME_LEN: usize = 24;
 pub const XLOG_BLCKSZ: usize = 8192;
@@ -41,7 +37,6 @@ pub const MAX_SEND_SIZE: usize = XLOG_BLCKSZ * 16;
 pub const XLOG_SIZE_OF_XLOG_SHORT_PHD: usize = std::mem::size_of::<XLogPageHeaderData>();
 pub const XLOG_SIZE_OF_XLOG_LONG_PHD: usize = std::mem::size_of::<XLogLongPageHeaderData>();
 pub const XLOG_SIZE_OF_XLOG_RECORD: usize = std::mem::size_of::<XLogRecord>();
-#[allow(clippy::identity_op)]
 pub const SIZE_OF_XLOG_RECORD_DATA_HEADER_SHORT: usize = 1 * 2;
 
 pub type XLogRecPtr = u64;
@@ -93,21 +88,6 @@ pub fn IsPartialXLogFileName(fname: &str) -> bool {
     fname.ends_with(".partial") && IsXLogFileName(&fname[0..fname.len() - 8])
 }
 
-/// If LSN points to the beginning of the page, then shift it to first record,
-/// otherwise align on 8-bytes boundary (required for WAL records)
-pub fn normalize_lsn(lsn: Lsn, seg_sz: usize) -> Lsn {
-    if lsn.0 % XLOG_BLCKSZ as u64 == 0 {
-        let hdr_size = if lsn.0 % seg_sz as u64 == 0 {
-            XLOG_SIZE_OF_XLOG_LONG_PHD
-        } else {
-            XLOG_SIZE_OF_XLOG_SHORT_PHD
-        };
-        lsn + hdr_size as u64
-    } else {
-        lsn.align()
-    }
-}
-
 pub fn get_current_timestamp() -> TimestampTz {
     const UNIX_EPOCH_JDATE: u64 = 2440588; /* == date2j(1970, 1, 1) */
     const POSTGRES_EPOCH_JDATE: u64 = 2451545; /* == date2j(2000, 1, 1) */
@@ -128,10 +108,8 @@ fn find_end_of_wal_segment(
     segno: XLogSegNo,
     tli: TimeLineID,
     wal_seg_size: usize,
-    start_offset: usize, // start reading at this point
-) -> Result<u32> {
-    // step back to the beginning of the page to read it in...
-    let mut offs: usize = start_offset - start_offset % XLOG_BLCKSZ;
+) -> u32 {
+    let mut offs: usize = 0;
     let mut contlen: usize = 0;
     let mut wal_crc: u32 = 0;
     let mut crc: u32 = 0;
@@ -140,33 +118,23 @@ fn find_end_of_wal_segment(
     let file_name = XLogFileName(tli, segno, wal_seg_size);
     let mut last_valid_rec_pos: usize = 0;
     let mut file = File::open(data_dir.join(file_name.clone() + ".partial")).unwrap();
-    file.seek(SeekFrom::Start(offs as u64))?;
     let mut rec_hdr = [0u8; XLOG_RECORD_CRC_OFFS];
 
     while offs < wal_seg_size {
-        // we are at the beginning of the page; read it in
         if offs % XLOG_BLCKSZ == 0 {
-            let bytes_read = file.read(&mut buf)?;
-            if bytes_read != buf.len() {
-                bail!(
-                    "failed to read {} bytes from {} at {}",
-                    XLOG_BLCKSZ,
-                    file_name,
-                    offs
-                );
+            if let Ok(bytes_read) = file.read(&mut buf) {
+                if bytes_read != buf.len() {
+                    break;
+                }
+            } else {
+                break;
             }
-
             let xlp_magic = LittleEndian::read_u16(&buf[0..2]);
             let xlp_info = LittleEndian::read_u16(&buf[2..4]);
             let xlp_rem_len = LittleEndian::read_u32(&buf[XLP_REM_LEN_OFFS..XLP_REM_LEN_OFFS + 4]);
-            // this is expected in current usage when valid WAL starts after page header
             if xlp_magic != XLOG_PAGE_MAGIC as u16 {
-                trace!(
-                    "invalid WAL file {}.partial magic {} at {:?}",
-                    file_name,
-                    xlp_magic,
-                    Lsn(XLogSegNoOffsetToRecPtr(segno, offs as u32, wal_seg_size)),
-                );
+                info!("Invalid WAL file {}.partial magic {}", file_name, xlp_magic);
+                break;
             }
             if offs == 0 {
                 offs = XLOG_SIZE_OF_XLOG_LONG_PHD;
@@ -176,19 +144,11 @@ fn find_end_of_wal_segment(
             } else {
                 offs += XLOG_SIZE_OF_XLOG_SHORT_PHD;
             }
-            // ... and step forward again if asked
-            offs = max(offs, start_offset);
-
-        // beginning of the next record
         } else if contlen == 0 {
             let page_offs = offs % XLOG_BLCKSZ;
             let xl_tot_len = LittleEndian::read_u32(&buf[page_offs..page_offs + 4]) as usize;
             if xl_tot_len == 0 {
-                info!(
-                    "find_end_of_wal_segment reached zeros at {:?}",
-                    Lsn(XLogSegNoOffsetToRecPtr(segno, offs as u32, wal_seg_size))
-                );
-                break; // zeros, reached the end
+                break;
             }
             last_valid_rec_pos = offs;
             offs += 4;
@@ -196,13 +156,12 @@ fn find_end_of_wal_segment(
             contlen = xl_tot_len - 4;
             rec_hdr[0..4].copy_from_slice(&buf[page_offs..page_offs + 4]);
         } else {
-            // we're continuing a record, possibly from previous page.
             let page_offs = offs % XLOG_BLCKSZ;
+            // we're continuing a record, possibly from previous page.
             let pageleft = XLOG_BLCKSZ - page_offs;
 
             // read the rest of the record, or as much as fits on this page.
             let n = min(contlen, pageleft);
-            // fill rec_hdr (header up to (but not including) xl_crc field)
             if rec_offs < XLOG_RECORD_CRC_OFFS {
                 let len = min(XLOG_RECORD_CRC_OFFS - rec_offs, n);
                 rec_hdr[rec_offs..rec_offs + len].copy_from_slice(&buf[page_offs..page_offs + len]);
@@ -211,11 +170,12 @@ fn find_end_of_wal_segment(
                 let crc_offs = page_offs - rec_offs + XLOG_RECORD_CRC_OFFS;
                 wal_crc = LittleEndian::read_u32(&buf[crc_offs..crc_offs + 4]);
                 crc = crc32c_append(0, &buf[crc_offs + 4..page_offs + n]);
+                crc = !crc;
             } else {
                 crc ^= 0xFFFFFFFFu32;
                 crc = crc32c_append(crc, &buf[page_offs..page_offs + n]);
+                crc = !crc;
             }
-            crc = !crc;
             rec_offs += n;
             offs += n;
             contlen -= n;
@@ -225,8 +185,6 @@ fn find_end_of_wal_segment(
                 crc = crc32c_append(crc, &rec_hdr);
                 offs = (offs + 7) & !7; // pad on 8 bytes boundary */
                 if crc == wal_crc {
-                    // record is valid, advance the result to its end (with
-                    // alignment to the next record taken into account)
                     last_valid_rec_pos = offs;
                 } else {
                     info!(
@@ -238,21 +196,17 @@ fn find_end_of_wal_segment(
             }
         }
     }
-    Ok(last_valid_rec_pos as u32)
+    last_valid_rec_pos as u32
 }
 
 ///
 /// Scan a directory that contains PostgreSQL WAL files, for the end of WAL.
-/// If precise, returns end LSN (next insertion point, basically);
-/// otherwise, start of the last segment.
-/// Returns (0, 0) if there is no WAL.
 ///
 pub fn find_end_of_wal(
     data_dir: &Path,
     wal_seg_size: usize,
     precise: bool,
-    start_lsn: Lsn, // start reading WAL at this point or later
-) -> Result<(XLogRecPtr, TimeLineID)> {
+) -> (XLogRecPtr, TimeLineID) {
     let mut high_segno: XLogSegNo = 0;
     let mut high_tli: TimeLineID = 0;
     let mut high_ispartial = false;
@@ -294,32 +248,19 @@ pub fn find_end_of_wal(
             high_segno += 1;
         } else if precise {
             /* otherwise locate last record in last partial segment */
-            if start_lsn.segment_number(wal_seg_size) > high_segno {
-                bail!(
-                    "provided start_lsn {:?} is beyond highest segno {:?} available",
-                    start_lsn,
-                    high_segno,
-                );
-            }
-            high_offs = find_end_of_wal_segment(
-                data_dir,
-                high_segno,
-                high_tli,
-                wal_seg_size,
-                start_lsn.segment_offset(wal_seg_size),
-            )?;
+            high_offs = find_end_of_wal_segment(data_dir, high_segno, high_tli, wal_seg_size);
         }
         let high_ptr = XLogSegNoOffsetToRecPtr(high_segno, high_offs, wal_seg_size);
-        return Ok((high_ptr, high_tli));
+        return (high_ptr, high_tli);
     }
-    Ok((0, 0))
+    (0, 0)
 }
 
 pub fn main() {
     let mut data_dir = PathBuf::new();
     data_dir.push(".");
     let wal_seg_size = 16 * 1024 * 1024;
-    let (wal_end, tli) = find_end_of_wal(&data_dir, wal_seg_size, true, Lsn(0)).unwrap();
+    let (wal_end, tli) = find_end_of_wal(&data_dir, wal_seg_size, true);
     println!(
         "wal_end={:>08X}{:>08X}, tli={}",
         (wal_end >> 32) as u32,
@@ -329,19 +270,24 @@ pub fn main() {
 }
 
 impl XLogRecord {
-    pub fn from_slice(buf: &[u8]) -> XLogRecord {
-        use zenith_utils::bin_ser::LeSer;
-        XLogRecord::des(buf).unwrap()
-    }
-
-    pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogRecord {
-        use zenith_utils::bin_ser::LeSer;
-        XLogRecord::des_from(&mut buf.reader()).unwrap()
+    pub fn from_bytes(buf: &mut Bytes) -> XLogRecord {
+        XLogRecord {
+            xl_tot_len: buf.get_u32_le(),
+            xl_xid: buf.get_u32_le(),
+            xl_prev: buf.get_u64_le(),
+            xl_info: buf.get_u8(),
+            xl_rmid: buf.get_u8(),
+            xl_crc: {
+                buf.advance(2);
+                buf.get_u32_le()
+            },
+        }
     }
 
     pub fn encode(&self) -> Bytes {
-        use zenith_utils::bin_ser::LeSer;
-        self.ser().unwrap().into()
+        let b: [u8; XLOG_SIZE_OF_XLOG_RECORD];
+        b = unsafe { std::mem::transmute::<XLogRecord, [u8; XLOG_SIZE_OF_XLOG_RECORD]>(*self) };
+        Bytes::copy_from_slice(&b[..])
     }
 
     // Is this record an XLOG_SWITCH record? They need some special processing,
@@ -352,42 +298,81 @@ impl XLogRecord {
 
 impl XLogPageHeaderData {
     pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogPageHeaderData {
-        use zenith_utils::bin_ser::LeSer;
-        XLogPageHeaderData::des_from(&mut buf.reader()).unwrap()
+        let hdr: XLogPageHeaderData = XLogPageHeaderData {
+            xlp_magic: buf.get_u16_le(),
+            xlp_info: buf.get_u16_le(),
+            xlp_tli: buf.get_u32_le(),
+            xlp_pageaddr: buf.get_u64_le(),
+            xlp_rem_len: buf.get_u32_le(),
+        };
+        buf.get_u32_le(); //padding
+        hdr
     }
 }
 
 impl XLogLongPageHeaderData {
     pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogLongPageHeaderData {
-        use zenith_utils::bin_ser::LeSer;
-        XLogLongPageHeaderData::des_from(&mut buf.reader()).unwrap()
+        XLogLongPageHeaderData {
+            std: XLogPageHeaderData::from_bytes(buf),
+            xlp_sysid: buf.get_u64_le(),
+            xlp_seg_size: buf.get_u32_le(),
+            xlp_xlog_blcksz: buf.get_u32_le(),
+        }
     }
 
     pub fn encode(&self) -> Bytes {
-        use zenith_utils::bin_ser::LeSer;
-        self.ser().unwrap().into()
+        let b: [u8; XLOG_SIZE_OF_XLOG_LONG_PHD];
+        b = unsafe {
+            std::mem::transmute::<XLogLongPageHeaderData, [u8; XLOG_SIZE_OF_XLOG_LONG_PHD]>(*self)
+        };
+        Bytes::copy_from_slice(&b[..])
     }
 }
 
 pub const SIZEOF_CHECKPOINT: usize = std::mem::size_of::<CheckPoint>();
 
 impl CheckPoint {
+    pub fn new(lsn: u64, timeline: u32) -> CheckPoint {
+        CheckPoint {
+            redo: lsn,
+            ThisTimeLineID: timeline,
+            PrevTimeLineID: timeline,
+            fullPageWrites: true, // TODO: get actual value of full_page_writes
+            nextXid: FullTransactionId {
+                value: pg_constants::FIRST_NORMAL_TRANSACTION_ID as u64,
+            }, // TODO: handle epoch?
+            nextOid: pg_constants::FIRST_BOOTSTRAP_OBJECT_ID,
+            nextMulti: 1,
+            nextMultiOffset: 0,
+            oldestXid: pg_constants::FIRST_NORMAL_TRANSACTION_ID,
+            oldestXidDB: 0,
+            oldestMulti: 1,
+            oldestMultiDB: 0,
+            time: 0,
+            oldestCommitTsXid: 0,
+            newestCommitTsXid: 0,
+            oldestActiveXid: pg_constants::INVALID_TRANSACTION_ID,
+        }
+    }
+
     pub fn encode(&self) -> Bytes {
-        use zenith_utils::bin_ser::LeSer;
-        self.ser().unwrap().into()
+        let b: [u8; SIZEOF_CHECKPOINT];
+        b = unsafe { std::mem::transmute::<CheckPoint, [u8; SIZEOF_CHECKPOINT]>(*self) };
+        Bytes::copy_from_slice(&b[..])
     }
 
     pub fn decode(buf: &[u8]) -> Result<CheckPoint, anyhow::Error> {
-        use zenith_utils::bin_ser::LeSer;
-        Ok(CheckPoint::des(buf)?)
+        let mut b = [0u8; SIZEOF_CHECKPOINT];
+        b.copy_from_slice(&buf[0..SIZEOF_CHECKPOINT]);
+        let checkpoint: CheckPoint;
+        checkpoint = unsafe { std::mem::transmute::<[u8; SIZEOF_CHECKPOINT], CheckPoint>(b) };
+        Ok(checkpoint)
     }
 
-    /// Update next XID based on provided new_xid and stored epoch.
-    /// Next XID should be greater than new_xid. This handles 32-bit
-    /// XID wraparound correctly.
-    ///
-    /// Returns 'true' if the XID was updated.
-    pub fn update_next_xid(&mut self, xid: u32) -> bool {
+    // Update next XID based on provided new_xid and stored epoch.
+    // Next XID should be greater than new_xid.
+    // Also take in account 32-bit wrap-around.
+    pub fn update_next_xid(&mut self, xid: u32) {
         let xid = xid.wrapping_add(XID_CHECKPOINT_INTERVAL - 1) & !(XID_CHECKPOINT_INTERVAL - 1);
         let full_xid = self.nextXid.value;
         let new_xid = std::cmp::max(xid + 1, pg_constants::FIRST_NORMAL_TRANSACTION_ID);
@@ -398,14 +383,10 @@ impl CheckPoint {
                 // wrap-around
                 epoch += 1;
             }
-            let nextXid = (epoch << 32) | new_xid as u64;
-
-            if nextXid != self.nextXid.value {
-                self.nextXid = FullTransactionId { value: nextXid };
-                return true;
-            }
+            self.nextXid = FullTransactionId {
+                value: (epoch << 32) | new_xid as u64,
+            };
         }
-        false
     }
 }
 
@@ -427,7 +408,6 @@ pub fn generate_wal_segment(pg_control: &ControlFileData) -> Bytes {
                 xlp_tli: 1, // FIXME: always use Postgres timeline 1
                 xlp_pageaddr: pg_control.checkPoint - XLOG_SIZE_OF_XLOG_LONG_PHD as u64,
                 xlp_rem_len: 0,
-                ..Default::default() // Put 0 in padding fields.
             }
         },
         xlp_sysid: pg_control.system_identifier,
@@ -447,7 +427,6 @@ pub fn generate_wal_segment(pg_control: &ControlFileData) -> Bytes {
         xl_info: pg_constants::XLOG_CHECKPOINT_SHUTDOWN,
         xl_rmid: pg_constants::RM_XLOG_ID,
         xl_crc: 0,
-        ..Default::default() // Put 0 in padding fields.
     };
 
     let mut rec_shord_hdr_bytes = BytesMut::new();
@@ -472,72 +451,3 @@ pub fn generate_wal_segment(pg_control: &ControlFileData) -> Bytes {
     seg_buf.resize(pg_constants::WAL_SEGMENT_SIZE, 0);
     seg_buf.freeze()
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use regex::Regex;
-    use std::{env, process::Command, str::FromStr};
-
-    // Run find_end_of_wal against file in test_wal dir
-    // Ensure that it finds last record correctly
-    #[test]
-    pub fn test_find_end_of_wal() {
-        // 1. Run initdb to generate some WAL
-        let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
-        let data_dir = top_path.join("test_output/test_find_end_of_wal");
-        let initdb_path = top_path.join("tmp_install/bin/initdb");
-        let lib_path = top_path.join("tmp_install/lib");
-        if data_dir.exists() {
-            fs::remove_dir_all(&data_dir).unwrap();
-        }
-        println!("Using initdb from '{}'", initdb_path.display());
-        println!("Data directory '{}'", data_dir.display());
-        let initdb_output = Command::new(initdb_path)
-            .args(&["-D", data_dir.to_str().unwrap()])
-            .arg("--no-instructions")
-            .arg("--no-sync")
-            .env_clear()
-            .env("LD_LIBRARY_PATH", &lib_path)
-            .env("DYLD_LIBRARY_PATH", &lib_path)
-            .output()
-            .unwrap();
-        assert!(initdb_output.status.success());
-
-        // 2. Pick WAL generated by initdb
-        let wal_dir = data_dir.join("pg_wal");
-        let wal_seg_size = 16 * 1024 * 1024;
-
-        // 3. Check end_of_wal on non-partial WAL segment (we treat it as fully populated)
-        let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true, Lsn(0)).unwrap();
-        let wal_end = Lsn(wal_end);
-        println!("wal_end={}, tli={}", wal_end, tli);
-        assert_eq!(wal_end, "0/2000000".parse::<Lsn>().unwrap());
-
-        // 4. Get the actual end of WAL by pg_waldump
-        let waldump_path = top_path.join("tmp_install/bin/pg_waldump");
-        let waldump_output = Command::new(waldump_path)
-            .arg(wal_dir.join("000000010000000000000001"))
-            .env_clear()
-            .env("LD_LIBRARY_PATH", &lib_path)
-            .env("DYLD_LIBRARY_PATH", &lib_path)
-            .output()
-            .unwrap();
-        let waldump_output = std::str::from_utf8(&waldump_output.stderr).unwrap();
-        println!("waldump_output = '{}'", &waldump_output);
-        let re = Regex::new(r"invalid record length at (.+):").unwrap();
-        let caps = re.captures(waldump_output).unwrap();
-        let waldump_wal_end = Lsn::from_str(caps.get(1).unwrap().as_str()).unwrap();
-
-        // 5. Rename file to partial to actually find last valid lsn
-        fs::rename(
-            wal_dir.join("000000010000000000000001"),
-            wal_dir.join("000000010000000000000001.partial"),
-        )
-        .unwrap();
-        let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true, Lsn(0)).unwrap();
-        let wal_end = Lsn(wal_end);
-        println!("wal_end={}, tli={}", wal_end, tli);
-        assert_eq!(wal_end, waldump_wal_end);
-    }
-}

pre-commit.py

@@ -1,89 +0,0 @@
-#!/usr/bin/env python3
-
-from typing import List
-import subprocess
-import sys
-import enum
-import argparse
-import os
-
-
-@enum.unique
-class Color(enum.Enum):
-    RED = "\033[0;31m"
-    GREEN = "\033[0;33m"
-    CYAN = "\033[0;36m"
-
-
-NC = "\033[0m"  # No Color
-
-
-def colorify(
-    s: str,
-    color: Color,
-    no_color: bool = False,
-):
-    if no_color:
-        return s
-    return f"{color.value}{s}{NC}"
-
-
-def rustfmt(fix_inplace: bool = False, no_color: bool = False) -> str:
-    cmd = "rustfmt --edition=2018"
-    if not fix_inplace:
-        cmd += " --check"
-    if no_color:
-        cmd += " --color=never"
-    return cmd
-
-
-def get_commit_files() -> List[str]:
-    files = subprocess.check_output(
-        "git diff --cached --name-only --diff-filter=ACM".split()
-    )
-    return files.decode().splitlines()
-
-
-def check(
-    name: str, suffix: str, cmd: str, changed_files: List[str], no_color: bool = False
-):
-    print(f"Checking: {name} ", end="")
-    applicable_files = list(
-        filter(lambda fname: fname.strip().endswith(suffix), changed_files)
-    )
-    if not applicable_files:
-        print(colorify("[NOT APPLICABLE]", Color.CYAN, no_color))
-        return
-
-    cmd = f'{cmd} {" ".join(applicable_files)}'
-    res = subprocess.run(cmd.split(), capture_output=True)
-    if res.returncode != 0:
-        print(colorify("[FAILED]", Color.RED, no_color))
-        print("Please inspect the output below and run make fmt to fix automatically\n")
-        print(res.stdout.decode())
-        exit(1)
-
-    print(colorify("[OK]", Color.GREEN, no_color))
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--fix-inplace", action="store_true", help="apply fixes inplace"
-    )
-    parser.add_argument(
-        "--no-color", action="store_true", help="disable colored output", default=not sys.stdout.isatty()
-    )
-    args = parser.parse_args()
-
-    files = get_commit_files()
-    # we use rustfmt here because cargo fmt does not accept list of files
-    # it internally gathers project files and feeds them to rustfmt
-    # so because we want to check only files included in the commit we use rustfmt directly
-    check(
-        name="rustfmt",
-        suffix=".rs",
-        cmd=rustfmt(fix_inplace=args.fix_inplace, no_color=args.no_color),
-        changed_files=files,
-        no_color=args.no_color,
-    )

proxy/Cargo.toml

@@ -14,10 +14,8 @@ rand = "0.8.3"
 hex = "0.4.3"
 serde = "1"
 serde_json = "1"
-tokio = "1.11"
-tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
+tokio = { version = "1.7.1", features = ["full"] }
+tokio-postgres = "0.7.2"
 clap = "2.33.0"
-rustls = "0.19.1"
-reqwest = { version = "0.11", features = ["blocking", "json"] }
 
 zenith_utils = { path = "../zenith_utils" }

proxy/src/cplane_api.rs

@@ -1,14 +1,17 @@
-use anyhow::{bail, Context, Result};
+use anyhow::{bail, Result};
 use serde::{Deserialize, Serialize};
-use std::net::{SocketAddr, ToSocketAddrs};
+use std::{
+    collections::HashMap,
+    net::{IpAddr, SocketAddr},
+};
 
 pub struct CPlaneApi {
-    auth_endpoint: &'static str,
+    // address: SocketAddr,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize)]
 pub struct DatabaseInfo {
-    pub host: String,
+    pub host: IpAddr, // TODO: allow host name here too
     pub port: u16,
     pub dbname: String,
     pub user: String,
@@ -16,13 +19,8 @@ pub struct DatabaseInfo {
 }
 
 impl DatabaseInfo {
-    pub fn socket_addr(&self) -> Result<SocketAddr> {
-        let host_port = format!("{}:{}", self.host, self.port);
-        host_port
-            .to_socket_addrs()
-            .with_context(|| format!("cannot resolve {} to SocketAddr", host_port))?
-            .next()
-            .ok_or_else(|| anyhow::Error::msg("cannot resolve at least one SocketAddr"))
+    pub fn socket_addr(&self) -> SocketAddr {
+        SocketAddr::new(self.host, self.port)
     }
 
     pub fn conn_string(&self) -> String {
@@ -33,35 +31,62 @@ impl DatabaseInfo {
     }
 }
 
+// mock cplane api
 impl CPlaneApi {
-    pub fn new(auth_endpoint: &'static str) -> CPlaneApi {
-        CPlaneApi { auth_endpoint }
+    pub fn new(_address: &SocketAddr) -> CPlaneApi {
+        CPlaneApi {
+            // address: address.clone(),
+        }
     }
 
-    pub fn authenticate_proxy_request(
-        &self,
-        user: &str,
-        database: &str,
-        md5_response: &[u8],
-        salt: &[u8; 4],
-    ) -> Result<DatabaseInfo> {
-        let mut url = reqwest::Url::parse(self.auth_endpoint)?;
-        url.query_pairs_mut()
-            .append_pair("login", user)
-            .append_pair("database", database)
-            .append_pair("md5response", std::str::from_utf8(md5_response)?)
-            .append_pair("salt", &hex::encode(salt));
+    pub fn check_auth(&self, user: &str, md5_response: &[u8], salt: &[u8; 4]) -> Result<()> {
+        // passwords for both is "mypass"
+        let auth_map: HashMap<_, &str> = vec![
+            ("stas@zenith", "716ee6e1c4a9364d66285452c47402b1"),
+            ("stas2@zenith", "3996f75df64c16a8bfaf01301b61d582"),
+        ]
+        .into_iter()
+        .collect();
 
-        println!("cplane request: {}", url.as_str());
+        let stored_hash = auth_map
+            .get(&user)
+            .ok_or_else(|| anyhow::Error::msg("user not found"))?;
+        let salted_stored_hash = format!(
+            "md5{:x}",
+            md5::compute([stored_hash.as_bytes(), salt].concat())
+        );
 
-        let resp = reqwest::blocking::get(url)?;
+        let received_hash = std::str::from_utf8(&md5_response)?;
 
-        if resp.status().is_success() {
-            let conn_info: DatabaseInfo = serde_json::from_str(resp.text()?.as_str())?;
-            println!("got conn info: #{:?}", conn_info);
-            Ok(conn_info)
+        println!(
+            "auth: {} rh={} sh={} ssh={} {:?}",
+            user, received_hash, stored_hash, salted_stored_hash, salt
+        );
+
+        if received_hash == salted_stored_hash {
+            Ok(())
         } else {
             bail!("Auth failed")
         }
     }
+
+    pub fn get_database_uri(&self, _user: &str, _database: &str) -> Result<DatabaseInfo> {
+        Ok(DatabaseInfo {
+            host: "127.0.0.1".parse()?,
+            port: 5432,
+            dbname: "stas".to_string(),
+            user: "stas".to_string(),
+            password: "mypass".to_string(),
+        })
+    }
+
+    // pub fn create_database(&self, _user: &String, _database: &String) -> Result<DatabaseInfo> {
+    //     Ok(DatabaseInfo {
+    //         host: "127.0.0.1".parse()?,
+    //         port: 5432,
+    //         dbname: "stas".to_string(),
+    //         user: "stas".to_string(),
+    //         password: "mypass".to_string(),
+    //     })
+    // }
 }

proxy/src/main.rs

@@ -8,15 +8,13 @@
 use std::{
     collections::HashMap,
     net::{SocketAddr, TcpListener},
-    sync::{mpsc, Arc, Mutex},
+    sync::{mpsc, Mutex},
     thread,
 };
 
-use anyhow::{anyhow, bail, ensure, Context};
-use clap::{App, Arg, ArgMatches};
+use clap::{App, Arg};
 
 use cplane_api::DatabaseInfo;
-use rustls::{internal::pemfile, NoClientAuth, ProtocolVersion, ServerConfig};
 
 mod cplane_api;
 mod mgmt;
@@ -34,9 +32,7 @@ pub struct ProxyConf {
     pub redirect_uri: String,
 
     /// control plane address where we would check auth.
-    pub auth_endpoint: String,
-
-    pub ssl_config: Option<Arc<ServerConfig>>,
+    pub cplane_address: SocketAddr,
 }
 
 pub struct ProxyState {
@@ -44,37 +40,6 @@ pub struct ProxyState {
     pub waiters: Mutex<HashMap<String, mpsc::Sender<anyhow::Result<DatabaseInfo>>>>,
 }
 
-fn configure_ssl(arg_matches: &ArgMatches) -> anyhow::Result<Option<Arc<ServerConfig>>> {
-    let (key_path, cert_path) = match (
-        arg_matches.value_of("ssl-key"),
-        arg_matches.value_of("ssl-cert"),
-    ) {
-        (Some(key_path), Some(cert_path)) => (key_path, cert_path),
-        (None, None) => return Ok(None),
-        _ => bail!("either both or neither ssl-key and ssl-cert must be specified"),
-    };
-
-    let key = {
-        let key_bytes = std::fs::read(key_path).context("SSL key file")?;
-        let mut keys = pemfile::pkcs8_private_keys(&mut &key_bytes[..])
-            .map_err(|_| anyhow!("couldn't read TLS keys"))?;
-        ensure!(keys.len() == 1, "keys.len() = {} (should be 1)", keys.len());
-        keys.pop().unwrap()
-    };
-
-    let cert_chain = {
-        let cert_chain_bytes = std::fs::read(cert_path).context("SSL cert file")?;
-        pemfile::certs(&mut &cert_chain_bytes[..])
-            .map_err(|_| anyhow!("couldn't read TLS certificates"))?
-    };
-
-    let mut config = ServerConfig::new(NoClientAuth::new());
-    config.set_single_cert(cert_chain, key)?;
-    config.versions = vec![ProtocolVersion::TLSv1_3];
-
-    Ok(Some(Arc::new(config)))
-}
-
 fn main() -> anyhow::Result<()> {
     let arg_matches = App::new("Zenith proxy/router")
         .arg(
@@ -99,29 +64,7 @@ fn main() -> anyhow::Result<()> {
                 .long("uri")
                 .takes_value(true)
                 .help("redirect unauthenticated users to given uri")
-                .default_value("http://127.0.0.1:3000/psql_session/"),
-        )
-        .arg(
-            Arg::with_name("auth-endpoint")
-                .short("a")
-                .long("auth-endpoint")
-                .takes_value(true)
-                .help("redirect unauthenticated users to given uri")
-                .default_value("http://127.0.0.1:3000/authenticate_proxy_request/"),
-        )
-        .arg(
-            Arg::with_name("ssl-key")
-                .short("k")
-                .long("ssl-key")
-                .takes_value(true)
-                .help("path to SSL key for client postgres connections"),
-        )
-        .arg(
-            Arg::with_name("ssl-cert")
-                .short("c")
-                .long("ssl-cert")
-                .takes_value(true)
-                .help("path to SSL cert for client postgres connections"),
+                .default_value("http://localhost:3000/psql_session/"),
         )
         .get_matches();
 
@@ -129,8 +72,7 @@ fn main() -> anyhow::Result<()> {
         proxy_address: arg_matches.value_of("proxy").unwrap().parse()?,
         mgmt_address: arg_matches.value_of("mgmt").unwrap().parse()?,
         redirect_uri: arg_matches.value_of("uri").unwrap().parse()?,
-        auth_endpoint: arg_matches.value_of("auth-endpoint").unwrap().parse()?,
-        ssl_config: configure_ssl(&arg_matches)?,
+        cplane_address: "127.0.0.1:3000".parse()?,
     };
     let state = ProxyState {
         conf,
@@ -150,10 +92,10 @@ fn main() -> anyhow::Result<()> {
         // for each connection.
         thread::Builder::new()
             .name("Proxy thread".into())
-            .spawn(move || proxy::thread_main(state, pageserver_listener))?,
+            .spawn(move || proxy::thread_main(&state, pageserver_listener))?,
         thread::Builder::new()
             .name("Mgmt thread".into())
-            .spawn(move || mgmt::thread_main(state, mgmt_listener))?,
+            .spawn(move || mgmt::thread_main(&state, mgmt_listener))?,
     ];
 
     for t in threads.into_iter() {

proxy/src/mgmt.rs

@@ -5,9 +5,9 @@ use std::{
 
 use anyhow::bail;
 use bytes::Bytes;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use zenith_utils::{
-    postgres_backend::{self, query_from_cstring, AuthType, PostgresBackend},
+    postgres_backend::{self, query_from_cstring, PostgresBackend},
     pq_proto::{BeMessage, SINGLE_COL_ROWDESC},
 };
 
@@ -34,7 +34,7 @@ pub fn thread_main(state: &'static ProxyState, listener: TcpListener) -> anyhow:
 
 pub fn mgmt_conn_main(state: &'static ProxyState, socket: TcpStream) -> anyhow::Result<()> {
     let mut conn_handler = MgmtHandler { state };
-    let pgbackend = PostgresBackend::new(socket, AuthType::Trust, None, true)?;
+    let mut pgbackend = PostgresBackend::new(socket, postgres_backend::AuthType::Trust)?;
     pgbackend.run(&mut conn_handler)
 }
 
@@ -49,7 +49,7 @@ struct MgmtHandler {
 //             "host": "127.0.0.1",
 //             "port": 5432,
 //             "dbname": "stas",
-//             "user": "stas",
+//             "user": "stas"
 //             "password": "mypass"
 //         }
 //     }
@@ -60,16 +60,13 @@ struct MgmtHandler {
 //         "Failure": "oops"
 //     }
 // }
-//
-// // to test manually by sending a query to mgmt interface:
-// psql -h 127.0.0.1 -p 9999 -c '{"session_id":"4f10dde522e14739","result":{"Success":{"host":"127.0.0.1","port":5432,"dbname":"stas","user":"stas","password":"stas"}}}'
-#[derive(Deserialize)]
+#[derive(Serialize, Deserialize)]
 pub struct PsqlSessionResponse {
     session_id: String,
     result: PsqlSessionResult,
 }
 
-#[derive(Deserialize)]
+#[derive(Serialize, Deserialize)]
 pub enum PsqlSessionResult {
     Success(DatabaseInfo),
     Failure(String),
@@ -81,47 +78,34 @@ impl postgres_backend::Handler for MgmtHandler {
         pgb: &mut PostgresBackend,
         query_string: Bytes,
     ) -> anyhow::Result<()> {
-        let res = try_process_query(self, pgb, query_string);
-        // intercept and log error message
-        if res.is_err() {
-            println!("Mgmt query failed: #{:?}", res);
-        }
-        res
-    }
-}
+        let query_string = query_from_cstring(query_string);
 
-fn try_process_query(
-    mgmt: &mut MgmtHandler,
-    pgb: &mut PostgresBackend,
-    query_string: Bytes,
-) -> anyhow::Result<()> {
-    let query_string = query_from_cstring(query_string);
+        println!("Got mgmt query: '{}'", std::str::from_utf8(&query_string)?);
 
-    println!("Got mgmt query: '{}'", std::str::from_utf8(&query_string)?);
+        let resp: PsqlSessionResponse = serde_json::from_slice(&query_string)?;
 
-    let resp: PsqlSessionResponse = serde_json::from_slice(&query_string)?;
+        let waiters = self.state.waiters.lock().unwrap();
 
-    let waiters = mgmt.state.waiters.lock().unwrap();
+        let sender = waiters
+            .get(&resp.session_id)
+            .ok_or_else(|| anyhow::Error::msg("psql_session_id is not found"))?;
 
-    let sender = waiters
-        .get(&resp.session_id)
-        .ok_or_else(|| anyhow::Error::msg("psql_session_id is not found"))?;
+        match resp.result {
+            PsqlSessionResult::Success(db_info) => {
+                sender.send(Ok(db_info))?;
 
-    match resp.result {
-        PsqlSessionResult::Success(db_info) => {
-            sender.send(Ok(db_info))?;
+                pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
+                    .write_message_noflush(&BeMessage::DataRow(&[Some(b"ok")]))?
+                    .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
+                pgb.flush()?;
+                Ok(())
+            }
 
-            pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
-                .write_message_noflush(&BeMessage::DataRow(&[Some(b"ok")]))?
-                .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
-            pgb.flush()?;
-            Ok(())
-        }
+            PsqlSessionResult::Failure(message) => {
+                sender.send(Err(anyhow::Error::msg(message.clone())))?;
 
-        PsqlSessionResult::Failure(message) => {
-            sender.send(Err(anyhow::Error::msg(message.clone())))?;
-
-            bail!("psql session request failed: {}", message)
+                bail!("psql session request failed: {}", message)
+            }
         }
     }
 }

proxy/src/proxy.rs

@@ -6,12 +6,11 @@ use anyhow::bail;
 use tokio_postgres::NoTls;
 
 use rand::Rng;
-use std::io::Write;
-use std::{io, sync::mpsc::channel, thread};
-use zenith_utils::postgres_backend::Stream;
+use std::sync::mpsc::channel;
+use std::thread;
+use tokio::io::AsyncWriteExt;
 use zenith_utils::postgres_backend::{PostgresBackend, ProtoState};
 use zenith_utils::pq_proto::*;
-use zenith_utils::sock_split::{ReadStream, WriteStream};
 use zenith_utils::{postgres_backend, pq_proto::BeMessage};
 
 ///
@@ -57,15 +56,10 @@ pub fn proxy_conn_main(
 ) -> anyhow::Result<()> {
     let mut conn = ProxyConnection {
         state,
-        cplane: CPlaneApi::new(&state.conf.auth_endpoint),
+        cplane: CPlaneApi::new(&state.conf.cplane_address),
         user: "".into(),
         database: "".into(),
-        pgb: PostgresBackend::new(
-            socket,
-            postgres_backend::AuthType::MD5,
-            state.conf.ssl_config.clone(),
-            false,
-        )?,
+        pgb: PostgresBackend::new(socket, postgres_backend::AuthType::MD5)?,
         md5_salt: [0u8; 4],
         psql_session_id: "".into(),
     };
@@ -81,9 +75,17 @@ pub fn proxy_conn_main(
         conn.handle_new_user()?
     };
 
-    // XXX: move that inside handle_new_user/handle_existing_user to be able to
-    // report wrong connection error.
-    proxy_pass(conn.pgb, db_info)
+    // ok, proxy pass user connection to database_uri
+    let runtime = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+
+    let _ = runtime.block_on(proxy_pass(conn.pgb, db_info))?;
+
+    println!("proxy_conn_main done;");
+
+    Ok(())
 }
 
 impl ProxyConnection {
@@ -92,7 +94,6 @@ impl ProxyConnection {
     }
 
     fn handle_startup(&mut self) -> anyhow::Result<()> {
-        let mut encrypted = false;
         loop {
             let msg = self.pgb.read_message()?;
             println!("got message {:?}", msg);
@@ -101,29 +102,11 @@ impl ProxyConnection {
                     println!("got startup message {:?}", m);
 
                     match m.kind {
-                        StartupRequestCode::NegotiateGss => {
-                            self.pgb
-                                .write_message(&BeMessage::EncryptionResponse(false))?;
-                        }
-                        StartupRequestCode::NegotiateSsl => {
+                        StartupRequestCode::NegotiateGss | StartupRequestCode::NegotiateSsl => {
                             println!("SSL requested");
-                            if self.pgb.tls_config.is_some() {
-                                self.pgb
-                                    .write_message(&BeMessage::EncryptionResponse(true))?;
-                                self.pgb.start_tls()?;
-                                encrypted = true;
-                            } else {
-                                self.pgb
-                                    .write_message(&BeMessage::EncryptionResponse(false))?;
-                            }
+                            self.pgb.write_message(&BeMessage::Negotiate)?;
                         }
                         StartupRequestCode::Normal => {
-                            if self.state.conf.ssl_config.is_some() && !encrypted {
-                                self.pgb.write_message(&BeMessage::ErrorResponse(
-                                    "must connect with TLS".to_string(),
-                                ))?;
-                                bail!("client did not connect with TLS");
-                            }
                             self.user = m
                                 .params
                                 .get("user")
@@ -175,31 +158,21 @@ impl ProxyConnection {
                 .split_last()
                 .ok_or_else(|| anyhow::Error::msg("unexpected password message"))?;
 
-            match self.cplane.authenticate_proxy_request(
-                self.user.as_str(),
-                self.database.as_str(),
-                md5_response,
-                &self.md5_salt,
-            ) {
-                Err(e) => {
-                    self.pgb
-                        .write_message(&BeMessage::ErrorResponse(format!("{}", e)))?;
-
-                    bail!("auth failed: {}", e);
-                }
-                Ok(conn_info) => {
-                    self.pgb
-                        .write_message_noflush(&BeMessage::AuthenticationOk)?;
-                    self.pgb
-                        .write_message_noflush(&BeMessage::ParameterStatus)?;
-                    self.pgb.write_message(&BeMessage::ReadyForQuery)?;
-
-                    Ok(conn_info)
-                }
+            if let Err(e) = self.check_auth_md5(md5_response) {
+                self.pgb
+                    .write_message(&BeMessage::ErrorResponse(format!("{}", e)))?;
+                bail!("auth failed: {}", e);
+            } else {
+                self.pgb
+                    .write_message_noflush(&BeMessage::AuthenticationOk)?;
+                self.pgb
+                    .write_message_noflush(&BeMessage::ParameterStatus)?;
+                self.pgb.write_message(&BeMessage::ReadyForQuery)?;
             }
-        } else {
-            bail!("protocol violation");
         }
+
+        // ok, we are authorized
+        self.cplane.get_database_uri(&self.user, &self.database)
     }
 
     fn handle_new_user(&mut self) -> anyhow::Result<DatabaseInfo> {
@@ -209,14 +182,14 @@ impl ProxyConnection {
 
         let hello_message = format!("☀️  Welcome to Zenith!
 
-To proceed with database creation, open the following link:
+To proceed with database creation open following link:
 
-    {redirect_uri}{sess_id}
+    {}{}
 
-It needs to be done once and we will send you '.pgpass' file, which will allow you to access or create
+It needed to be done once and we will send you '.pgpass' file which will allow you to access or create
 databases without opening the browser.
 
-", redirect_uri = self.state.conf.redirect_uri, sess_id = self.psql_session_id);
+", self.state.conf.redirect_uri,self.psql_session_id);
 
         self.pgb
             .write_message_noflush(&BeMessage::AuthenticationOk)?;
@@ -245,56 +218,39 @@ databases without opening the browser.
 
         Ok(dbinfo)
     }
+
+    fn check_auth_md5(&self, md5_response: &[u8]) -> anyhow::Result<()> {
+        assert!(self.is_existing_user());
+        self.cplane
+            .check_auth(self.user.as_str(), md5_response, &self.md5_salt)
+    }
 }
 
-/// Create a TCP connection to a postgres database, authenticate with it, and receive the ReadyForQuery message
-async fn connect_to_db(db_info: DatabaseInfo) -> anyhow::Result<tokio::net::TcpStream> {
-    let mut socket = tokio::net::TcpStream::connect(db_info.socket_addr()?).await?;
+async fn proxy_pass(pgb: PostgresBackend, db_info: DatabaseInfo) -> anyhow::Result<()> {
+    let mut socket = tokio::net::TcpStream::connect(db_info.socket_addr()).await?;
     let config = db_info.conn_string().parse::<tokio_postgres::Config>()?;
     let _ = config.connect_raw(&mut socket, NoTls).await?;
-    Ok(socket)
-}
 
-/// Concurrently proxy both directions of the client and server connections
-fn proxy(
-    client_read: ReadStream,
-    client_write: WriteStream,
-    server_read: ReadStream,
-    server_write: WriteStream,
-) -> anyhow::Result<()> {
-    fn do_proxy(mut reader: ReadStream, mut writer: WriteStream) -> io::Result<()> {
-        std::io::copy(&mut reader, &mut writer)?;
-        writer.flush()?;
-        writer.shutdown(std::net::Shutdown::Both)
-    }
+    println!("Connected to pg, proxying");
 
-    let client_to_server_jh = thread::spawn(move || do_proxy(client_read, server_write));
+    let incoming_std = pgb.into_stream();
+    incoming_std.set_nonblocking(true)?;
+    let mut incoming_conn = tokio::net::TcpStream::from_std(incoming_std)?;
 
-    let res1 = do_proxy(server_read, client_write);
-    let res2 = client_to_server_jh.join().unwrap();
-    res1?;
-    res2?;
+    let (mut ri, mut wi) = incoming_conn.split();
+    let (mut ro, mut wo) = socket.split();
+
+    let client_to_server = async {
+        tokio::io::copy(&mut ri, &mut wo).await?;
+        wo.shutdown().await
+    };
+
+    let server_to_client = async {
+        tokio::io::copy(&mut ro, &mut wi).await?;
+        wi.shutdown().await
+    };
+
+    tokio::try_join!(client_to_server, server_to_client)?;
 
     Ok(())
 }
-
-/// Proxy a client connection to a postgres database
-fn proxy_pass(pgb: PostgresBackend, db_info: DatabaseInfo) -> anyhow::Result<()> {
-    let runtime = tokio::runtime::Builder::new_current_thread()
-        .enable_all()
-        .build()?;
-    let db_stream = runtime.block_on(connect_to_db(db_info))?;
-    let db_stream = db_stream.into_std()?;
-    db_stream.set_nonblocking(false)?;
-
-    let db_stream = zenith_utils::sock_split::BidiStream::from_tcp(db_stream);
-    let (db_read, db_write) = db_stream.split();
-
-    let stream = match pgb.into_stream() {
-        Stream::Bidirectional(bidi_stream) => bidi_stream,
-        _ => bail!("invalid stream"),
-    };
-
-    let (client_read, client_write) = stream.split();
-    proxy(client_read, client_write, db_read, db_write)
-}

run_clippy.sh

@@ -8,8 +8,4 @@
 # warnings and errors right in the editor.
 # In vscode, this setting is Rust-analyzer>Check On Save:Command
 
-
-# * `-A unknown_lints` – do not warn about unknown lint suppressions
-#                        that people with newer toolchains might use
-# * `-D warnings`      - fail on any warnings (`cargo` returns non-zero exit status)
-cargo clippy "${@:2}" --all-targets --all-features --all --tests -- -A unknown_lints -D warnings
+cargo clippy "${@:2}" -- -A clippy::new_without_default -A clippy::manual_range_contains -A clippy::comparison_chain

test_runner/Pipfile

@@ -7,10 +7,6 @@ name = "pypi"
 pytest = ">=6.0.0"
 psycopg2 = "*"
 typing-extensions = "*"
-pyjwt = {extras = ["crypto"], version = "*"}
-requests = "*"
-pytest-xdist = "*"
-asyncpg = "*"
 
 [dev-packages]
 yapf = "*"

test_runner/Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "3cdc048691824d0b93912b6b78a0aa01dc98f278212c1badb0cc2edbd2103c3a"
+            "sha256": "4c20c05c20c50cf7e8f78ab461ab23841125345e63e00e2efa7661c165b6b364"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -16,25 +16,6 @@
         ]
     },
     "default": {
-        "asyncpg": {
-            "hashes": [
-                "sha256:129d501f3d30616afd51eb8d3142ef51ba05374256bd5834cec3ef4956a9b317",
-                "sha256:29ef6ae0a617fc13cc2ac5dc8e9b367bb83cba220614b437af9b67766f4b6b20",
-                "sha256:41704c561d354bef01353835a7846e5606faabbeb846214dfcf666cf53319f18",
-                "sha256:556b0e92e2b75dc028b3c4bc9bd5162ddf0053b856437cf1f04c97f9c6837d03",
-                "sha256:8ff5073d4b654e34bd5eaadc01dc4d68b8a9609084d835acd364cd934190a08d",
-                "sha256:a458fc69051fbb67d995fdda46d75a012b5d6200f91e17d23d4751482640ed4c",
-                "sha256:a7095890c96ba36f9f668eb552bb020dddb44f8e73e932f8573efc613ee83843",
-                "sha256:a738f4807c853623d3f93f0fea11f61be6b0e5ca16ea8aeb42c2c7ee742aa853",
-                "sha256:c4fc0205fe4ddd5aeb3dfdc0f7bafd43411181e1f5650189608e5971cceacff1",
-                "sha256:dd2fa063c3344823487d9ddccb40802f02622ddf8bf8a6cc53885ee7a2c1c0c6",
-                "sha256:ddffcb85227bf39cd1bedd4603e0082b243cf3b14ced64dce506a15b05232b83",
-                "sha256:e36c6806883786b19551bb70a4882561f31135dc8105a59662e0376cf5b2cbc5",
-                "sha256:eed43abc6ccf1dc02e0d0efc06ce46a411362f3358847c6b0ec9a43426f91ece"
-            ],
-            "index": "pypi",
-            "version": "==0.24.0"
-        },
         "attrs": {
             "hashes": [
                 "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1",
@@ -43,110 +24,13 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==21.2.0"
         },
-        "certifi": {
+        "importlib-metadata": {
             "hashes": [
-                "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee",
-                "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"
+                "sha256:833b26fb89d5de469b24a390e9df088d4e52e4ba33b01dc5e0e4f41b81a16c00",
+                "sha256:b142cc1dd1342f31ff04bb7d022492b09920cb64fed867cd3ea6f80fe3ebd139"
             ],
-            "version": "==2021.5.30"
-        },
-        "cffi": {
-            "hashes": [
-                "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d",
-                "sha256:0c0591bee64e438883b0c92a7bed78f6290d40bf02e54c5bf0978eaf36061771",
-                "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872",
-                "sha256:22b9c3c320171c108e903d61a3723b51e37aaa8c81255b5e7ce102775bd01e2c",
-                "sha256:26bb2549b72708c833f5abe62b756176022a7b9a7f689b571e74c8478ead51dc",
-                "sha256:33791e8a2dc2953f28b8d8d300dde42dd929ac28f974c4b4c6272cb2955cb762",
-                "sha256:3c8d896becff2fa653dc4438b54a5a25a971d1f4110b32bd3068db3722c80202",
-                "sha256:4373612d59c404baeb7cbd788a18b2b2a8331abcc84c3ba40051fcd18b17a4d5",
-                "sha256:487d63e1454627c8e47dd230025780e91869cfba4c753a74fda196a1f6ad6548",
-                "sha256:48916e459c54c4a70e52745639f1db524542140433599e13911b2f329834276a",
-                "sha256:4922cd707b25e623b902c86188aca466d3620892db76c0bdd7b99a3d5e61d35f",
-                "sha256:55af55e32ae468e9946f741a5d51f9896da6b9bf0bbdd326843fec05c730eb20",
-                "sha256:57e555a9feb4a8460415f1aac331a2dc833b1115284f7ded7278b54afc5bd218",
-                "sha256:5d4b68e216fc65e9fe4f524c177b54964af043dde734807586cf5435af84045c",
-                "sha256:64fda793737bc4037521d4899be780534b9aea552eb673b9833b01f945904c2e",
-                "sha256:6d6169cb3c6c2ad50db5b868db6491a790300ade1ed5d1da29289d73bbe40b56",
-                "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224",
-                "sha256:80b06212075346b5546b0417b9f2bf467fea3bfe7352f781ffc05a8ab24ba14a",
-                "sha256:818014c754cd3dba7229c0f5884396264d51ffb87ec86e927ef0be140bfdb0d2",
-                "sha256:8eb687582ed7cd8c4bdbff3df6c0da443eb89c3c72e6e5dcdd9c81729712791a",
-                "sha256:99f27fefe34c37ba9875f224a8f36e31d744d8083e00f520f133cab79ad5e819",
-                "sha256:9f3e33c28cd39d1b655ed1ba7247133b6f7fc16fa16887b120c0c670e35ce346",
-                "sha256:a8661b2ce9694ca01c529bfa204dbb144b275a31685a075ce123f12331be790b",
-                "sha256:a9da7010cec5a12193d1af9872a00888f396aba3dc79186604a09ea3ee7c029e",
-                "sha256:aedb15f0a5a5949ecb129a82b72b19df97bbbca024081ed2ef88bd5c0a610534",
-                "sha256:b315d709717a99f4b27b59b021e6207c64620790ca3e0bde636a6c7f14618abb",
-                "sha256:ba6f2b3f452e150945d58f4badd92310449876c4c954836cfb1803bdd7b422f0",
-                "sha256:c33d18eb6e6bc36f09d793c0dc58b0211fccc6ae5149b808da4a62660678b156",
-                "sha256:c9a875ce9d7fe32887784274dd533c57909b7b1dcadcc128a2ac21331a9765dd",
-                "sha256:c9e005e9bd57bc987764c32a1bee4364c44fdc11a3cc20a40b93b444984f2b87",
-                "sha256:d2ad4d668a5c0645d281dcd17aff2be3212bc109b33814bbb15c4939f44181cc",
-                "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195",
-                "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33",
-                "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f",
-                "sha256:e9dc245e3ac69c92ee4c167fbdd7428ec1956d4e754223124991ef29eb57a09d",
-                "sha256:eb687a11f0a7a1839719edd80f41e459cc5366857ecbed383ff376c4e3cc6afd",
-                "sha256:eb9e2a346c5238a30a746893f23a9535e700f8192a68c07c0258e7ece6ff3728",
-                "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7",
-                "sha256:f0010c6f9d1a4011e429109fda55a225921e3206e7f62a0c22a35344bfd13cca",
-                "sha256:f0c5d1acbfca6ebdd6b1e3eded8d261affb6ddcf2186205518f1428b8569bb99",
-                "sha256:f10afb1004f102c7868ebfe91c28f4a712227fe4cb24974350ace1f90e1febbf",
-                "sha256:f174135f5609428cc6e1b9090f9268f5c8935fddb1b25ccb8255a2d50de6789e",
-                "sha256:f3ebe6e73c319340830a9b2825d32eb6d8475c1dac020b4f0aa774ee3b898d1c",
-                "sha256:f627688813d0a4140153ff532537fbe4afea5a3dffce1f9deb7f91f848a832b5",
-                "sha256:fd4305f86f53dfd8cd3522269ed7fc34856a8ee3709a5e28b2836b2db9d4cd69"
-            ],
-            "version": "==1.14.6"
-        },
-        "charset-normalizer": {
-            "hashes": [
-                "sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6",
-                "sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f"
-            ],
-            "markers": "python_version >= '3'",
-            "version": "==2.0.6"
-        },
-        "cryptography": {
-            "hashes": [
-                "sha256:0a7dcbcd3f1913f664aca35d47c1331fce738d44ec34b7be8b9d332151b0b01e",
-                "sha256:1eb7bb0df6f6f583dd8e054689def236255161ebbcf62b226454ab9ec663746b",
-                "sha256:21ca464b3a4b8d8e86ba0ee5045e103a1fcfac3b39319727bc0fc58c09c6aff7",
-                "sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085",
-                "sha256:3520667fda779eb788ea00080124875be18f2d8f0848ec00733c0ec3bb8219fc",
-                "sha256:3c4129fc3fdc0fa8e40861b5ac0c673315b3c902bbdc05fc176764815b43dd1d",
-                "sha256:3fa3a7ccf96e826affdf1a0a9432be74dc73423125c8f96a909e3835a5ef194a",
-                "sha256:5b0fbfae7ff7febdb74b574055c7466da334a5371f253732d7e2e7525d570498",
-                "sha256:695104a9223a7239d155d7627ad912953b540929ef97ae0c34c7b8bf30857e89",
-                "sha256:8695456444f277af73a4877db9fc979849cd3ee74c198d04fc0776ebc3db52b9",
-                "sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c",
-                "sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7",
-                "sha256:9965c46c674ba8cc572bc09a03f4c649292ee73e1b683adb1ce81e82e9a6a0fb",
-                "sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14",
-                "sha256:a305600e7a6b7b855cd798e00278161b681ad6e9b7eca94c721d5f588ab212af",
-                "sha256:cd65b60cfe004790c795cc35f272e41a3df4631e2fb6b35aa7ac6ef2859d554e",
-                "sha256:d2a6e5ef66503da51d2110edf6c403dc6b494cc0082f85db12f54e9c5d4c3ec5",
-                "sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06",
-                "sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7"
-            ],
-            "version": "==3.4.8"
-        },
-        "execnet": {
-            "hashes": [
-                "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5",
-                "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==1.9.0"
-        },
-        "idna": {
-            "hashes": [
-                "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a",
-                "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"
-            ],
-            "markers": "python_version >= '3'",
-            "version": "==3.2"
+            "markers": "python_version < '3.8'",
+            "version": "==4.5.0"
         },
         "iniconfig": {
             "hashes": [
@@ -157,34 +41,34 @@
         },
         "packaging": {
             "hashes": [
-                "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7",
-                "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"
+                "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5",
+                "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==21.0"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==20.9"
         },
         "pluggy": {
             "hashes": [
-                "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159",
-                "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"
+                "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
+                "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.0.0"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==0.13.1"
         },
         "psycopg2": {
             "hashes": [
-                "sha256:079d97fc22de90da1d370c90583659a9f9a6ee4007355f5825e5f1c70dffc1fa",
-                "sha256:2087013c159a73e09713294a44d0c8008204d06326006b7f652bef5ace66eebb",
-                "sha256:2c992196719fadda59f72d44603ee1a2fdcc67de097eea38d41c7ad9ad246e62",
-                "sha256:7640e1e4d72444ef012e275e7b53204d7fab341fb22bc76057ede22fe6860b25",
-                "sha256:7f91312f065df517187134cce8e395ab37f5b601a42446bdc0f0d51773621854",
-                "sha256:830c8e8dddab6b6716a4bf73a09910c7954a92f40cf1d1e702fb93c8a919cc56",
-                "sha256:89409d369f4882c47f7ea20c42c5046879ce22c1e4ea20ef3b00a4dfc0a7f188",
-                "sha256:bf35a25f1aaa8a3781195595577fcbb59934856ee46b4f252f56ad12b8043bcf",
-                "sha256:de5303a6f1d0a7a34b9d40e4d3bef684ccc44a49bbe3eb85e3c0bffb4a131b7c"
+                "sha256:03a485bf71498870e38b535c0e6e7162d6ac06a91487edddc3b959894d65f79c",
+                "sha256:22102cfeb904898254f287b1a77360bf66c636858e7476593acd5267e5c24ff9",
+                "sha256:8f4c1800e57ad128d20b2e91d222ca238fffd316cef65be781361cdf35e37979",
+                "sha256:b12073fdf2002e828e5921be2c39ff9c6eab361c5c0bd6c529619fc23677accc",
+                "sha256:b6f47af317af8110818d255e693cfa80b7f1e435285be09778db7b66efd95789",
+                "sha256:d549db98fc0e6db41a2aa0d65f7434c4308a9f64012adb209b9e489f26fe87c6",
+                "sha256:e44e39a46af7c30566b7667fb27e701e652ab0a51e05c263a01d3ff0e223b765",
+                "sha256:e84c80be7a238d3c9c099b71f6890eaa35fc881146232cce888a88ab1bfb431e",
+                "sha256:f3d42bd42302293767b84206d9a446abc67ed4a133e4fe04dad8952de06c2091"
             ],
             "index": "pypi",
-            "version": "==2.9.1"
+            "version": "==2.9"
         },
         "py": {
             "hashes": [
@@ -194,25 +78,6 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.10.0"
         },
-        "pycparser": {
-            "hashes": [
-                "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",
-                "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==2.20"
-        },
-        "pyjwt": {
-            "extras": [
-                "crypto"
-            ],
-            "hashes": [
-                "sha256:934d73fbba91b0483d3857d1aff50e96b2a892384ee2c17417ed3203f173fca1",
-                "sha256:fba44e7898bbca160a2b2b501f492824fc8382485d3a6f11ba5d0c1937ce6130"
-            ],
-            "index": "pypi",
-            "version": "==2.1.0"
-        },
         "pyparsing": {
             "hashes": [
                 "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
@@ -223,35 +88,11 @@
         },
         "pytest": {
             "hashes": [
-                "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89",
-                "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"
+                "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b",
+                "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890"
             ],
             "index": "pypi",
-            "version": "==6.2.5"
-        },
-        "pytest-forked": {
-            "hashes": [
-                "sha256:6aa9ac7e00ad1a539c41bec6d21011332de671e938c7637378ec9710204e37ca",
-                "sha256:dc4147784048e70ef5d437951728825a131b81714b398d5d52f17c7c144d8815"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==1.3.0"
-        },
-        "pytest-xdist": {
-            "hashes": [
-                "sha256:7b61ebb46997a0820a263553179d6d1e25a8c50d8a8620cd1aa1e20e3be99168",
-                "sha256:89b330316f7fc475f999c81b577c2b926c9569f3d397ae432c0c2e2496d61ff9"
-            ],
-            "index": "pypi",
-            "version": "==2.4.0"
-        },
-        "requests": {
-            "hashes": [
-                "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24",
-                "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"
-            ],
-            "index": "pypi",
-            "version": "==2.26.0"
+            "version": "==6.2.4"
         },
         "toml": {
             "hashes": [
@@ -263,20 +104,20 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e",
-                "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7",
-                "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"
+                "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497",
+                "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342",
+                "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"
             ],
             "index": "pypi",
-            "version": "==3.10.0.2"
+            "version": "==3.10.0.0"
         },
-        "urllib3": {
+        "zipp": {
             "hashes": [
-                "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4",
-                "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"
+                "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76",
+                "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
-            "version": "==1.26.6"
+            "markers": "python_version >= '3.6'",
+            "version": "==3.4.1"
         }
     },
     "develop": {
@@ -288,6 +129,14 @@
             "index": "pypi",
             "version": "==3.9.2"
         },
+        "importlib-metadata": {
+            "hashes": [
+                "sha256:833b26fb89d5de469b24a390e9df088d4e52e4ba33b01dc5e0e4f41b81a16c00",
+                "sha256:b142cc1dd1342f31ff04bb7d022492b09920cb64fed867cd3ea6f80fe3ebd139"
+            ],
+            "markers": "python_version < '3.8'",
+            "version": "==4.5.0"
+        },
         "mccabe": {
             "hashes": [
                 "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
@@ -297,32 +146,32 @@
         },
         "mypy": {
             "hashes": [
-                "sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9",
-                "sha256:0aadfb2d3935988ec3815952e44058a3100499f5be5b28c34ac9d79f002a4a9a",
-                "sha256:119bed3832d961f3a880787bf621634ba042cb8dc850a7429f643508eeac97b9",
-                "sha256:1a85e280d4d217150ce8cb1a6dddffd14e753a4e0c3cf90baabb32cefa41b59e",
-                "sha256:3c4b8ca36877fc75339253721f69603a9c7fdb5d4d5a95a1a1b899d8b86a4de2",
-                "sha256:3e382b29f8e0ccf19a2df2b29a167591245df90c0b5a2542249873b5c1d78212",
-                "sha256:42c266ced41b65ed40a282c575705325fa7991af370036d3f134518336636f5b",
-                "sha256:53fd2eb27a8ee2892614370896956af2ff61254c275aaee4c230ae771cadd885",
-                "sha256:704098302473cb31a218f1775a873b376b30b4c18229421e9e9dc8916fd16150",
-                "sha256:7df1ead20c81371ccd6091fa3e2878559b5c4d4caadaf1a484cf88d93ca06703",
-                "sha256:866c41f28cee548475f146aa4d39a51cf3b6a84246969f3759cb3e9c742fc072",
-                "sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457",
-                "sha256:adaeee09bfde366d2c13fe6093a7df5df83c9a2ba98638c7d76b010694db760e",
-                "sha256:b6fb13123aeef4a3abbcfd7e71773ff3ff1526a7d3dc538f3929a49b42be03f0",
-                "sha256:b94e4b785e304a04ea0828759172a15add27088520dc7e49ceade7834275bedb",
-                "sha256:c0df2d30ed496a08de5daed2a9ea807d07c21ae0ab23acf541ab88c24b26ab97",
-                "sha256:c6c2602dffb74867498f86e6129fd52a2770c48b7cd3ece77ada4fa38f94eba8",
-                "sha256:ceb6e0a6e27fb364fb3853389607cf7eb3a126ad335790fa1e14ed02fba50811",
-                "sha256:d9dd839eb0dc1bbe866a288ba3c1afc33a202015d2ad83b31e875b5905a079b6",
-                "sha256:e4dab234478e3bd3ce83bac4193b2ecd9cf94e720ddd95ce69840273bf44f6de",
-                "sha256:ec4e0cd079db280b6bdabdc807047ff3e199f334050db5cbb91ba3e959a67504",
-                "sha256:ecd2c3fe726758037234c93df7e98deb257fd15c24c9180dacf1ef829da5f921",
-                "sha256:ef565033fa5a958e62796867b1df10c40263ea9ded87164d67572834e57a174d"
+                "sha256:0190fb77e93ce971954c9e54ea61de2802065174e5e990c9d4c1d0f54fbeeca2",
+                "sha256:0756529da2dd4d53d26096b7969ce0a47997123261a5432b48cc6848a2cb0bd4",
+                "sha256:2f9fedc1f186697fda191e634ac1d02f03d4c260212ccb018fabbb6d4b03eee8",
+                "sha256:353aac2ce41ddeaf7599f1c73fed2b75750bef3b44b6ad12985a991bc002a0da",
+                "sha256:3f12705eabdd274b98f676e3e5a89f247ea86dc1af48a2d5a2b080abac4e1243",
+                "sha256:4efc67b9b3e2fddbe395700f91d5b8deb5980bfaaccb77b306310bd0b9e002eb",
+                "sha256:517e7528d1be7e187a5db7f0a3e479747307c1b897d9706b1c662014faba3116",
+                "sha256:68a098c104ae2b75e946b107ef69dd8398d54cb52ad57580dfb9fc78f7f997f0",
+                "sha256:746e0b0101b8efec34902810047f26a8c80e1efbb4fc554956d848c05ef85d76",
+                "sha256:8be7bbd091886bde9fcafed8dd089a766fa76eb223135fe5c9e9798f78023a20",
+                "sha256:9236c21194fde5df1b4d8ebc2ef2c1f2a5dc7f18bcbea54274937cae2e20a01c",
+                "sha256:9ef5355eaaf7a23ab157c21a44c614365238a7bdb3552ec3b80c393697d974e1",
+                "sha256:9f1d74eeb3f58c7bd3f3f92b8f63cb1678466a55e2c4612bf36909105d0724ab",
+                "sha256:a26d0e53e90815c765f91966442775cf03b8a7514a4e960de7b5320208b07269",
+                "sha256:ae94c31bb556ddb2310e4f913b706696ccbd43c62d3331cd3511caef466871d2",
+                "sha256:b5ba1f0d5f9087e03bf5958c28d421a03a4c1ad260bf81556195dffeccd979c4",
+                "sha256:b5dfcd22c6bab08dfeded8d5b44bdcb68c6f1ab261861e35c470b89074f78a70",
+                "sha256:cd01c599cf9f897b6b6c6b5d8b182557fb7d99326bcdf5d449a0fbbb4ccee4b9",
+                "sha256:e89880168c67cf4fde4506b80ee42f1537ad66ad366c101d388b3fd7d7ce2afd",
+                "sha256:ebe2bc9cb638475f5d39068d2dbe8ae1d605bb8d8d3ff281c695df1670ab3987",
+                "sha256:f89bfda7f0f66b789792ab64ce0978e4a991a0e4dd6197349d0767b0f1095b21",
+                "sha256:fc4d63da57ef0e8cd4ab45131f3fe5c286ce7dd7f032650d0fbc239c6190e167",
+                "sha256:fd634bc17b1e2d6ce716f0e43446d0d61cdadb1efcad5c56ca211c22b246ebc8"
             ],
             "index": "pypi",
-            "version": "==0.910"
+            "version": "==0.902"
         },
         "mypy-extensions": {
             "hashes": [
@@ -355,14 +204,50 @@
             "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==0.10.2"
         },
+        "typed-ast": {
+            "hashes": [
+                "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace",
+                "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff",
+                "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266",
+                "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528",
+                "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6",
+                "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808",
+                "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4",
+                "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363",
+                "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341",
+                "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04",
+                "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41",
+                "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e",
+                "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3",
+                "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899",
+                "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805",
+                "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c",
+                "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c",
+                "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39",
+                "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a",
+                "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3",
+                "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7",
+                "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f",
+                "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075",
+                "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0",
+                "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40",
+                "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428",
+                "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927",
+                "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3",
+                "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f",
+                "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"
+            ],
+            "markers": "python_version < '3.8'",
+            "version": "==1.4.3"
+        },
         "typing-extensions": {
             "hashes": [
-                "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e",
-                "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7",
-                "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"
+                "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497",
+                "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342",
+                "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"
             ],
             "index": "pypi",
-            "version": "==3.10.0.2"
+            "version": "==3.10.0.0"
         },
         "yapf": {
             "hashes": [
@@ -371,6 +256,14 @@
             ],
             "index": "pypi",
             "version": "==0.31.0"
+        },
+        "zipp": {
+            "hashes": [
+                "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76",
+                "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==3.4.1"
         }
     }
 }

test_runner/README.md

@@ -7,8 +7,7 @@ Prerequisites:
 - Dependencies: install them via `pipenv install`. Note that Debian/Ubuntu
   packages are stale, as it commonly happens, so manual installation is not
   recommended.
-  Run `pipenv shell` to activate the venv or use `pipenv run` to run a single
-  command in the venv, e.g. `pipenv run pytest`.
+  Run `pipenv shell` to activate the venv.
 - Zenith and Postgres binaries
     - See the root README.md for build directions
     - Tests can be run from the git tree; or see the environment variables

test_runner/batch_others/test_auth.py

@@ -1,79 +0,0 @@
-
-from contextlib import closing
-from typing import Iterator
-from uuid import uuid4
-import psycopg2
-from fixtures.zenith_fixtures import PortDistributor, Postgres, ZenithCli, ZenithPageserver, PgBin
-import pytest
-
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-
-def test_pageserver_auth(pageserver_auth_enabled: ZenithPageserver):
-    ps = pageserver_auth_enabled
-
-    tenant_token = ps.auth_keys.generate_tenant_token(ps.initial_tenant)
-    invalid_tenant_token = ps.auth_keys.generate_tenant_token(uuid4().hex)
-    management_token = ps.auth_keys.generate_management_token()
-
-    # this does not invoke auth check and only decodes jwt and checks it for validity
-    # check both tokens
-    ps.safe_psql("status", password=tenant_token)
-    ps.safe_psql("status", password=management_token)
-
-    # tenant can create branches
-    ps.safe_psql(f"branch_create {ps.initial_tenant} new1 main", password=tenant_token)
-    # console can create branches for tenant
-    ps.safe_psql(f"branch_create {ps.initial_tenant} new2 main", password=management_token)
-
-    # fail to create branch using token with different tenantid
-    with pytest.raises(psycopg2.DatabaseError, match='Tenant id mismatch. Permission denied'):
-        ps.safe_psql(f"branch_create {ps.initial_tenant} new2 main", password=invalid_tenant_token)
-
-    # create tenant using management token
-    ps.safe_psql(f"tenant_create {uuid4().hex}", password=management_token)
-
-    # fail to create tenant using tenant token
-    with pytest.raises(psycopg2.DatabaseError, match='Attempt to access management api with tenant scope. Permission denied'):
-        ps.safe_psql(f"tenant_create {uuid4().hex}", password=tenant_token)
-
-
-@pytest.mark.parametrize('with_wal_acceptors', [False, True])
-def test_compute_auth_to_pageserver(
-    zenith_cli: ZenithCli,
-    wa_factory,
-    pageserver_auth_enabled: ZenithPageserver,
-    repo_dir: str,
-    with_wal_acceptors: bool,
-    pg_bin: PgBin,
-    port_distributor: PortDistributor,
-):
-    ps = pageserver_auth_enabled
-    # since we are in progress of refactoring protocols between compute safekeeper and page server
-    # use hardcoded management token in safekeeper
-    management_token = ps.auth_keys.generate_management_token()
-
-    branch = f"test_compute_auth_to_pageserver{with_wal_acceptors}"
-    zenith_cli.run(["branch", branch, "empty"])
-    if with_wal_acceptors:
-        wa_factory.start_n_new(3, management_token)
-
-    with Postgres(
-        zenith_cli=zenith_cli,
-        repo_dir=repo_dir,
-        pg_bin=pg_bin,
-        tenant_id=ps.initial_tenant,
-        port=port_distributor.get_port(),
-    ).create_start(
-        branch,
-        wal_acceptors=wa_factory.get_connstrs() if with_wal_acceptors else None,
-    ) as pg:
-        with closing(pg.connect()) as conn:
-            with conn.cursor() as cur:
-                # we rely upon autocommit after each statement
-                # as waiting for acceptors happens there
-                cur.execute('CREATE TABLE t(key int primary key, value text)')
-                cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
-                cur.execute('SELECT sum(key) FROM t')
-                assert cur.fetchone() == (5000050000, )

test_runner/batch_others/test_branch_behind.py

@@ -1,19 +1,15 @@
-import subprocess
-from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
-from fixtures.log_helper import log
-
 pytest_plugins = ("fixtures.zenith_fixtures")
 
 
 #
 # Create a couple of branches off the main branch, at a historical point in time.
 #
-def test_branch_behind(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+def test_branch_behind(zenith_cli, pageserver, postgres, pg_bin):
     # Branch at the point where only 100 rows were inserted
     zenith_cli.run(["branch", "test_branch_behind", "empty"])
 
     pgmain = postgres.create_start('test_branch_behind')
-    log.info("postgres is running on 'test_branch_behind' branch")
+    print("postgres is running on 'test_branch_behind' branch")
 
     main_pg_conn = pgmain.connect()
     main_cur = main_pg_conn.cursor()
@@ -27,17 +23,17 @@ def test_branch_behind(zenith_cli, pageserver: ZenithPageserver, postgres: Postg
     ''')
     main_cur.execute('SELECT pg_current_wal_insert_lsn()')
     lsn_a = main_cur.fetchone()[0]
-    log.info(f'LSN after 100 rows: {lsn_a}')
+    print('LSN after 100 rows: ' + lsn_a)
 
     # Insert some more rows. (This generates enough WAL to fill a few segments.)
     main_cur.execute('''
         INSERT INTO foo
             SELECT 'long string to consume some space' || g
-            FROM generate_series(1, 200000) g
+            FROM generate_series(1, 100000) g
     ''')
     main_cur.execute('SELECT pg_current_wal_insert_lsn()')
     lsn_b = main_cur.fetchone()[0]
-    log.info(f'LSN after 200100 rows: {lsn_b}')
+    print('LSN after 100100 rows: ' + lsn_b)
 
     # Branch at the point where only 100 rows were inserted
     zenith_cli.run(["branch", "test_branch_behind_hundred", "test_branch_behind@" + lsn_a])
@@ -46,15 +42,15 @@ def test_branch_behind(zenith_cli, pageserver: ZenithPageserver, postgres: Postg
     main_cur.execute('''
         INSERT INTO foo
             SELECT 'long string to consume some space' || g
-            FROM generate_series(1, 200000) g
+            FROM generate_series(1, 100000) g
     ''')
     main_cur.execute('SELECT pg_current_wal_insert_lsn()')
 
     main_cur.execute('SELECT pg_current_wal_insert_lsn()')
     lsn_c = main_cur.fetchone()[0]
-    log.info(f'LSN after 400100 rows: {lsn_c}')
+    print('LSN after 200100 rows: ' + lsn_c)
 
-    # Branch at the point where only 200100 rows were inserted
+    # Branch at the point where only 200 rows were inserted
     zenith_cli.run(["branch", "test_branch_behind_more", "test_branch_behind@" + lsn_b])
 
     pg_hundred = postgres.create_start("test_branch_behind_hundred")
@@ -70,23 +66,8 @@ def test_branch_behind(zenith_cli, pageserver: ZenithPageserver, postgres: Postg
     more_pg_conn = pg_more.connect()
     more_cur = more_pg_conn.cursor()
     more_cur.execute('SELECT count(*) FROM foo')
-    assert more_cur.fetchone() == (200100, )
+    assert more_cur.fetchone() == (100100, )
 
     # All the rows are visible on the main branch
     main_cur.execute('SELECT count(*) FROM foo')
-    assert main_cur.fetchone() == (400100, )
-
-    # Check bad lsn's for branching
-
-    # branch at segment boundary
-    zenith_cli.run(["branch", "test_branch_segment_boundary", "test_branch_behind@0/3000000"])
-    pg = postgres.create_start("test_branch_segment_boundary")
-    cur = pg.connect().cursor()
-    cur.execute('SELECT 1')
-    assert cur.fetchone() == (1, )
-
-    # branch at pre-initdb lsn
-    try:
-        zenith_cli.run(["branch", "test_branch_preinitdb", "test_branch_behind@0/42"])
-    except subprocess.CalledProcessError:
-        log.info("Branch creation with pre-initdb LSN failed (as expected)")
+    assert main_cur.fetchone() == (200100, )

test_runner/batch_others/test_bulk_insert.py

@@ -0,0 +1,25 @@
+from contextlib import closing
+import psycopg2.extras
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+#
+# Test insertion of larg number of records
+#
+# This test is pretty tightly coupled with the current implementation of page version storage
+# and garbage collection in object_repository.rs.
+#
+def test_bulk_insert(zenith_cli, pageserver, postgres, pg_bin):
+    zenith_cli.run(["branch", "test_bulk_insert", "empty"])
+    pg = postgres.create_start('test_bulk_insert')
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+          cur.execute("create table t(c1 bigint, c2 bigint, c3 bigint, c4 bigint, c5 bigint)")
+          cur.execute("create index on t(c1)")
+          cur.execute("create index on t(c2)")
+          cur.execute("create index on t(c3)")
+          cur.execute("create index on t(c4)")
+          cur.execute("create index on t(c5)")
+          cur.execute("insert into t values (generate_series(1,1000000),generate_series(1,1000000),generate_series(1,1000000),generate_series(1,1000000),generate_series(1,1000000))")
+          cur.execute("insert into t values (generate_series(1,1000000),random()*1000000,random()*1000000,random()*1000000,random()*1000000)")

test_runner/batch_others/test_clog_truncate.py

@@ -1,73 +0,0 @@
-import time
-import os
-
-from contextlib import closing
-
-from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
-from fixtures.log_helper import log
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-
-#
-# Test compute node start after clog truncation
-#
-def test_clog_truncate(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
-    # Create a branch for us
-    zenith_cli.run(["branch", "test_clog_truncate", "empty"])
-
-    # set agressive autovacuum to make sure that truncation will happen
-    config = [
-        'autovacuum_max_workers=10', 'autovacuum_vacuum_threshold=0',
-        'autovacuum_vacuum_insert_threshold=0', 'autovacuum_vacuum_cost_delay=0',
-        'autovacuum_vacuum_cost_limit=10000', 'autovacuum_naptime =1s',
-        'autovacuum_freeze_max_age=100000'
-    ]
-
-    pg = postgres.create_start('test_clog_truncate', config_lines=config)
-    log.info('postgres is running on test_clog_truncate branch')
-
-    # Install extension containing function needed for test
-    pg.safe_psql('CREATE EXTENSION zenith_test_utils')
-
-    # Consume many xids to advance clog
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            cur.execute('select test_consume_xids(1000*1000*10);')
-            log.info('xids consumed')
-
-            # call a checkpoint to trigger TruncateSubtrans
-            cur.execute('CHECKPOINT;')
-
-            # ensure WAL flush
-            cur.execute('select txid_current()')
-            log.info(cur.fetchone())
-
-    # wait for autovacuum to truncate the pg_xact
-    # XXX Is it worth to add a timeout here?
-    pg_xact_0000_path = os.path.join(pg.pg_xact_dir_path(), '0000')
-    log.info(f"pg_xact_0000_path = {pg_xact_0000_path}")
-
-    while os.path.isfile(pg_xact_0000_path):
-        log.info(f"file exists. wait for truncation. " "pg_xact_0000_path = {pg_xact_0000_path}")
-        time.sleep(5)
-
-    # checkpoint to advance latest lsn
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            cur.execute('CHECKPOINT;')
-            cur.execute('select pg_current_wal_insert_lsn()')
-            lsn_after_truncation = cur.fetchone()[0]
-
-    # create new branch after clog truncation and start a compute node on it
-    log.info(f'create branch at lsn_after_truncation {lsn_after_truncation}')
-    zenith_cli.run(
-        ["branch", "test_clog_truncate_new", "test_clog_truncate@" + lsn_after_truncation])
-
-    pg2 = postgres.create_start('test_clog_truncate_new')
-    log.info('postgres is running on test_clog_truncate_new branch')
-
-    # check that new node doesn't contain truncated segment
-    pg_xact_0000_path_new = os.path.join(pg2.pg_xact_dir_path(), '0000')
-    log.info(f"pg_xact_0000_path_new = {pg_xact_0000_path_new}")
-    assert os.path.isfile(pg_xact_0000_path_new) is False

test_runner/batch_others/test_config.py

@@ -1,21 +1,18 @@
 from contextlib import closing
 
-from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
-from fixtures.log_helper import log
-
 pytest_plugins = ("fixtures.zenith_fixtures")
 
 
 #
 # Test starting Postgres with custom options
 #
-def test_config(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+def test_config(zenith_cli, pageserver, postgres, pg_bin):
     # Create a branch for us
     zenith_cli.run(["branch", "test_config", "empty"])
 
     # change config
     pg = postgres.create_start('test_config', config_lines=['log_min_messages=debug1'])
-    log.info('postgres is running on test_config branch')
+    print('postgres is running on test_config branch')
 
     with closing(pg.connect()) as conn:
         with conn.cursor() as cur:

test_runner/batch_others/test_createdb.py

@@ -0,0 +1,32 @@
+from contextlib import closing
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+#
+# Test CREATE DATABASE when there have been relmapper changes
+#
+def test_createdb(zenith_cli, pageserver, postgres, pg_bin):
+    zenith_cli.run(["branch", "test_createdb", "empty"])
+
+    pg = postgres.create_start('test_createdb')
+    print("postgres is running on 'test_createdb' branch")
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            # Cause a 'relmapper' change in the original branch
+            cur.execute('VACUUM FULL pg_class')
+
+            cur.execute('CREATE DATABASE foodb')
+
+            cur.execute('SELECT pg_current_wal_insert_lsn()')
+            lsn = cur.fetchone()[0]
+
+    # Create a branch
+    zenith_cli.run(["branch", "test_createdb2", "test_createdb@" + lsn])
+
+    pg2 = postgres.create_start('test_createdb2')
+
+    # Test that you can connect to the new database on both branches
+    for db in (pg, pg2):
+        db.connect(dbname='foodb').close()

test_runner/batch_others/test_createdropdb.py

@@ -1,103 +0,0 @@
-import os
-import pathlib
-
-from contextlib import closing
-from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory, ZenithCli, check_restored_datadir_content
-from fixtures.log_helper import log
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-
-#
-# Test CREATE DATABASE when there have been relmapper changes
-#
-def test_createdb(
-    zenith_cli: ZenithCli,
-    pageserver: ZenithPageserver,
-    postgres: PostgresFactory,
-    pg_bin,
-):
-    zenith_cli.run(["branch", "test_createdb", "empty"])
-
-    pg = postgres.create_start('test_createdb')
-    log.info("postgres is running on 'test_createdb' branch")
-
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            # Cause a 'relmapper' change in the original branch
-            cur.execute('VACUUM FULL pg_class')
-
-            cur.execute('CREATE DATABASE foodb')
-
-            cur.execute('SELECT pg_current_wal_insert_lsn()')
-            lsn = cur.fetchone()[0]
-
-    # Create a branch
-    zenith_cli.run(["branch", "test_createdb2", "test_createdb@" + lsn])
-
-    pg2 = postgres.create_start('test_createdb2')
-
-    # Test that you can connect to the new database on both branches
-    for db in (pg, pg2):
-        db.connect(dbname='foodb').close()
-
-#
-# Test DROP DATABASE
-#
-def test_dropdb(
-    zenith_cli: ZenithCli,
-    pageserver: ZenithPageserver,
-    postgres: PostgresFactory,
-    pg_bin,
-    test_output_dir
-):
-    zenith_cli.run(["branch", "test_dropdb", "empty"])
-
-    pg = postgres.create_start('test_dropdb')
-    log.info("postgres is running on 'test_dropdb' branch")
-
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            cur.execute('CREATE DATABASE foodb')
-
-            cur.execute('SELECT pg_current_wal_insert_lsn()')
-            lsn_before_drop = cur.fetchone()[0]
-
-            cur.execute("SELECT oid FROM pg_database WHERE datname='foodb';")
-            dboid = cur.fetchone()[0]
-
-
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            cur.execute('DROP DATABASE foodb')
-
-            cur.execute('CHECKPOINT')
-
-            cur.execute('SELECT pg_current_wal_insert_lsn()')
-            lsn_after_drop = cur.fetchone()[0]
-
-
-    # Create two branches before and after database drop.
-    zenith_cli.run(["branch", "test_before_dropdb", "test_dropdb@" + lsn_before_drop])
-    pg_before = postgres.create_start('test_before_dropdb')
-
-    zenith_cli.run(["branch", "test_after_dropdb", "test_dropdb@" + lsn_after_drop])
-    pg_after = postgres.create_start('test_after_dropdb')
-
-    # Test that database exists on the branch before drop
-    pg_before.connect(dbname='foodb').close()
-
-    # Test that database subdir exists on the branch before drop
-    dbpath = pathlib.Path(pg_before.pgdata_dir) / 'base' / str(dboid)
-    log.info(dbpath)
-
-    assert os.path.isdir(dbpath) == True
-
-    # Test that database subdir doesn't exist on the branch after drop
-    dbpath = pathlib.Path(pg_after.pgdata_dir) / 'base' / str(dboid)
-    log.info(dbpath)
-
-    assert os.path.isdir(dbpath) == False
-
-    # Check that we restore the content of the datadir correctly
-    check_restored_datadir_content(zenith_cli, test_output_dir, pg, pageserver.service_port.pg)

test_runner/batch_others/test_createuser.py

@@ -1,19 +1,16 @@
 from contextlib import closing
 
-from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
-from fixtures.log_helper import log
-
 pytest_plugins = ("fixtures.zenith_fixtures")
 
 
 #
 # Test CREATE USER to check shared catalog restore
 #
-def test_createuser(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+def test_createuser(zenith_cli, pageserver, postgres, pg_bin):
     zenith_cli.run(["branch", "test_createuser", "empty"])
 
     pg = postgres.create_start('test_createuser')
-    log.info("postgres is running on 'test_createuser' branch")
+    print("postgres is running on 'test_createuser' branch")
 
     with closing(pg.connect()) as conn:
         with conn.cursor() as cur:

test_runner/batch_others/test_gc.py

@@ -0,0 +1,97 @@
+from contextlib import closing
+import psycopg2.extras
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+#
+# Test Garbage Collection of old page versions.
+#
+# This test is pretty tightly coupled with the current implementation of page version storage
+# and garbage collection in object_repository.rs.
+#
+def test_gc(zenith_cli, pageserver, postgres, pg_bin):
+    zenith_cli.run(["branch", "test_gc", "empty"])
+    pg = postgres.create_start('test_gc')
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            with closing(pageserver.connect()) as psconn:
+                with psconn.cursor(cursor_factory = psycopg2.extras.DictCursor) as pscur:
+
+                    # Get the timeline ID of our branch. We need it for the 'do_gc' command
+                    cur.execute("SHOW zenith.zenith_timeline")
+                    timeline = cur.fetchone()[0]
+
+                    # Create a test table
+                    cur.execute("CREATE TABLE foo(x integer)")
+
+                    # Run GC, to clear out any old page versions left behind in the catalogs by
+                    # the CREATE TABLE command. We want to have a clean slate with no garbage
+                    # before running the actual tests below, otherwise the counts won't match
+                    # what we expect.
+                    print("Running GC before test")
+                    pscur.execute(f"do_gc {timeline} 0")
+                    row = pscur.fetchone()
+                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
+                    # remember the number of relations
+                    n_relations = row['n_relations']
+                    assert n_relations > 0
+
+                    # Insert a row. The first insert will also create a metadata entry for the
+                    # relation, with size == 1 block. Hence, bump up the expected relation count.
+                    n_relations += 1;
+                    print("Inserting one row and running GC")
+                    cur.execute("INSERT INTO foo VALUES (1)")
+                    pscur.execute(f"do_gc {timeline} 0")
+                    row = pscur.fetchone()
+                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
+                    assert row['n_relations'] == n_relations
+                    assert row['dropped'] == 0
+                    assert row['truncated'] == 30
+                    assert row['deleted'] == 3
+
+                    # Insert two more rows and run GC.
+                    print("Inserting two more rows and running GC")
+                    cur.execute("INSERT INTO foo VALUES (2)")
+                    cur.execute("INSERT INTO foo VALUES (3)")
+
+                    pscur.execute(f"do_gc {timeline} 0")
+                    row = pscur.fetchone()
+                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
+                    assert row['n_relations'] == n_relations
+                    assert row['dropped'] == 0
+                    assert row['truncated'] == 30
+                    assert row['deleted'] == 2
+
+                    # Insert one more row. It creates one more page version, but doesn't affect the
+                    # relation size.
+                    print("Inserting one more row")
+                    cur.execute("INSERT INTO foo VALUES (3)")
+
+                    pscur.execute(f"do_gc {timeline} 0")
+                    row = pscur.fetchone()
+                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
+                    assert row['n_relations'] == n_relations
+                    assert row['dropped'] == 0
+                    assert row['truncated'] == 30
+                    assert row['deleted'] == 1
+
+                    # Run GC again, with no changes in the database. Should not remove anything.
+                    pscur.execute(f"do_gc {timeline} 0")
+                    row = pscur.fetchone()
+                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
+                    assert row['n_relations'] == n_relations
+                    assert row['dropped'] == 0
+                    assert row['truncated'] == 30
+                    assert row['deleted'] == 0
+
+                    #
+                    # Test DROP TABLE checks that relation data and metadata was deleted by GC from object storage
+                    #
+                    cur.execute("DROP TABLE foo")
+
+                    pscur.execute(f"do_gc {timeline} 0")
+                    row = pscur.fetchone()
+                    print("GC duration {elapsed} ms, relations: {n_relations}, dropped {dropped}, truncated: {truncated}, deleted: {deleted}".format_map(row))
+                    # Each relation fork is counted separately, hence 3.
+                    assert row['dropped'] == 3

test_runner/batch_others/test_multixact.py

@@ -1,6 +1,3 @@
-from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver, check_restored_datadir_content
-from fixtures.log_helper import log
-
 pytest_plugins = ("fixtures.zenith_fixtures")
 
 
@@ -10,13 +7,12 @@ pytest_plugins = ("fixtures.zenith_fixtures")
 # it only checks next_multixact_id field in restored pg_control,
 # since we don't have functions to check multixact internals.
 #
-def test_multixact(pageserver: ZenithPageserver, postgres: PostgresFactory,
-                    pg_bin, zenith_cli, base_dir, test_output_dir):
+def test_multixact(pageserver, postgres, pg_bin, zenith_cli, base_dir):
     # Create a branch for us
     zenith_cli.run(["branch", "test_multixact", "empty"])
     pg = postgres.create_start('test_multixact')
 
-    log.info("postgres is running on 'test_multixact' branch")
+    print("postgres is running on 'test_multixact' branch")
     pg_conn = pg.connect()
     cur = pg_conn.cursor()
 
@@ -56,7 +52,7 @@ def test_multixact(pageserver: ZenithPageserver, postgres: PostgresFactory,
     zenith_cli.run(["branch", "test_multixact_new", "test_multixact@" + lsn])
     pg_new = postgres.create_start('test_multixact_new')
 
-    log.info("postgres is running on 'test_multixact_new' branch")
+    print("postgres is running on 'test_multixact_new' branch")
     pg_new_conn = pg_new.connect()
     cur_new = pg_new_conn.cursor()
 
@@ -65,6 +61,3 @@ def test_multixact(pageserver: ZenithPageserver, postgres: PostgresFactory,
 
     # Check that we restored pg_controlfile correctly
     assert next_multixact_id_new == next_multixact_id
-
-    # Check that we restore the content of the datadir correctly
-    check_restored_datadir_content(zenith_cli, test_output_dir, pg_new, pageserver.service_port.pg)