pageserver - add naive layer IDs

.circleci/config.yml

@@ -4,16 +4,17 @@ executors:
   zenith-build-executor:
     resource_class: xlarge
     docker:
-      - image: cimg/rust:1.56.1
+      - image: cimg/rust:1.55.0
   zenith-python-executor:
     docker:
       - image: cimg/python:3.7.10  # Oldest available 3.7 with Ubuntu 20.04 (for GLIBC and Rust) at CirlceCI
 
 jobs:
-  check-codestyle-rust:
+  check-codestyle:
     executor: zenith-build-executor
     steps:
       - checkout
+
       - run:
           name: rustfmt
           when: always
@@ -80,8 +81,6 @@ jobs:
       build_type:
         type: enum
         enum: ["debug", "release"]
-    environment:
-      BUILD_TYPE: << parameters.build_type >>
     steps:
       - run:
           name: apt install dependencies
@@ -117,16 +116,15 @@ jobs:
       - run:
           name: Rust build << parameters.build_type >>
           command: |
-            if [[ $BUILD_TYPE == "debug" ]]; then
-              cov_prefix=(scripts/coverage "--profraw-prefix=$CIRCLE_JOB" --dir=/tmp/zenith/coverage run)
-              CARGO_FLAGS=
-            elif [[ $BUILD_TYPE == "release" ]]; then
-              cov_prefix=()
-              CARGO_FLAGS=--release
-            fi
-
             export CARGO_INCREMENTAL=0
-            "${cov_prefix[@]}" cargo build $CARGO_FLAGS --bins --tests
+            BUILD_TYPE="<< parameters.build_type >>"
+            if [[ $BUILD_TYPE == "debug" ]]; then
+              echo "Build in debug mode"
+              cargo build --bins --tests
+            elif [[ $BUILD_TYPE == "release" ]]; then
+              echo "Build in release mode"
+              cargo build --release --bins --tests
+            fi
 
       - save_cache:
           name: Save rust cache
@@ -140,96 +138,67 @@ jobs:
         # has to run separately from cargo fmt section
         # since needs to run with dependencies
       - run:
-          name: cargo clippy
+          name: clippy
           command: |
-            if [[ $BUILD_TYPE == "debug" ]]; then
-              cov_prefix=(scripts/coverage "--profraw-prefix=$CIRCLE_JOB" --dir=/tmp/zenith/coverage run)
-            elif [[ $BUILD_TYPE == "release" ]]; then
-              cov_prefix=()
-            fi
-
-            "${cov_prefix[@]}" ./run_clippy.sh
+            ./run_clippy.sh
 
         # Run rust unit tests
-      - run:
-          name: cargo test
-          command: |
-            if [[ $BUILD_TYPE == "debug" ]]; then
-              cov_prefix=(scripts/coverage "--profraw-prefix=$CIRCLE_JOB" --dir=/tmp/zenith/coverage run)
-            elif [[ $BUILD_TYPE == "release" ]]; then
-              cov_prefix=()
-            fi
-
-            "${cov_prefix[@]}" cargo test
+      - run: cargo test
 
         # Install the rust binaries, for use by test jobs
+        # `--locked` is required; otherwise, `cargo install` will ignore Cargo.lock.
+        # FIXME: this is a really silly way to install; maybe we should just output
+        # a tarball as an artifact? Or a .deb package?
       - run:
-          name: Install rust binaries
+          name: cargo install
           command: |
+            export CARGO_INCREMENTAL=0
+            BUILD_TYPE="<< parameters.build_type >>"
             if [[ $BUILD_TYPE == "debug" ]]; then
-              cov_prefix=(scripts/coverage "--profraw-prefix=$CIRCLE_JOB" --dir=/tmp/zenith/coverage run)
+              echo "Install debug mode"
+              CARGO_FLAGS="--debug"
             elif [[ $BUILD_TYPE == "release" ]]; then
-              cov_prefix=()
-            fi
-
-            binaries=$(
-              "${cov_prefix[@]}" cargo metadata --format-version=1 --no-deps |
-              jq -r '.packages[].targets[] | select(.kind | index("bin")) | .name'
-            )
-
-            test_exe_paths=$(
-              "${cov_prefix[@]}" cargo test --message-format=json --no-run |
-              jq -r '.executable | select(. != null)'
-            )
-
-            mkdir -p /tmp/zenith/bin
-            mkdir -p /tmp/zenith/test_bin
-            mkdir -p /tmp/zenith/etc
-
-            # Install target binaries
-            for bin in $binaries; do
-              SRC=target/$BUILD_TYPE/$bin
-              DST=/tmp/zenith/bin/$bin
-              cp $SRC $DST
-              echo $DST >> /tmp/zenith/etc/binaries.list
-            done
-
-            # Install test executables (for code coverage)
-            if [[ $BUILD_TYPE == "debug" ]]; then
-              for bin in $test_exe_paths; do
-                SRC=$bin
-                DST=/tmp/zenith/test_bin/$(basename $bin)
-                cp $SRC $DST
-                echo $DST >> /tmp/zenith/etc/binaries.list
-              done
+              echo "Install release mode"
+              # The default is release mode; there is no --release flag.
+              CARGO_FLAGS=""
             fi
+            cargo install $CARGO_FLAGS --locked --root /tmp/zenith --path pageserver
+            cargo install $CARGO_FLAGS --locked --root /tmp/zenith --path walkeeper
+            cargo install $CARGO_FLAGS --locked --root /tmp/zenith --path zenith
 
         # Install the postgres binaries, for use by test jobs
+        # FIXME: this is a silly way to do "install"; maybe just output a standard
+        # postgres package, whatever the favored form is (tarball? .deb package?)
+        # Note that pg_regress needs some build artifacts that probably aren't
+        # in the usual package...?
       - run:
-          name: Install postgres binaries
+          name: postgres install
           command: |
             cp -a tmp_install /tmp/zenith/pg_install
 
-        # Save the rust binaries and coverage data for other jobs in this workflow.
+        # Save the rust output binaries for other jobs in this workflow.
       - persist_to_workspace:
           root: /tmp/zenith
           paths:
             - "*"
 
-  check-codestyle-python:
+  check-python:
     executor: zenith-python-executor
     steps:
       - checkout
       - run:
           name: Install deps
+          working_directory: test_runner
           command: pipenv --python 3.7 install --dev
       - run:
           name: Run yapf to ensure code format
           when: always
+          working_directory: test_runner
           command: pipenv run yapf --recursive --diff .
       - run:
           name: Run mypy to check types
           when: always
+          working_directory: test_runner
           command: pipenv run mypy .
 
   run-pytest:
@@ -259,11 +228,6 @@ jobs:
       run_in_parallel:
         type: boolean
         default: true
-      save_perf_report:
-        type: boolean
-        default: false
-    environment:
-      BUILD_TYPE: << parameters.build_type >>
     steps:
       - attach_workspace:
           at: /tmp/zenith
@@ -274,25 +238,17 @@ jobs:
             - run: git submodule update --init --depth 1
       - run:
           name: Install deps
+          working_directory: test_runner
           command: pipenv --python 3.7 install
       - run:
           name: Run pytest
-          # pytest doesn't output test logs in real time, so CI job may fail with
-          # `Too long with no output` error, if a test is running for a long time.
-          # In that case, tests should have internal timeouts that are less than
-          # no_output_timeout, specified here.
-          no_output_timeout: 10m
+          working_directory: test_runner
           environment:
             - ZENITH_BIN: /tmp/zenith/bin
             - POSTGRES_DISTRIB_DIR: /tmp/zenith/pg_install
             - TEST_OUTPUT: /tmp/test_output
-            # this variable will be embedded in perf test report
-            # and is needed to distinguish different environments
-            - PLATFORM: zenith-local-ci
           command: |
-            PERF_REPORT_DIR="$(realpath test_runner/perf-report-local)"
-
-            TEST_SELECTION="test_runner/<< parameters.test_selection >>"
+            TEST_SELECTION="<< parameters.test_selection >>"
             EXTRA_PARAMS="<< parameters.extra_params >>"
             if [ -z "$TEST_SELECTION" ]; then
               echo "test_selection must be set"
@@ -300,22 +256,7 @@ jobs:
             fi
             if << parameters.run_in_parallel >>; then
               EXTRA_PARAMS="-n4 $EXTRA_PARAMS"
-            fi
-            if << parameters.save_perf_report >>; then
-              if [[ $CIRCLE_BRANCH == "main" ]]; then
-                mkdir -p "$PERF_REPORT_DIR"
-                EXTRA_PARAMS="--out-dir $PERF_REPORT_DIR $EXTRA_PARAMS"
-              fi
-            fi
-
-            export GITHUB_SHA=$CIRCLE_SHA1
-
-            if [[ $BUILD_TYPE == "debug" ]]; then
-              cov_prefix=(scripts/coverage "--profraw-prefix=$CIRCLE_JOB" --dir=/tmp/zenith/coverage run)
-            elif [[ $BUILD_TYPE == "release" ]]; then
-              cov_prefix=()
-            fi
-
+            fi;
             # Run the tests.
             #
             # The junit.xml file allows CircleCI to display more fine-grained test information
@@ -326,21 +267,7 @@ jobs:
             # -n4 uses four processes to run tests via pytest-xdist
             # -s is not used to prevent pytest from capturing output, because tests are running
             # in parallel and logs are mixed between different tests
-            "${cov_prefix[@]}" pipenv run pytest \
-              --junitxml=$TEST_OUTPUT/junit.xml \
-              --tb=short \
-              --verbose \
-              -m "not remote_cluster" \
-              -rA $TEST_SELECTION $EXTRA_PARAMS
-
-            if << parameters.save_perf_report >>; then
-              if [[ $CIRCLE_BRANCH == "main" ]]; then
-                # TODO: reuse scripts/git-upload
-                export REPORT_FROM="$PERF_REPORT_DIR"
-                export REPORT_TO=local
-                scripts/generate_and_push_perf_report.sh
-              fi
-            fi
+            pipenv run pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short --verbose -rA $TEST_SELECTION $EXTRA_PARAMS
       - run:
           # CircleCI artifacts are preserved one file at a time, so skipping
           # this step isn't a good idea. If you want to extract the
@@ -356,65 +283,6 @@ jobs:
       # The store_test_results step tells CircleCI where to find the junit.xml file.
       - store_test_results:
           path: /tmp/test_output
-      # Save coverage data (if any)
-      - persist_to_workspace:
-          root: /tmp/zenith
-          paths:
-            - "*"
-
-  coverage-report:
-    executor: zenith-build-executor
-    steps:
-      - attach_workspace:
-          at: /tmp/zenith
-      - checkout
-      - restore_cache:
-          name: Restore rust cache
-          keys:
-            # Require an exact match. While an out of date cache might speed up the build,
-            # there's no way to clean out old packages, so the cache grows every time something
-            # changes.
-            - v04-rust-cache-deps-debug-{{ checksum "Cargo.lock" }}
-      - run:
-          name: Install llvm-tools
-          command: |
-            # TODO: install a proper symbol demangler, e.g. rustfilt
-            # TODO: we should embed this into a docker image
-            rustup component add llvm-tools-preview
-      - run:
-          name: Build coverage report
-          command: |
-            COMMIT_URL=https://github.com/zenithdb/zenith/commit/$CIRCLE_SHA1
-
-            scripts/coverage \
-              --dir=/tmp/zenith/coverage report \
-              --input-objects=/tmp/zenith/etc/binaries.list \
-              --commit-url=$COMMIT_URL \
-              --format=github
-      - run:
-          name: Upload coverage report
-          command: |
-            LOCAL_REPO=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
-            REPORT_URL=https://zenithdb.github.io/zenith-coverage-data/$CIRCLE_SHA1
-            COMMIT_URL=https://github.com/zenithdb/zenith/commit/$CIRCLE_SHA1
-
-            scripts/git-upload \
-              --repo=https://$VIP_VAP_ACCESS_TOKEN@github.com/zenithdb/zenith-coverage-data.git \
-              --message="Add code coverage for $COMMIT_URL" \
-              copy /tmp/zenith/coverage/report $CIRCLE_SHA1 # COPY FROM TO_RELATIVE
-
-            # Add link to the coverage report to the commit
-            curl -f -X POST \
-            https://api.github.com/repos/$LOCAL_REPO/statuses/$CIRCLE_SHA1 \
-            -H "Accept: application/vnd.github.v3+json" \
-            --user "$CI_ACCESS_TOKEN" \
-            --data \
-              "{
-                \"state\": \"success\",
-                \"context\": \"zenith-coverage\",
-                \"description\": \"Coverage report is ready\",
-                \"target_url\": \"$REPORT_URL\"
-              }"
 
   # Build zenithdb/zenith:latest image and push it to Docker hub
   docker-image:
@@ -431,101 +299,7 @@ jobs:
           name: Build and push Docker image
           command: |
             echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
-            DOCKER_TAG=$(git log --oneline|wc -l)
-            docker build --build-arg GIT_VERSION=$CIRCLE_SHA1 -t zenithdb/zenith:latest . && docker push zenithdb/zenith:latest
-            docker tag zenithdb/zenith:latest zenithdb/zenith:${DOCKER_TAG} && docker push zenithdb/zenith:${DOCKER_TAG}
-
-  # Build zenithdb/compute-node:latest image and push it to Docker hub
-  docker-image-compute:
-    docker:
-      - image: cimg/base:2021.04
-    steps:
-      - checkout
-      - setup_remote_docker:
-          docker_layer_caching: true
-      # Build zenithdb/compute-tools:latest image and push it to Docker hub
-      # TODO: this should probably also use versioned tag, not just :latest.
-      # XXX: but should it? We build and use it only locally now.
-      - run:
-          name: Build and push compute-tools Docker image
-          command: |
-            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
-            docker build -t zenithdb/compute-tools:latest ./compute_tools/
-            docker push zenithdb/compute-tools:latest
-      - run:
-          name: Init postgres submodule
-          command: git submodule update --init --depth 1
-      - run:
-          name: Build and push compute-node Docker image
-          command: |
-            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
-            DOCKER_TAG=$(git log --oneline|wc -l)
-            docker build -t zenithdb/compute-node:latest vendor/postgres && docker push zenithdb/compute-node:latest
-            docker tag zenithdb/compute-node:latest zenithdb/compute-node:${DOCKER_TAG} && docker push zenithdb/compute-node:${DOCKER_TAG}
-
-  deploy-staging:
-    docker:
-      - image: cimg/python:3.10
-    steps:
-      - checkout
-      - setup_remote_docker
-      - run:
-          name: Get Zenith binaries
-          command: |
-            rm -rf zenith_install postgres_install.tar.gz zenith_install.tar.gz
-            mkdir zenith_install
-            DOCKER_TAG=$(git log --oneline|wc -l)
-            docker pull --quiet zenithdb/zenith:${DOCKER_TAG}
-            ID=$(docker create zenithdb/zenith:${DOCKER_TAG})
-            docker cp $ID:/data/postgres_install.tar.gz .
-            tar -xzf postgres_install.tar.gz -C zenith_install && rm postgres_install.tar.gz
-            docker cp $ID:/usr/local/bin/pageserver zenith_install/bin/
-            docker cp $ID:/usr/local/bin/safekeeper zenith_install/bin/
-            docker cp $ID:/usr/local/bin/proxy zenith_install/bin/
-            docker cp $ID:/usr/local/bin/postgres zenith_install/bin/
-            docker rm -v $ID
-            echo ${DOCKER_TAG} | tee zenith_install/.zenith_current_version
-            tar -czf zenith_install.tar.gz -C zenith_install .
-            ls -la zenith_install.tar.gz
-      - run:
-          name: Setup ansible
-          command: |
-            pip install --progress-bar off --user ansible boto3
-            ansible-galaxy collection install amazon.aws
-      - run:
-          name: Apply re-deploy playbook
-          environment:
-            ANSIBLE_HOST_KEY_CHECKING: false
-          command: |
-            echo "${STAGING_SSH_KEY}" | base64 --decode | ssh-add -
-            export AWS_REGION=${STAGING_AWS_REGION}
-            export AWS_ACCESS_KEY_ID=${STAGING_AWS_ACCESS_KEY_ID}
-            export AWS_SECRET_ACCESS_KEY=${STAGING_AWS_SECRET_ACCESS_KEY}
-            ansible-playbook .circleci/storage-redeploy.playbook.yml
-            rm -f zenith_install.tar.gz
-
-  deploy-staging-proxy:
-    docker:
-      - image: cimg/base:2021.04
-    environment:
-      KUBECONFIG: .kubeconfig
-    steps:
-      - checkout
-      - run:
-          name: Store kubeconfig file
-          command: |
-            echo "${STAGING_KUBECONFIG_DATA}" | base64 --decode > ${KUBECONFIG}
-            chmod 0600 ${KUBECONFIG}
-      - run:
-          name: Setup helm v3
-          command: |
-            curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
-            helm repo add zenithdb https://zenithdb.github.io/helm-charts
-      - run:
-          name: Re-deploy proxy
-          command: |
-            DOCKER_TAG=$(git log --oneline|wc -l)
-            helm upgrade zenith-proxy zenithdb/zenith-proxy --install -f .circleci/proxy.staging.yaml --set image.tag=${DOCKER_TAG} --wait
+            docker build -t zenithdb/zenith:latest . && docker push zenithdb/zenith:latest
 
   # Trigger a new remote CI job
   remote-ci-trigger:
@@ -571,60 +345,11 @@ jobs:
                 }
               }"
 
-  #
-  #
-  # compute-tools jobs
-  # TODO: unify with main build_and_test pipeline
-  #
-  #
-  compute-tools-test:
-    executor: zenith-build-executor
-    working_directory: ~/repo/compute_tools
-    steps:
-      - checkout:
-          path: ~/repo
-
-      - restore_cache:
-          name: Restore rust cache
-          keys:
-            # Require an exact match. While an out of date cache might speed up the build,
-            # there's no way to clean out old packages, so the cache grows every time something
-            # changes.
-            - v03-rust-cache-deps-debug-{{ checksum "Cargo.lock" }}
-
-      # Build the rust code, including test binaries
-      - run:
-          name: Rust build
-          environment:
-            CARGO_INCREMENTAL: 0
-          command: cargo build --bins --tests
-
-      - save_cache:
-          name: Save rust cache
-          key: v03-rust-cache-deps-debug-{{ checksum "Cargo.lock" }}
-          paths:
-            - ~/.cargo/registry
-            - ~/.cargo/git
-            - target
-
-      # Run Rust formatting checks
-      - run:
-          name: cargo fmt check
-          command: cargo fmt --all -- --check
-
-      # Run Rust linter (clippy)
-      - run:
-          name: cargo clippy check
-          command: cargo clippy --all --all-targets -- -Dwarnings -Drust-2018-idioms
-
-      # Run Rust integration and unittests
-      - run: cargo test
-
 workflows:
   build_and_test:
     jobs:
-      - check-codestyle-rust
-      - check-codestyle-python
+      - check-codestyle
+      - check-python
       - build-postgres:
           name: build-postgres-<< matrix.build_type >>
           matrix:
@@ -659,16 +384,8 @@ workflows:
           build_type: release
           test_selection: performance
           run_in_parallel: false
-          save_perf_report: true
           requires:
             - build-zenith-release
-      - coverage-report:
-          # Context passes credentials for gh api
-          context: CI_ACCESS_TOKEN
-          requires:
-            # TODO: consider adding more
-            - other-tests-debug
-      - compute-tools-test
       - docker-image:
           # Context gives an ability to login
           context: Docker Hub
@@ -680,36 +397,6 @@ workflows:
           requires:
             - pg_regress-tests-release
             - other-tests-release
-      - docker-image-compute:
-          # Context gives an ability to login
-          context: Docker Hub
-          # Build image only for commits to main
-          filters:
-            branches:
-              only:
-                - main
-          requires:
-            - pg_regress-tests-release
-            - other-tests-release
-            - compute-tools-test
-      - deploy-staging:
-          # Context gives an ability to login
-          context: Docker Hub
-          # deploy only for commits to main
-          filters:
-            branches:
-              only:
-                - main
-          requires:
-            - docker-image
-      - deploy-staging-proxy:
-          # deploy only for commits to main
-          filters:
-            branches:
-              only:
-                - main
-          requires:
-            - docker-image
       - remote-ci-trigger:
           # Context passes credentials for gh api
           context: CI_ACCESS_TOKEN

.circleci/proxy.staging.yaml

@@ -1,13 +0,0 @@
-# Helm chart values for zenith-proxy.
-# This is a YAML-formatted file.
-
-settings:
-  authEndpoint: "https://console.stage.zenith.tech/authenticate_proxy_request/"
-  uri: "https://console.stage.zenith.tech/psql_session/"
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: start.stage.zenith.tech

.circleci/storage-redeploy.playbook.yml

@@ -1,138 +0,0 @@
-- name: discover storage nodes
-  hosts: localhost
-  connection: local
-  gather_facts: False
-
-  tasks:
-
-    - name: discover safekeepers
-      no_log: true
-      ec2_instance_info:
-        filters:
-          "tag:zenith_env": "staging"
-          "tag:zenith_service": "safekeeper"
-      register: ec2_safekeepers
-
-    - name: discover pageservers
-      no_log: true
-      ec2_instance_info:
-        filters:
-          "tag:zenith_env": "staging"
-          "tag:zenith_service": "pageserver"
-      register: ec2_pageservers
-
-    - name: add safekeepers to host group
-      no_log: true
-      add_host:
-        name: safekeeper-{{ ansible_loop.index }}
-        ansible_host: "{{ item.public_ip_address }}"
-        groups:
-          - storage
-          - safekeepers
-      with_items: "{{ ec2_safekeepers.instances }}"
-      loop_control:
-        extended: yes
-
-    - name: add pageservers to host group
-      no_log: true
-      add_host:
-        name: pageserver-{{ ansible_loop.index }}
-        ansible_host: "{{ item.public_ip_address }}"
-        groups:
-          - storage
-          - pageservers
-      with_items: "{{ ec2_pageservers.instances }}"
-      loop_control:
-        extended: yes
-
-- name: Retrive versions
-  hosts: storage
-  gather_facts: False
-  remote_user: admin
-
-  tasks:
-
-    - name: Get current version of binaries
-      set_fact:
-        current_version: "{{lookup('file', '../zenith_install/.zenith_current_version') }}"
-
-    - name: Check that file with version exists on host
-      stat:
-        path: /usr/local/.zenith_current_version
-      register: version_file
-
-    - name: Try to get current version from the host
-      when: version_file.stat.exists
-      ansible.builtin.fetch:
-        src: /usr/local/.zenith_current_version
-        dest: .remote_version.{{ inventory_hostname }}
-        fail_on_missing: no
-        flat: yes
-
-    - name: Store remote version to variable
-      when: version_file.stat.exists
-      set_fact:
-        remote_version: "{{ lookup('file', '.remote_version.{{ inventory_hostname }}') }}"
-
-    - name: Store default value of remote version to variable in case when remote version file not found
-      when: not version_file.stat.exists
-      set_fact:
-        remote_version: "000"
-
-- name: Extract Zenith binaries
-  hosts: storage
-  gather_facts: False
-  remote_user: admin
-
-  tasks:
-
-    - name: Inform about version conflict
-      when: current_version <= remote_version
-      debug: msg="Current version {{ current_version }} LE than remote {{ remote_version }}"
-
-    - name: Extract Zenith binaries to /usr/local
-      when: current_version > remote_version
-      ansible.builtin.unarchive:
-        src: ../zenith_install.tar.gz
-        dest: /usr/local
-      become: true
-
-- name: Restart safekeepers
-  hosts: safekeepers
-  gather_facts: False
-  remote_user: admin
-
-  tasks:
-
-    - name: Inform about version conflict
-      when: current_version <= remote_version
-      debug: msg="Current version {{ current_version }} LE than remote {{ remote_version }}"
-
-    - name: Restart systemd service
-      when: current_version > remote_version
-      ansible.builtin.systemd:
-        daemon_reload: yes
-        name: safekeeper
-        enabled: yes
-        state: restarted
-      become: true
-
-- name: Restart pageservers
-  hosts: pageservers
-  gather_facts: False
-  remote_user: admin
-
-  tasks:
-
-    - name: Inform about version conflict
-      when: current_version <= remote_version
-      debug: msg="Current version {{ current_version }} LE than remote {{ remote_version }}"
-
-    - name: Restart systemd service
-      when: current_version > remote_version
-      ansible.builtin.systemd:
-        daemon_reload: yes
-        name: pageserver
-        enabled: yes
-        state: restarted
-      become: true

.github/workflows/benchmarking.yml

@@ -1,99 +0,0 @@
-name: benchmarking
-
-on:
-  # uncomment to run on push for debugging your PR
-  # push:
-  #   branches: [ mybranch ]
-  schedule:
-    # * is a special character in YAML so you have to quote this string
-    #          ┌───────────── minute (0 - 59)
-    #          │ ┌───────────── hour (0 - 23)
-    #          │ │ ┌───────────── day of the month (1 - 31)
-    #          │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
-    #          │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
-    - cron:  '36 7 * * *' # run once a day, timezone is utc
-
-  workflow_dispatch: # adds ability to run this manually
-
-jobs:
-  bench:
-    # this workflow runs on self hosteed runner
-    # it's environment is quite different from usual guthub runner
-    # probably the most important difference is that it doesnt start from clean workspace each time
-    # e g if you install system packages they are not cleaned up since you install them directly in host machine
-    # not a container or something
-    # See documentation for more info: https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners
-    runs-on: [self-hosted, zenith-benchmarker]
-
-    env:
-      PG_BIN: "/usr/pgsql-13/bin"
-
-    steps:
-    - name: Checkout zenith repo
-      uses: actions/checkout@v2
-
-    # actions/setup-python@v2 is not working correctly on self-hosted runners
-    # see https://github.com/actions/setup-python/issues/162
-    # and probably https://github.com/actions/setup-python/issues/162#issuecomment-865387976 in particular
-    # so the simplest solution to me is to use already installed system python and spin virtualenvs for job runs.
-    # there is Python 3.7.10 already installed on the machine so use it to install pipenv and then use pipenv's virtuealenvs
-    - name: Install pipenv & deps
-      run: |
-        python3 -m pip install --upgrade pipenv wheel
-        # since pip/pipenv caches are reused there shouldn't be any troubles with install every time
-        pipenv install
-
-    - name: Show versions
-      run: |
-        echo Python
-        python3 --version
-        pipenv run python3 --version
-        echo Pipenv
-        pipenv --version
-        echo Pgbench
-        $PG_BIN/pgbench --version
-
-    # FIXME cluster setup is skipped due to various changes in console API
-    # for now pre created cluster is used. When API gain some stability
-    # after massive changes dynamic cluster setup will be revived.
-    # So use pre created cluster. It needs to be started manually, but stop is automatic after 5 minutes of inactivity
-    - name: Setup cluster
-      env:
-        BENCHMARK_CONNSTR: "${{ secrets.BENCHMARK_STAGING_CONNSTR }}"
-      shell: bash
-      run: |
-        set -e
-
-        echo "Starting cluster"
-        # wake up the cluster
-        $PG_BIN/psql $BENCHMARK_CONNSTR -c "SELECT 1"
-
-    - name: Run benchmark
-      # pgbench is installed system wide from official repo
-      # https://download.postgresql.org/pub/repos/yum/13/redhat/rhel-7-x86_64/
-      # via
-      # sudo tee /etc/yum.repos.d/pgdg.repo<<EOF
-      # [pgdg13]
-      # name=PostgreSQL 13 for RHEL/CentOS 7 - x86_64
-      # baseurl=https://download.postgresql.org/pub/repos/yum/13/redhat/rhel-7-x86_64/
-      # enabled=1
-      # gpgcheck=0
-      # EOF
-      # sudo yum makecache
-      # sudo yum install postgresql13-contrib
-      # actual binaries are located in /usr/pgsql-13/bin/
-      env:
-        TEST_PG_BENCH_TRANSACTIONS_MATRIX: "5000,10000,20000"
-        TEST_PG_BENCH_SCALES_MATRIX: "10,15"
-        PLATFORM: "zenith-staging"
-        BENCHMARK_CONNSTR: "${{ secrets.BENCHMARK_STAGING_CONNSTR }}"
-        REMOTE_ENV: "1" # indicate to test harness that we do not have zenith binaries locally
-      run: |
-        mkdir -p perf-report-staging
-        pipenv run pytest test_runner/performance/ -v -m "remote_cluster" --skip-interfering-proc-check --out-dir perf-report-staging
-
-    - name: Submit result
-      env:
-        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
-      run: |
-        REPORT_FROM=$(realpath perf-report-staging) REPORT_TO=staging scripts/generate_and_push_perf_report.sh

.github/workflows/testing.yml

@@ -64,11 +64,10 @@ jobs:
             target
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
 
-      # Use `env CARGO_INCREMENTAL=0` to mitigate https://github.com/rust-lang/rust/issues/91696 for rustc 1.57.0
       - name: Run cargo build
         run: |
-          env CARGO_INCREMENTAL=0 cargo build --workspace --bins --examples --tests
+          cargo build --workspace --bins --examples --tests
 
       - name: Run cargo test
         run: |
-          env CARGO_INCREMENTAL=0 cargo test -- --nocapture --test-threads=1
+          cargo test -- --nocapture --test-threads=1

.gitignore

@@ -7,7 +7,3 @@ test_output/
 .vscode
 /.zenith
 /integration_tests/.zenith
-
-# Coverage
-*.profraw
-*.profdata

.yapfignore

@@ -1,10 +0,0 @@
-# This file is only read when `yapf` is run from this directory.
-# Hence we only top-level directories here to avoid confusion.
-# See source code for the exact file format: https://github.com/google/yapf/blob/c6077954245bc3add82dafd853a1c7305a6ebd20/yapf/yapflib/file_resources.py#L40-L43
-vendor/
-target/
-tmp_install/
-__pycache__/
-test_output/
-.zenith/
-.git/

Cargo.lock

@@ -41,20 +41,6 @@ version = "1.0.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1"
 
-[[package]]
-name = "async-compression"
-version = "0.3.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5443ccbb270374a2b1055fc72da40e1f237809cd6bb0e97e66d264cd138473a6"
-dependencies = [
- "futures-core",
- "memchr",
- "pin-project-lite",
- "tokio",
- "zstd",
- "zstd-safe",
-]
-
 [[package]]
 name = "async-trait"
 version = "0.1.51"
@@ -68,18 +54,17 @@ dependencies = [
 
 [[package]]
 name = "attohttpc"
-version = "0.18.0"
+version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e69e13a99a7e6e070bb114f7ff381e58c7ccc188630121fc4c2fe4bcf24cd072"
+checksum = "9a8bda305457262b339322106c776e3fd21df860018e566eb6a5b1aa4b6ae02d"
 dependencies = [
  "http",
  "log",
- "rustls 0.20.2",
+ "native-tls",
+ "openssl",
  "serde",
  "serde_json",
  "url",
- "webpki 0.22.0",
- "webpki-roots 0.22.1",
  "wildmatch",
 ]
 
@@ -126,9 +111,9 @@ dependencies = [
 
 [[package]]
 name = "aws-creds"
-version = "0.27.1"
+version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "460a75eac8f3cb7683e0a9a588a83c3ff039331ea7bfbfbfcecf1dacab276e11"
+checksum = "a5e1c8f64305d3f3096cb247983a3cae16f8c2960129699bcb70639e31180794"
 dependencies = [
  "anyhow",
  "attohttpc",
@@ -223,7 +208,8 @@ dependencies = [
 [[package]]
 name = "bookfile"
 version = "0.3.0"
-source = "git+https://github.com/zenithdb/bookfile.git?branch=generic-readext#d51a99c7a0be48c3d9cc7cb85c9b7fb05ce1100c"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "efa3e2086414e1bbecbc10730f265e5b079ab4ea0b830e7219a70dab6471e753"
 dependencies = [
  "aversion",
  "byteorder",
@@ -263,9 +249,6 @@ name = "cc"
 version = "1.0.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd"
-dependencies = [
- "jobserver",
-]
 
 [[package]]
 name = "cexpr"
@@ -321,16 +304,6 @@ dependencies = [
  "vec_map",
 ]
 
-[[package]]
-name = "combine"
-version = "4.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2b2f5d0ee456f3928812dfc8c6d9a1d592b98678f6d56db9b0cd2b7bc6c8db5"
-dependencies = [
- "bytes",
- "memchr",
-]
-
 [[package]]
 name = "const_format"
 version = "0.2.22"
@@ -356,21 +329,43 @@ name = "control_plane"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "bytes",
+ "hex",
  "lazy_static",
  "nix",
  "pageserver",
  "postgres",
+ "postgres_ffi",
+ "rand",
  "regex",
  "reqwest",
  "serde",
+ "serde_json",
  "tar",
  "thiserror",
  "toml",
  "url",
+ "walkeeper",
  "workspace_hack",
  "zenith_utils",
 ]
 
+[[package]]
+name = "core-foundation"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6888e10551bb93e424d8df1d07f1a8b4fceb0001a3a4b048bfc47554946f47b3"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
+
 [[package]]
 name = "cpufeatures"
 version = "0.2.1"
@@ -389,16 +384,6 @@ dependencies = [
  "rustc_version",
 ]
 
-[[package]]
-name = "crossbeam-utils"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db"
-dependencies = [
- "cfg-if",
- "lazy_static",
-]
-
 [[package]]
 name = "crypto-mac"
 version = "0.10.1"
@@ -467,12 +452,6 @@ dependencies = [
  "rand",
 ]
 
-[[package]]
-name = "either"
-version = "1.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
-
 [[package]]
 name = "encoding_rs"
 version = "0.8.28"
@@ -519,6 +498,21 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.0.1"
@@ -547,9 +541,9 @@ checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7"
 
 [[package]]
 name = "futures"
-version = "0.3.18"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cd0210d8c325c245ff06fd95a3b13689a1a276ac8cfa8e8720cb840bfb84b9e"
+checksum = "a12aa0eb539080d55c3f2d45a67c3b58b6b0773c1a3ca2dfec66d58c97fd66ca"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -562,9 +556,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.18"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fc8cd39e3dbf865f7340dce6a2d401d24fd37c6fe6c4f0ee0de8bfca2252d27"
+checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -572,15 +566,15 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.18"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "629316e42fe7c2a0b9a65b47d159ceaa5453ab14e8f0a3c5eedbb8cd55b4a445"
+checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.18"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b808bf53348a36cab739d7e04755909b9fcaaa69b7d7e588b37b6ec62704c97"
+checksum = "45025be030969d763025784f7f355043dc6bc74093e4ecc5000ca4dc50d8745c"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -589,16 +583,18 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.18"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e481354db6b5c353246ccf6a728b0c5511d752c08da7260546fc0933869daa11"
+checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377"
 
 [[package]]
 name = "futures-macro"
-version = "0.3.18"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a89f17b21645bc4ed773c69af9c9a0effd4a3f1a3876eadd453469f8854e7fdd"
+checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb"
 dependencies = [
+ "autocfg",
+ "proc-macro-hack",
  "proc-macro2",
  "quote",
  "syn",
@@ -606,22 +602,23 @@ dependencies = [
 
 [[package]]
 name = "futures-sink"
-version = "0.3.18"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "996c6442437b62d21a32cd9906f9c41e7dc1e19a9579843fad948696769305af"
+checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11"
 
 [[package]]
 name = "futures-task"
-version = "0.3.18"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dabf1872aaab32c886832f2276d2f5399887e2bd613698a02359e4ea83f8de12"
+checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99"
 
 [[package]]
 name = "futures-util"
-version = "0.3.18"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d22213122356472061ac0f1ab2cee28d2bac8491410fd68c2af53d1cedb83e"
+checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481"
 dependencies = [
+ "autocfg",
  "futures-channel",
  "futures-core",
  "futures-io",
@@ -631,6 +628,8 @@ dependencies = [
  "memchr",
  "pin-project-lite",
  "pin-utils",
+ "proc-macro-hack",
+ "proc-macro-nested",
  "slab",
 ]
 
@@ -655,28 +654,6 @@ dependencies = [
  "wasi",
 ]
 
-[[package]]
-name = "git-version"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6b0decc02f4636b9ccad390dcbe77b722a77efedfa393caf8379a51d5c61899"
-dependencies = [
- "git-version-macro",
- "proc-macro-hack",
-]
-
-[[package]]
-name = "git-version-macro"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe69f1cbdb6e28af2bac214e943b99ce8a0a06b447d15d3e61161b0423139f3f"
-dependencies = [
- "proc-macro-hack",
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "glob"
 version = "0.3.0"
@@ -832,18 +809,16 @@ dependencies = [
 ]
 
 [[package]]
-name = "hyper-rustls"
-version = "0.22.1"
+name = "hyper-tls"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64"
+checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
 dependencies = [
- "futures-util",
+ "bytes",
  "hyper",
- "log",
- "rustls 0.19.1",
+ "native-tls",
  "tokio",
- "tokio-rustls",
- "webpki 0.21.4",
+ "tokio-native-tls",
 ]
 
 [[package]]
@@ -882,30 +857,12 @@ version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9"
 
-[[package]]
-name = "itertools"
-version = "0.10.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itoa"
 version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
 
-[[package]]
-name = "jobserver"
-version = "0.1.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "js-sys"
 version = "0.3.55"
@@ -929,15 +886,6 @@ dependencies = [
  "simple_asn1",
 ]
 
-[[package]]
-name = "kstring"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b310ccceade8121d7d77fee406160e457c2f4e7c7982d589da3499bc7ea4526"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -1048,6 +996,15 @@ version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
 
+[[package]]
+name = "minidom"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "332592c2149fc7dd40a64fc9ef6f0d65607284b474cef9817d1fc8c7e7b3608e"
+dependencies = [
+ "quick-xml",
+]
+
 [[package]]
 name = "mio"
 version = "0.7.13"
@@ -1070,6 +1027,24 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "native-tls"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d"
+dependencies = [
+ "lazy_static",
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
 [[package]]
 name = "nix"
 version = "0.23.0"
@@ -1156,6 +1131,39 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
+[[package]]
+name = "openssl"
+version = "0.10.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d9facdb76fec0b73c406f125d44d86fdad818d66fef0531eec9233ca425ff4a"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.67"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69df2d8dfc6ce3aaf44b40dec6f487d5a886516cf6879c49e98e0710f310a058"
+dependencies = [
+ "autocfg",
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "ordered-multimap"
 version = "0.3.1"
@@ -1171,7 +1179,6 @@ name = "pageserver"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "async-compression",
  "async-trait",
  "bookfile",
  "byteorder",
@@ -1180,7 +1187,6 @@ dependencies = [
  "clap",
  "const_format",
  "crc32c",
- "crossbeam-utils",
  "daemonize",
  "futures",
  "hex",
@@ -1189,9 +1195,6 @@ dependencies = [
  "hyper",
  "lazy_static",
  "log",
- "nix",
- "once_cell",
- "parking_lot",
  "postgres",
  "postgres-protocol",
  "postgres-types",
@@ -1208,11 +1211,8 @@ dependencies = [
  "tempfile",
  "thiserror",
  "tokio",
- "tokio-postgres",
- "tokio-stream",
- "toml_edit",
+ "toml",
  "tracing",
- "tracing-futures",
  "url",
  "workspace_hack",
  "zenith_metrics",
@@ -1285,26 +1285,6 @@ dependencies = [
  "siphasher",
 ]
 
-[[package]]
-name = "pin-project"
-version = "1.0.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08"
-dependencies = [
- "pin-project-internal",
-]
-
-[[package]]
-name = "pin-project-internal"
-version = "1.0.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "pin-project-lite"
 version = "0.2.7"
@@ -1317,6 +1297,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "pkg-config"
+version = "0.3.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb"
+
 [[package]]
 name = "postgres"
 version = "0.19.1"
@@ -1393,6 +1379,12 @@ version = "0.5.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
 
+[[package]]
+name = "proc-macro-nested"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086"
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.30"
@@ -1424,12 +1416,10 @@ dependencies = [
  "bytes",
  "clap",
  "hex",
- "lazy_static",
  "md5",
- "parking_lot",
  "rand",
  "reqwest",
- "rustls 0.19.1",
+ "rustls",
  "serde",
  "serde_json",
  "tokio",
@@ -1437,6 +1427,15 @@ dependencies = [
  "zenith_utils",
 ]
 
+[[package]]
+name = "quick-xml"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.10"
@@ -1560,25 +1559,24 @@ dependencies = [
  "http",
  "http-body",
  "hyper",
- "hyper-rustls",
+ "hyper-tls",
  "ipnet",
  "js-sys",
  "lazy_static",
  "log",
  "mime",
+ "native-tls",
  "percent-encoding",
  "pin-project-lite",
- "rustls 0.19.1",
  "serde",
  "serde_json",
  "serde_urlencoded",
  "tokio",
- "tokio-rustls",
+ "tokio-native-tls",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
  "web-sys",
- "webpki-roots 0.21.1",
  "winreg",
 ]
 
@@ -1622,9 +1620,9 @@ dependencies = [
 
 [[package]]
 name = "rust-s3"
-version = "0.28.0"
+version = "0.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18c58d4682844a5d6301efbf915dd7a9d3d638006f9bb821527a0bbbf2a4cfc2"
+checksum = "b2f26775d15f43dc848ef0ec65f83de8775549e486c7a3a576652049a7122d32"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -1633,12 +1631,14 @@ dependencies = [
  "base64 0.13.0",
  "cfg-if",
  "chrono",
+ "futures",
  "hex",
  "hmac 0.11.0",
  "http",
  "log",
  "maybe-async",
  "md5",
+ "minidom",
  "percent-encoding",
  "reqwest",
  "serde",
@@ -1674,20 +1674,8 @@ dependencies = [
  "base64 0.13.0",
  "log",
  "ring",
- "sct 0.6.1",
- "webpki 0.21.4",
-]
-
-[[package]]
-name = "rustls"
-version = "0.20.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d37e5e2290f3e040b594b1a9e04377c2c671f1a1cfd9bfdef82106ac1c113f84"
-dependencies = [
- "log",
- "ring",
- "sct 0.7.0",
- "webpki 0.22.0",
+ "sct",
+ "webpki",
 ]
 
 [[package]]
@@ -1696,7 +1684,7 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7fb079b52cfdb005752b7c3c646048e702003576a8321058e4c8b38227c11aa6"
 dependencies = [
- "rustls 0.19.1",
+ "rustls",
 ]
 
 [[package]]
@@ -1714,6 +1702,16 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "schannel"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75"
+dependencies = [
+ "lazy_static",
+ "winapi",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -1731,13 +1729,26 @@ dependencies = [
 ]
 
 [[package]]
-name = "sct"
-version = "0.7.0"
+name = "security-framework"
+version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"
+checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467"
 dependencies = [
- "ring",
- "untrusted",
+ "bitflags",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
 ]
 
 [[package]]
@@ -1854,6 +1865,7 @@ version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c98891d737e271a2954825ef19e46bd16bdb98e2746f2eec4f7a4ef7946efd1"
 dependencies = [
+ "cc",
  "libc",
  "signal-hook-registry",
 ]
@@ -2079,6 +2091,16 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "tokio-native-tls"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-postgres"
 version = "0.7.1"
@@ -2101,22 +2123,11 @@ dependencies = [
  "tokio-util",
 ]
 
-[[package]]
-name = "tokio-rustls"
-version = "0.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"
-dependencies = [
- "rustls 0.19.1",
- "tokio",
- "webpki 0.21.4",
-]
-
 [[package]]
 name = "tokio-stream"
-version = "0.1.8"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3"
+checksum = "7b2f3f698253f03119ac0102beaa64f67a67e08074d03a22d18784104543727f"
 dependencies = [
  "futures-core",
  "pin-project-lite",
@@ -2146,19 +2157,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "toml_edit"
-version = "0.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c14e16aafed5fb7a1bdc293270ca28f17d0a09b4f528f8de34521f2a32d99198"
-dependencies = [
- "combine",
- "indexmap",
- "itertools",
- "kstring",
- "serde",
-]
-
 [[package]]
 name = "tower-service"
 version = "0.3.1"
@@ -2197,16 +2195,6 @@ dependencies = [
  "lazy_static",
 ]
 
-[[package]]
-name = "tracing-futures"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
-dependencies = [
- "pin-project",
- "tracing",
-]
-
 [[package]]
 name = "tracing-log"
 version = "0.1.2"
@@ -2293,6 +2281,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
 [[package]]
 name = "vec_map"
 version = "0.8.2"
@@ -2321,6 +2315,7 @@ name = "walkeeper"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "bincode",
  "byteorder",
  "bytes",
  "clap",
@@ -2332,6 +2327,8 @@ dependencies = [
  "humantime",
  "hyper",
  "lazy_static",
+ "log",
+ "pageserver",
  "postgres",
  "postgres-protocol",
  "postgres_ffi",
@@ -2340,11 +2337,8 @@ dependencies = [
  "rust-s3",
  "serde",
  "serde_json",
- "signal-hook",
- "tempfile",
  "tokio",
- "tokio-postgres",
- "tracing",
+ "tokio-stream",
  "walkdir",
  "workspace_hack",
  "zenith_metrics",
@@ -2453,34 +2447,6 @@ dependencies = [
  "untrusted",
 ]
 
-[[package]]
-name = "webpki"
-version = "0.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"
-dependencies = [
- "ring",
- "untrusted",
-]
-
-[[package]]
-name = "webpki-roots"
-version = "0.21.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940"
-dependencies = [
- "webpki 0.21.4",
-]
-
-[[package]]
-name = "webpki-roots"
-version = "0.22.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c475786c6f47219345717a043a37ec04cb4bc185e28853adcc4fa0a947eba630"
-dependencies = [
- "webpki 0.22.0",
-]
-
 [[package]]
 name = "which"
 version = "3.1.1"
@@ -2492,9 +2458,9 @@ dependencies = [
 
 [[package]]
 name = "wildmatch"
-version = "2.1.0"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6c48bd20df7e4ced539c12f570f937c6b4884928a87fee70a479d72f031d4e0"
+checksum = "7f44b95f62d34113cf558c93511ac93027e03e9c29a60dd0fd70e6e025c7270a"
 
 [[package]]
 name = "winapi"
@@ -2607,7 +2573,6 @@ dependencies = [
  "bincode",
  "byteorder",
  "bytes",
- "git-version",
  "hex",
  "hex-literal",
  "hyper",
@@ -2617,46 +2582,16 @@ dependencies = [
  "postgres",
  "rand",
  "routerify",
- "rustls 0.19.1",
+ "rustls",
  "rustls-split",
  "serde",
  "serde_json",
- "signal-hook",
  "tempfile",
  "thiserror",
  "tokio",
  "tracing",
  "tracing-subscriber",
- "webpki 0.21.4",
+ "webpki",
  "workspace_hack",
  "zenith_metrics",
 ]
-
-[[package]]
-name = "zstd"
-version = "0.7.0+zstd.1.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9428752481d8372e15b1bf779ea518a179ad6c771cca2d2c60e4fbff3cc2cd52"
-dependencies = [
- "zstd-safe",
-]
-
-[[package]]
-name = "zstd-safe"
-version = "3.1.0+zstd.1.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aa1926623ad7fe406e090555387daf73db555b948134b4d73eac5eb08fb666d"
-dependencies = [
- "libc",
- "zstd-sys",
-]
-
-[[package]]
-name = "zstd-sys"
-version = "1.5.0+zstd.1.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e6c094340240369025fc6b731b054ee2a834328fa584310ac96aa4baebdc465"
-dependencies = [
- "cc",
- "libc",
-]

Dockerfile

@@ -21,15 +21,11 @@ RUN rm -rf postgres_install/build
 # net time waste in a lot of cases. Copying Cargo.lock with empty lib.rs should do the work.
 #
 FROM zenithdb/build:buster AS build
-
-ARG GIT_VERSION
-RUN if [ -z "$GIT_VERSION" ]; then echo "GIT_VERSION is reqired, use build_arg to pass it"; exit 1; fi
-
 WORKDIR /zenith
 COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
 
 COPY . .
-RUN GIT_VERSION=$GIT_VERSION cargo build --release
+RUN cargo build --release
 
 #
 # Copy binaries to resulting image.

Pipfile

@@ -1,30 +0,0 @@
-[[source]]
-url = "https://pypi.python.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-pytest = ">=6.0.0"
-typing-extensions = "*"
-pyjwt = {extras = ["crypto"], version = "*"}
-requests = "*"
-pytest-xdist = "*"
-asyncpg = "*"
-cached-property = "*"
-psycopg2-binary = "*"
-jinja2 = "*"
-
-[dev-packages]
-# Behavior may change slightly between versions. These are run continuously,
-# so we pin exact versions to avoid suprising breaks. Update if comfortable.
-yapf = "==0.31.0"
-mypy = "==0.910"
-# Non-pinned packages follow.
-pipenv = "*"
-flake8 = "*"
-types-requests = "*"
-types-psycopg2 = "*"
-
-[requires]
-# we need at least 3.7, but pipenv doesn't allow to say this directly
-python_version = "3"

Pipfile

@@ -0,0 +1 @@
+./test_runner/Pipfile

Pipfile.lock

@@ -1,652 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "c309cb963a7b07ae3d30e9cbf08b495f77bdecc0e5356fc89d133c4fbcb65b2b"
-        },
-        "pipfile-spec": 6,
-        "requires": {
-            "python_version": "3"
-        },
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.python.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "asyncpg": {
-            "hashes": [
-                "sha256:129d501f3d30616afd51eb8d3142ef51ba05374256bd5834cec3ef4956a9b317",
-                "sha256:29ef6ae0a617fc13cc2ac5dc8e9b367bb83cba220614b437af9b67766f4b6b20",
-                "sha256:41704c561d354bef01353835a7846e5606faabbeb846214dfcf666cf53319f18",
-                "sha256:556b0e92e2b75dc028b3c4bc9bd5162ddf0053b856437cf1f04c97f9c6837d03",
-                "sha256:8ff5073d4b654e34bd5eaadc01dc4d68b8a9609084d835acd364cd934190a08d",
-                "sha256:a458fc69051fbb67d995fdda46d75a012b5d6200f91e17d23d4751482640ed4c",
-                "sha256:a7095890c96ba36f9f668eb552bb020dddb44f8e73e932f8573efc613ee83843",
-                "sha256:a738f4807c853623d3f93f0fea11f61be6b0e5ca16ea8aeb42c2c7ee742aa853",
-                "sha256:c4fc0205fe4ddd5aeb3dfdc0f7bafd43411181e1f5650189608e5971cceacff1",
-                "sha256:dd2fa063c3344823487d9ddccb40802f02622ddf8bf8a6cc53885ee7a2c1c0c6",
-                "sha256:ddffcb85227bf39cd1bedd4603e0082b243cf3b14ced64dce506a15b05232b83",
-                "sha256:e36c6806883786b19551bb70a4882561f31135dc8105a59662e0376cf5b2cbc5",
-                "sha256:eed43abc6ccf1dc02e0d0efc06ce46a411362f3358847c6b0ec9a43426f91ece"
-            ],
-            "index": "pypi",
-            "version": "==0.24.0"
-        },
-        "attrs": {
-            "hashes": [
-                "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1",
-                "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==21.2.0"
-        },
-        "cached-property": {
-            "hashes": [
-                "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130",
-                "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0"
-            ],
-            "index": "pypi",
-            "version": "==1.5.2"
-        },
-        "certifi": {
-            "hashes": [
-                "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872",
-                "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"
-            ],
-            "version": "==2021.10.8"
-        },
-        "cffi": {
-            "hashes": [
-                "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3",
-                "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2",
-                "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636",
-                "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20",
-                "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728",
-                "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27",
-                "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66",
-                "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443",
-                "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0",
-                "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7",
-                "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39",
-                "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605",
-                "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a",
-                "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37",
-                "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029",
-                "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139",
-                "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc",
-                "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df",
-                "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14",
-                "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880",
-                "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2",
-                "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a",
-                "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e",
-                "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474",
-                "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024",
-                "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8",
-                "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0",
-                "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e",
-                "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a",
-                "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e",
-                "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032",
-                "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6",
-                "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e",
-                "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b",
-                "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e",
-                "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954",
-                "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962",
-                "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c",
-                "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4",
-                "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55",
-                "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962",
-                "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023",
-                "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c",
-                "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6",
-                "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8",
-                "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382",
-                "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7",
-                "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc",
-                "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997",
-                "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796"
-            ],
-            "version": "==1.15.0"
-        },
-        "charset-normalizer": {
-            "hashes": [
-                "sha256:e019de665e2bcf9c2b64e2e5aa025fa991da8720daa3c1138cadd2fd1856aed0",
-                "sha256:f7af805c321bfa1ce6714c51f254e0d5bb5e5834039bc17db7ebe3a4cec9492b"
-            ],
-            "markers": "python_version >= '3'",
-            "version": "==2.0.7"
-        },
-        "cryptography": {
-            "hashes": [
-                "sha256:07bb7fbfb5de0980590ddfc7f13081520def06dc9ed214000ad4372fb4e3c7f6",
-                "sha256:18d90f4711bf63e2fb21e8c8e51ed8189438e6b35a6d996201ebd98a26abbbe6",
-                "sha256:1ed82abf16df40a60942a8c211251ae72858b25b7421ce2497c2eb7a1cee817c",
-                "sha256:22a38e96118a4ce3b97509443feace1d1011d0571fae81fc3ad35f25ba3ea999",
-                "sha256:2d69645f535f4b2c722cfb07a8eab916265545b3475fdb34e0be2f4ee8b0b15e",
-                "sha256:4a2d0e0acc20ede0f06ef7aa58546eee96d2592c00f450c9acb89c5879b61992",
-                "sha256:54b2605e5475944e2213258e0ab8696f4f357a31371e538ef21e8d61c843c28d",
-                "sha256:7075b304cd567694dc692ffc9747f3e9cb393cc4aa4fb7b9f3abd6f5c4e43588",
-                "sha256:7b7ceeff114c31f285528ba8b390d3e9cfa2da17b56f11d366769a807f17cbaa",
-                "sha256:7eba2cebca600a7806b893cb1d541a6e910afa87e97acf2021a22b32da1df52d",
-                "sha256:928185a6d1ccdb816e883f56ebe92e975a262d31cc536429041921f8cb5a62fd",
-                "sha256:9933f28f70d0517686bd7de36166dda42094eac49415459d9bdf5e7df3e0086d",
-                "sha256:a688ebcd08250eab5bb5bca318cc05a8c66de5e4171a65ca51db6bd753ff8953",
-                "sha256:abb5a361d2585bb95012a19ed9b2c8f412c5d723a9836418fab7aaa0243e67d2",
-                "sha256:c10c797ac89c746e488d2ee92bd4abd593615694ee17b2500578b63cad6b93a8",
-                "sha256:ced40344e811d6abba00295ced98c01aecf0c2de39481792d87af4fa58b7b4d6",
-                "sha256:d57e0cdc1b44b6cdf8af1d01807db06886f10177469312fbde8f44ccbb284bc9",
-                "sha256:d99915d6ab265c22873f1b4d6ea5ef462ef797b4140be4c9d8b179915e0985c6",
-                "sha256:eb80e8a1f91e4b7ef8b33041591e6d89b2b8e122d787e87eeb2b08da71bb16ad",
-                "sha256:ebeddd119f526bcf323a89f853afb12e225902a24d29b55fe18dd6fcb2838a76"
-            ],
-            "version": "==35.0.0"
-        },
-        "execnet": {
-            "hashes": [
-                "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5",
-                "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==1.9.0"
-        },
-        "idna": {
-            "hashes": [
-                "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
-                "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
-            ],
-            "markers": "python_version >= '3'",
-            "version": "==3.3"
-        },
-        "importlib-metadata": {
-            "hashes": [
-                "sha256:b618b6d2d5ffa2f16add5697cf57a46c76a56229b0ed1c438322e4e95645bd15",
-                "sha256:f284b3e11256ad1e5d03ab86bb2ccd6f5339688ff17a4d797a0fe7df326f23b1"
-            ],
-            "markers": "python_version < '3.8'",
-            "version": "==4.8.1"
-        },
-        "iniconfig": {
-            "hashes": [
-                "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
-                "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
-            ],
-            "version": "==1.1.1"
-        },
-        "jinja2": {
-            "hashes": [
-                "sha256:827a0e32839ab1600d4eb1c4c33ec5a8edfbc5cb42dafa13b81f182f97784b45",
-                "sha256:8569982d3f0889eed11dd620c706d39b60c36d6d25843961f33f77fb6bc6b20c"
-            ],
-            "index": "pypi",
-            "version": "==3.0.2"
-        },
-        "markupsafe": {
-            "hashes": [
-                "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298",
-                "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64",
-                "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b",
-                "sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194",
-                "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567",
-                "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff",
-                "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724",
-                "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74",
-                "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646",
-                "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35",
-                "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6",
-                "sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a",
-                "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6",
-                "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad",
-                "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26",
-                "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38",
-                "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac",
-                "sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7",
-                "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6",
-                "sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047",
-                "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75",
-                "sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f",
-                "sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b",
-                "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135",
-                "sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8",
-                "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a",
-                "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a",
-                "sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1",
-                "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9",
-                "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864",
-                "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914",
-                "sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee",
-                "sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f",
-                "sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18",
-                "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8",
-                "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2",
-                "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d",
-                "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b",
-                "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b",
-                "sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86",
-                "sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6",
-                "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f",
-                "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb",
-                "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833",
-                "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28",
-                "sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e",
-                "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415",
-                "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902",
-                "sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f",
-                "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d",
-                "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9",
-                "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d",
-                "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145",
-                "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066",
-                "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c",
-                "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1",
-                "sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a",
-                "sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207",
-                "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f",
-                "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53",
-                "sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd",
-                "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134",
-                "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85",
-                "sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9",
-                "sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5",
-                "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94",
-                "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509",
-                "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51",
-                "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==2.0.1"
-        },
-        "packaging": {
-            "hashes": [
-                "sha256:096d689d78ca690e4cd8a89568ba06d07ca097e3306a4381635073ca91479966",
-                "sha256:14317396d1e8cdb122989b916fa2c7e9ca8e2be9e8060a6eff75b6b7b4d8a7e0"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==21.2"
-        },
-        "pluggy": {
-            "hashes": [
-                "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159",
-                "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.0.0"
-        },
-        "psycopg2-binary": {
-            "hashes": [
-                "sha256:0b7dae87f0b729922e06f85f667de7bf16455d411971b2043bbd9577af9d1975",
-                "sha256:0f2e04bd2a2ab54fa44ee67fe2d002bb90cee1c0f1cc0ebc3148af7b02034cbd",
-                "sha256:123c3fb684e9abfc47218d3784c7b4c47c8587951ea4dd5bc38b6636ac57f616",
-                "sha256:1473c0215b0613dd938db54a653f68251a45a78b05f6fc21af4326f40e8360a2",
-                "sha256:14db1752acdd2187d99cb2ca0a1a6dfe57fc65c3281e0f20e597aac8d2a5bd90",
-                "sha256:1e3a362790edc0a365385b1ac4cc0acc429a0c0d662d829a50b6ce743ae61b5a",
-                "sha256:1e85b74cbbb3056e3656f1cc4781294df03383127a8114cbc6531e8b8367bf1e",
-                "sha256:20f1ab44d8c352074e2d7ca67dc00843067788791be373e67a0911998787ce7d",
-                "sha256:24b0b6688b9f31a911f2361fe818492650795c9e5d3a1bc647acbd7440142a4f",
-                "sha256:2f62c207d1740b0bde5c4e949f857b044818f734a3d57f1d0d0edc65050532ed",
-                "sha256:3242b9619de955ab44581a03a64bdd7d5e470cc4183e8fcadd85ab9d3756ce7a",
-                "sha256:35c4310f8febe41f442d3c65066ca93cccefd75013df3d8c736c5b93ec288140",
-                "sha256:4235f9d5ddcab0b8dbd723dca56ea2922b485ea00e1dafacf33b0c7e840b3d32",
-                "sha256:542875f62bc56e91c6eac05a0deadeae20e1730be4c6334d8f04c944fcd99759",
-                "sha256:5ced67f1e34e1a450cdb48eb53ca73b60aa0af21c46b9b35ac3e581cf9f00e31",
-                "sha256:661509f51531ec125e52357a489ea3806640d0ca37d9dada461ffc69ee1e7b6e",
-                "sha256:7360647ea04db2e7dff1648d1da825c8cf68dc5fbd80b8fb5b3ee9f068dcd21a",
-                "sha256:736b8797b58febabb85494142c627bd182b50d2a7ec65322983e71065ad3034c",
-                "sha256:8c13d72ed6af7fd2c8acbd95661cf9477f94e381fce0792c04981a8283b52917",
-                "sha256:988b47ac70d204aed01589ed342303da7c4d84b56c2f4c4b8b00deda123372bf",
-                "sha256:995fc41ebda5a7a663a254a1dcac52638c3e847f48307b5416ee373da15075d7",
-                "sha256:a36c7eb6152ba5467fb264d73844877be8b0847874d4822b7cf2d3c0cb8cdcb0",
-                "sha256:aed4a9a7e3221b3e252c39d0bf794c438dc5453bc2963e8befe9d4cd324dff72",
-                "sha256:aef9aee84ec78af51107181d02fe8773b100b01c5dfde351184ad9223eab3698",
-                "sha256:b0221ca5a9837e040ebf61f48899926b5783668b7807419e4adae8175a31f773",
-                "sha256:b4d7679a08fea64573c969f6994a2631908bb2c0e69a7235648642f3d2e39a68",
-                "sha256:c250a7ec489b652c892e4f0a5d122cc14c3780f9f643e1a326754aedf82d9a76",
-                "sha256:ca86db5b561b894f9e5f115d6a159fff2a2570a652e07889d8a383b5fae66eb4",
-                "sha256:cfc523edecddaef56f6740d7de1ce24a2fdf94fd5e704091856a201872e37f9f",
-                "sha256:d92272c7c16e105788efe2cfa5d680f07e34e0c29b03c1908f8636f55d5f915a",
-                "sha256:da113b70f6ec40e7d81b43d1b139b9db6a05727ab8be1ee559f3a69854a69d34",
-                "sha256:f6fac64a38f6768e7bc7b035b9e10d8a538a9fadce06b983fb3e6fa55ac5f5ce",
-                "sha256:f8559617b1fcf59a9aedba2c9838b5b6aa211ffedecabca412b92a1ff75aac1a",
-                "sha256:fbb42a541b1093385a2d8c7eec94d26d30437d0e77c1d25dae1dcc46741a385e"
-            ],
-            "index": "pypi",
-            "version": "==2.9.1"
-        },
-        "py": {
-            "hashes": [
-                "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3",
-                "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.10.0"
-        },
-        "pycparser": {
-            "hashes": [
-                "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",
-                "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==2.20"
-        },
-        "pyjwt": {
-            "extras": [
-                "crypto"
-            ],
-            "hashes": [
-                "sha256:b888b4d56f06f6dcd777210c334e69c737be74755d3e5e9ee3fe67dc18a0ee41",
-                "sha256:e0c4bb8d9f0af0c7f5b1ec4c5036309617d03d56932877f2f7a0beeb5318322f"
-            ],
-            "index": "pypi",
-            "version": "==2.3.0"
-        },
-        "pyparsing": {
-            "hashes": [
-                "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
-                "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
-            ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==2.4.7"
-        },
-        "pytest": {
-            "hashes": [
-                "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89",
-                "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"
-            ],
-            "index": "pypi",
-            "version": "==6.2.5"
-        },
-        "pytest-forked": {
-            "hashes": [
-                "sha256:6aa9ac7e00ad1a539c41bec6d21011332de671e938c7637378ec9710204e37ca",
-                "sha256:dc4147784048e70ef5d437951728825a131b81714b398d5d52f17c7c144d8815"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==1.3.0"
-        },
-        "pytest-xdist": {
-            "hashes": [
-                "sha256:7b61ebb46997a0820a263553179d6d1e25a8c50d8a8620cd1aa1e20e3be99168",
-                "sha256:89b330316f7fc475f999c81b577c2b926c9569f3d397ae432c0c2e2496d61ff9"
-            ],
-            "index": "pypi",
-            "version": "==2.4.0"
-        },
-        "requests": {
-            "hashes": [
-                "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24",
-                "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"
-            ],
-            "index": "pypi",
-            "version": "==2.26.0"
-        },
-        "toml": {
-            "hashes": [
-                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
-                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
-            ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==0.10.2"
-        },
-        "typing-extensions": {
-            "hashes": [
-                "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e",
-                "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7",
-                "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"
-            ],
-            "index": "pypi",
-            "version": "==3.10.0.2"
-        },
-        "urllib3": {
-            "hashes": [
-                "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece",
-                "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
-            "version": "==1.26.7"
-        },
-        "zipp": {
-            "hashes": [
-                "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832",
-                "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.6.0"
-        }
-    },
-    "develop": {
-        "backports.entry-points-selectable": {
-            "hashes": [
-                "sha256:988468260ec1c196dab6ae1149260e2f5472c9110334e5d51adcb77867361f6a",
-                "sha256:a6d9a871cde5e15b4c4a53e3d43ba890cc6861ec1332c9c2428c92f977192acc"
-            ],
-            "markers": "python_version >= '2.7'",
-            "version": "==1.1.0"
-        },
-        "certifi": {
-            "hashes": [
-                "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872",
-                "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"
-            ],
-            "version": "==2021.10.8"
-        },
-        "distlib": {
-            "hashes": [
-                "sha256:c8b54e8454e5bf6237cc84c20e8264c3e991e824ef27e8f1e81049867d861e31",
-                "sha256:d982d0751ff6eaaab5e2ec8e691d949ee80eddf01a62eaa96ddb11531fe16b05"
-            ],
-            "version": "==0.3.3"
-        },
-        "filelock": {
-            "hashes": [
-                "sha256:7afc856f74fa7006a289fd10fa840e1eebd8bbff6bffb69c26c54a0512ea8cf8",
-                "sha256:bb2a1c717df74c48a2d00ed625e5a66f8572a3a30baacb7657add1d7bac4097b"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.3.2"
-        },
-        "flake8": {
-            "hashes": [
-                "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d",
-                "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"
-            ],
-            "index": "pypi",
-            "version": "==4.0.1"
-        },
-        "importlib-metadata": {
-            "hashes": [
-                "sha256:b618b6d2d5ffa2f16add5697cf57a46c76a56229b0ed1c438322e4e95645bd15",
-                "sha256:f284b3e11256ad1e5d03ab86bb2ccd6f5339688ff17a4d797a0fe7df326f23b1"
-            ],
-            "markers": "python_version < '3.8'",
-            "version": "==4.8.1"
-        },
-        "mccabe": {
-            "hashes": [
-                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
-                "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
-            ],
-            "version": "==0.6.1"
-        },
-        "mypy": {
-            "hashes": [
-                "sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9",
-                "sha256:0aadfb2d3935988ec3815952e44058a3100499f5be5b28c34ac9d79f002a4a9a",
-                "sha256:119bed3832d961f3a880787bf621634ba042cb8dc850a7429f643508eeac97b9",
-                "sha256:1a85e280d4d217150ce8cb1a6dddffd14e753a4e0c3cf90baabb32cefa41b59e",
-                "sha256:3c4b8ca36877fc75339253721f69603a9c7fdb5d4d5a95a1a1b899d8b86a4de2",
-                "sha256:3e382b29f8e0ccf19a2df2b29a167591245df90c0b5a2542249873b5c1d78212",
-                "sha256:42c266ced41b65ed40a282c575705325fa7991af370036d3f134518336636f5b",
-                "sha256:53fd2eb27a8ee2892614370896956af2ff61254c275aaee4c230ae771cadd885",
-                "sha256:704098302473cb31a218f1775a873b376b30b4c18229421e9e9dc8916fd16150",
-                "sha256:7df1ead20c81371ccd6091fa3e2878559b5c4d4caadaf1a484cf88d93ca06703",
-                "sha256:866c41f28cee548475f146aa4d39a51cf3b6a84246969f3759cb3e9c742fc072",
-                "sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457",
-                "sha256:adaeee09bfde366d2c13fe6093a7df5df83c9a2ba98638c7d76b010694db760e",
-                "sha256:b6fb13123aeef4a3abbcfd7e71773ff3ff1526a7d3dc538f3929a49b42be03f0",
-                "sha256:b94e4b785e304a04ea0828759172a15add27088520dc7e49ceade7834275bedb",
-                "sha256:c0df2d30ed496a08de5daed2a9ea807d07c21ae0ab23acf541ab88c24b26ab97",
-                "sha256:c6c2602dffb74867498f86e6129fd52a2770c48b7cd3ece77ada4fa38f94eba8",
-                "sha256:ceb6e0a6e27fb364fb3853389607cf7eb3a126ad335790fa1e14ed02fba50811",
-                "sha256:d9dd839eb0dc1bbe866a288ba3c1afc33a202015d2ad83b31e875b5905a079b6",
-                "sha256:e4dab234478e3bd3ce83bac4193b2ecd9cf94e720ddd95ce69840273bf44f6de",
-                "sha256:ec4e0cd079db280b6bdabdc807047ff3e199f334050db5cbb91ba3e959a67504",
-                "sha256:ecd2c3fe726758037234c93df7e98deb257fd15c24c9180dacf1ef829da5f921",
-                "sha256:ef565033fa5a958e62796867b1df10c40263ea9ded87164d67572834e57a174d"
-            ],
-            "index": "pypi",
-            "version": "==0.910"
-        },
-        "mypy-extensions": {
-            "hashes": [
-                "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
-                "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
-            ],
-            "version": "==0.4.3"
-        },
-        "pipenv": {
-            "hashes": [
-                "sha256:05958fadcd70b2de6a27542fcd2bd72dd5c59c6d35307fdac3e06361fb06e30e",
-                "sha256:d180f5be4775c552fd5e69ae18a9d6099d9dafb462efe54f11c72cb5f4d5e977"
-            ],
-            "index": "pypi",
-            "version": "==2021.5.29"
-        },
-        "platformdirs": {
-            "hashes": [
-                "sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2",
-                "sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==2.4.0"
-        },
-        "pycodestyle": {
-            "hashes": [
-                "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20",
-                "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==2.8.0"
-        },
-        "pyflakes": {
-            "hashes": [
-                "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c",
-                "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==2.4.0"
-        },
-        "six": {
-            "hashes": [
-                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
-                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.16.0"
-        },
-        "toml": {
-            "hashes": [
-                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
-                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
-            ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==0.10.2"
-        },
-        "typed-ast": {
-            "hashes": [
-                "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace",
-                "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff",
-                "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266",
-                "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528",
-                "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6",
-                "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808",
-                "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4",
-                "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363",
-                "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341",
-                "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04",
-                "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41",
-                "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e",
-                "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3",
-                "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899",
-                "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805",
-                "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c",
-                "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c",
-                "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39",
-                "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a",
-                "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3",
-                "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7",
-                "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f",
-                "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075",
-                "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0",
-                "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40",
-                "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428",
-                "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927",
-                "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3",
-                "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f",
-                "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"
-            ],
-            "markers": "python_version < '3.8'",
-            "version": "==1.4.3"
-        },
-        "types-psycopg2": {
-            "hashes": [
-                "sha256:77ed80f2668582654623e04fb3d741ecce93effcc39c929d7e02f4a917a538ce",
-                "sha256:98a6e0e9580cd7eb4bd4d20f7c7063d154b2589a2b90c0ce4e3ca6085cde77c6"
-            ],
-            "index": "pypi",
-            "version": "==2.9.1"
-        },
-        "types-requests": {
-            "hashes": [
-                "sha256:b279284e51f668e38ee12d9665e4d789089f532dc2a0be4a1508ca0efd98ba9e",
-                "sha256:ba1d108d512e294b6080c37f6ae7cb2a2abf527560e2b671d1786c1fc46b541a"
-            ],
-            "index": "pypi",
-            "version": "==2.25.11"
-        },
-        "typing-extensions": {
-            "hashes": [
-                "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e",
-                "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7",
-                "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"
-            ],
-            "index": "pypi",
-            "version": "==3.10.0.2"
-        },
-        "virtualenv": {
-            "hashes": [
-                "sha256:4b02e52a624336eece99c96e3ab7111f469c24ba226a53ec474e8e787b365814",
-                "sha256:576d05b46eace16a9c348085f7d0dc8ef28713a2cabaa1cf0aea41e8f12c9218"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==20.10.0"
-        },
-        "virtualenv-clone": {
-            "hashes": [
-                "sha256:418ee935c36152f8f153c79824bb93eaf6f0f7984bae31d3f48f350b9183501a",
-                "sha256:44d5263bceed0bac3e1424d64f798095233b64def1c5689afa43dc3223caf5b0"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==0.5.7"
-        },
-        "yapf": {
-            "hashes": [
-                "sha256:408fb9a2b254c302f49db83c59f9aa0b4b0fd0ec25be3a5c51181327922ff63d",
-                "sha256:e3a234ba8455fe201eaa649cdac872d590089a18b661e39bbac7020978dd9c2e"
-            ],
-            "index": "pypi",
-            "version": "==0.31.0"
-        },
-        "zipp": {
-            "hashes": [
-                "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832",
-                "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.6.0"
-        }
-    }
-}

Pipfile.lock

@@ -0,0 +1 @@
+./test_runner/Pipfile.lock

README.md

@@ -1,12 +1,12 @@
 # Zenith
 
-Zenith is a serverless open source alternative to AWS Aurora Postgres. It separates storage and compute and substitutes PostgreSQL storage layer by redistributing data across a cluster of nodes.
+Zenith substitutes PostgreSQL storage layer and redistributes data across a cluster of nodes
 
 ## Architecture overview
 
-A Zenith installation consists of compute nodes and Zenith storage engine.
+A Zenith installation consists of Compute nodes and Storage engine.
 
-Compute nodes are stateless PostgreSQL nodes, backed by Zenith storage engine.
+Compute nodes are stateless PostgreSQL nodes, backed by zenith storage.
 
 Zenith storage engine consists of two major components:
 - Pageserver. Scalable storage backend for compute nodes.
@@ -32,8 +32,8 @@ libssl-dev clang pkg-config libpq-dev
 
 To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively.
 
-To run the integration tests or Python scripts (not required to use the code), install
-Python (3.7 or higher), and install python3 packages using `pipenv install` in the project directory.
+To run the integration tests (not required to use the code), install
+Python (3.7 or higher), and install python3 packages with `pipenv` using `pipenv install` in the project directory.
 
 2. Build zenith and patched postgres
 ```sh
@@ -129,7 +129,7 @@ INSERT 0 1
 git clone --recursive https://github.com/zenithdb/zenith.git
 make # builds also postgres and installs it to ./tmp_install
 cd test_runner
-pipenv run pytest
+pytest
 ```
 
 ## Documentation

compute_tools/.dockerignore

@@ -1 +0,0 @@
-target

compute_tools/.gitignore

@@ -1 +0,0 @@
-target

compute_tools/Cargo.toml

@@ -1,28 +0,0 @@
-[package]
-name = "compute_tools"
-version = "0.1.0"
-authors = ["Alexey Kondratov <kondratov.aleksey@gmail.com>"]
-edition = "2018"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[workspace]
-# TODO: make it a part of global zenith worksapce
-
-[dependencies]
-libc = "0.2"
-anyhow = "1.0"
-chrono = "0.4"
-clap = "2.33"
-env_logger = "0.8"
-hyper = { version = "0.14", features = ["full"] }
-log = { version = "0.4", features = ["std", "serde"] }
-postgres = "0.19"
-regex = "1"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-tar = "0.4"
-tokio = { version = "1", features = ["full"] }
-
-[profile.release]
-debug = true

compute_tools/Dockerfile

@@ -1,14 +0,0 @@
-# First transient image to build compute_tools binaries
-FROM rust:slim-buster AS rust-build
-
-RUN mkdir /compute_tools
-WORKDIR /compute_tools
-
-COPY . /compute_tools/
-
-RUN cargo build --release
-
-# Final image that only has one binary
-FROM debian:buster-slim
-
-COPY --from=rust-build /compute_tools/target/release/zenith_ctl /usr/local/bin/zenith_ctl

compute_tools/README.md

@@ -1,81 +0,0 @@
-# Compute node tools
-
-Postgres wrapper (`zenith_ctl`) is intended to be run as a Docker entrypoint or as a `systemd`
-`ExecStart` option. It will handle all the `zenith` specifics during compute node
-initialization:
-- `zenith_ctl` accepts cluster (compute node) specification as a JSON file.
-- Every start is a fresh start, so the data directory is removed and
-  initialized again on each run.
-- Next it will put configuration files into the `PGDATA` directory.
-- Sync safekeepers and get commit LSN.
-- Get `basebackup` from pageserver using the returned on the previous step LSN.
-- Try to start `postgres` and wait until it is ready to accept connections.
-- Check and alter/drop/create roles and databases.
-- Hang waiting on the `postmaster` process to exit.
-
-Also `zenith_ctl` spawns two separate service threads:
-- `compute-monitor` checks the last Postgres activity timestamp and saves it
-  into the shared `ComputeState`;
-- `http-endpoint` runs a Hyper HTTP API server, which serves readiness and the
-  last activity requests.
-
-Usage example:
-```sh
-zenith_ctl -D /var/db/postgres/compute \
-           -C 'postgresql://zenith_admin@localhost/postgres' \
-           -S /var/db/postgres/specs/current.json \
-           -b /usr/local/bin/postgres
-```
-
-## Tests
-
-Cargo formatter:
-```sh
-cargo fmt
-```
-
-Run tests:
-```sh
-cargo test
-```
-
-Clippy linter:
-```sh
-cargo clippy --all --all-targets -- -Dwarnings -Drust-2018-idioms
-```
-
-## Cross-platform compilation
-
-Imaging that you are on macOS (x86) and you want a Linux GNU (`x86_64-unknown-linux-gnu` platform in `rust` terminology) executable.
-
-### Using docker
-
-You can use a throw-away Docker container ([rustlang/rust](https://hub.docker.com/r/rustlang/rust/) image) for doing that:
-```sh
-docker run --rm \
-    -v $(pwd):/compute_tools \
-    -w /compute_tools \
-    -t rustlang/rust:nightly cargo build --release --target=x86_64-unknown-linux-gnu
-```
-or one-line:
-```sh
-docker run --rm -v $(pwd):/compute_tools -w /compute_tools -t rust:latest cargo build --release --target=x86_64-unknown-linux-gnu
-```
-
-### Using rust native cross-compilation
-
-Another way is to add `x86_64-unknown-linux-gnu` target on your host system:
-```sh
-rustup target add x86_64-unknown-linux-gnu
-```
-
-Install macOS cross-compiler toolchain:
-```sh
-brew tap SergioBenitez/osxct
-brew install x86_64-unknown-linux-gnu
-```
-
-And finally run `cargo build`:
-```sh
-CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-unknown-linux-gnu-gcc cargo build --target=x86_64-unknown-linux-gnu --release
-```

compute_tools/rustfmt.toml

@@ -1 +0,0 @@
-max_width = 100

compute_tools/src/bin/zenith_ctl.rs

@@ -1,251 +0,0 @@
-//!
-//! Postgres wrapper (`zenith_ctl`) is intended to be run as a Docker entrypoint or as a `systemd`
-//! `ExecStart` option. It will handle all the `zenith` specifics during compute node
-//! initialization:
-//! - `zenith_ctl` accepts cluster (compute node) specification as a JSON file.
-//! - Every start is a fresh start, so the data directory is removed and
-//!   initialized again on each run.
-//! - Next it will put configuration files into the `PGDATA` directory.
-//! - Sync safekeepers and get commit LSN.
-//! - Get `basebackup` from pageserver using the returned on the previous step LSN.
-//! - Try to start `postgres` and wait until it is ready to accept connections.
-//! - Check and alter/drop/create roles and databases.
-//! - Hang waiting on the `postmaster` process to exit.
-//!
-//! Also `zenith_ctl` spawns two separate service threads:
-//! - `compute-monitor` checks the last Postgres activity timestamp and saves it
-//!   into the shared `ComputeState`;
-//! - `http-endpoint` runs a Hyper HTTP API server, which serves readiness and the
-//!   last activity requests.
-//!
-//! Usage example:
-//! ```sh
-//! zenith_ctl -D /var/db/postgres/compute \
-//!            -C 'postgresql://zenith_admin@localhost/postgres' \
-//!            -S /var/db/postgres/specs/current.json \
-//!            -b /usr/local/bin/postgres
-//! ```
-//!
-use std::fs::File;
-use std::path::Path;
-use std::process::{exit, Command, ExitStatus};
-use std::sync::{Arc, RwLock};
-use std::{env, panic};
-
-use anyhow::Result;
-use chrono::Utc;
-use libc::{prctl, PR_SET_PDEATHSIG, SIGINT};
-use log::info;
-use postgres::{Client, NoTls};
-
-use compute_tools::config;
-use compute_tools::http_api::launch_http_server;
-use compute_tools::logger::*;
-use compute_tools::monitor::launch_monitor;
-use compute_tools::params::*;
-use compute_tools::pg_helpers::*;
-use compute_tools::spec::*;
-use compute_tools::zenith::*;
-
-/// Do all the preparations like PGDATA directory creation, configuration,
-/// safekeepers sync, basebackup, etc.
-fn prepare_pgdata(state: &Arc<RwLock<ComputeState>>) -> Result<()> {
-    let state = state.read().unwrap();
-    let spec = &state.spec;
-    let pgdata_path = Path::new(&state.pgdata);
-    let pageserver_connstr = spec
-        .cluster
-        .settings
-        .find("zenith.page_server_connstring")
-        .expect("pageserver connstr should be provided");
-    let tenant = spec
-        .cluster
-        .settings
-        .find("zenith.zenith_tenant")
-        .expect("tenant id should be provided");
-    let timeline = spec
-        .cluster
-        .settings
-        .find("zenith.zenith_timeline")
-        .expect("tenant id should be provided");
-
-    info!(
-        "applying spec for cluster #{}, operation #{}",
-        spec.cluster.cluster_id,
-        spec.operation_uuid.as_ref().unwrap()
-    );
-
-    // Remove/create an empty pgdata directory and put configuration there.
-    create_pgdata(&state.pgdata)?;
-    config::write_postgres_conf(&pgdata_path.join("postgresql.conf"), spec)?;
-
-    info!("starting safekeepers syncing");
-    let lsn = sync_safekeepers(&state.pgdata, &state.pgbin)?;
-    info!("safekeepers synced at LSN {}", lsn);
-
-    get_basebackup(&state.pgdata, &pageserver_connstr, &tenant, &timeline, &lsn)?;
-    // Update pg_hba.conf received with basebackup.
-    update_pg_hba(pgdata_path)?;
-
-    Ok(())
-}
-
-/// Start Postgres as a child process and manage DBs/roles.
-/// After that this will hang waiting on the postmaster process to exit.
-fn run_compute(state: &Arc<RwLock<ComputeState>>) -> Result<ExitStatus> {
-    let read_state = state.read().unwrap();
-    let pgdata_path = Path::new(&read_state.pgdata);
-
-    // Run postgres as a child process.
-    let mut pg = Command::new(&read_state.pgbin)
-        .args(&["-D", &read_state.pgdata])
-        .spawn()
-        .expect("cannot start postgres process");
-
-    // Try default Postgres port if it is not provided
-    let port = read_state
-        .spec
-        .cluster
-        .settings
-        .find("port")
-        .unwrap_or_else(|| "5432".to_string());
-    wait_for_postgres(&port, pgdata_path)?;
-
-    let mut client = Client::connect(&read_state.connstr, NoTls)?;
-
-    handle_roles(&read_state.spec, &mut client)?;
-    handle_databases(&read_state.spec, &mut client)?;
-
-    // 'Close' connection
-    drop(client);
-
-    info!(
-        "finished configuration of cluster #{}",
-        read_state.spec.cluster.cluster_id
-    );
-
-    // Release the read lock.
-    drop(read_state);
-
-    // Get the write lock, update state and release the lock, so HTTP API
-    // was able to serve requests, while we are blocked waiting on
-    // Postgres.
-    let mut state = state.write().unwrap();
-    state.ready = true;
-    drop(state);
-
-    // Wait for child postgres process basically forever. In this state Ctrl+C
-    // will be propagated to postgres and it will be shut down as well.
-    let ecode = pg.wait().expect("failed to wait on postgres");
-
-    Ok(ecode)
-}
-
-fn main() -> Result<()> {
-    // During configuration we are starting Postgres as a child process. If we
-    // fail we do not want to leave it running. PR_SET_PDEATHSIG sets the signal
-    // that will be sent to the child process when the parent dies. NB: this is
-    // cleared for the child of a fork(). SIGINT means fast shutdown for Postgres.
-    // This does not matter much for Docker, where `zenith_ctl` is an entrypoint,
-    // so the whole container will exit if it exits. But could be useful when
-    // `zenith_ctl` is used in e.g. systemd.
-    unsafe {
-        prctl(PR_SET_PDEATHSIG, SIGINT);
-    }
-
-    // TODO: re-use `zenith_utils::logging` later
-    init_logger(DEFAULT_LOG_LEVEL)?;
-
-    let matches = clap::App::new("zenith_ctl")
-        .version("0.1.0")
-        .arg(
-            clap::Arg::with_name("connstr")
-                .short("C")
-                .long("connstr")
-                .value_name("DATABASE_URL")
-                .required(true),
-        )
-        .arg(
-            clap::Arg::with_name("pgdata")
-                .short("D")
-                .long("pgdata")
-                .value_name("DATADIR")
-                .required(true),
-        )
-        .arg(
-            clap::Arg::with_name("pgbin")
-                .short("b")
-                .long("pgbin")
-                .value_name("POSTGRES_PATH"),
-        )
-        .arg(
-            clap::Arg::with_name("spec")
-                .short("s")
-                .long("spec")
-                .value_name("SPEC_JSON"),
-        )
-        .arg(
-            clap::Arg::with_name("spec-path")
-                .short("S")
-                .long("spec-path")
-                .value_name("SPEC_PATH"),
-        )
-        .get_matches();
-
-    let pgdata = matches.value_of("pgdata").expect("PGDATA path is required");
-    let connstr = matches
-        .value_of("connstr")
-        .expect("Postgres connection string is required");
-    let spec = matches.value_of("spec");
-    let spec_path = matches.value_of("spec-path");
-
-    // Try to use just 'postgres' if no path is provided
-    let pgbin = matches.value_of("pgbin").unwrap_or("postgres");
-
-    let spec: ClusterSpec = match spec {
-        // First, try to get cluster spec from the cli argument
-        Some(json) => serde_json::from_str(json)?,
-        None => {
-            // Second, try to read it from the file if path is provided
-            if let Some(sp) = spec_path {
-                let path = Path::new(sp);
-                let file = File::open(path)?;
-                serde_json::from_reader(file)?
-            } else {
-                // Finally, try to fetch it from the env
-                // XXX: not tested well and kept as a backup option for k8s, Docker, etc.
-                // TODO: remove later
-                match env::var("CLUSTER_SPEC") {
-                    Ok(json) => serde_json::from_str(&json)?,
-                    Err(_) => panic!("cluster spec should be provided via --spec, --spec-path or env variable CLUSTER_SPEC")
-                }
-            }
-        }
-    };
-
-    let compute_state = ComputeState {
-        connstr: connstr.to_string(),
-        pgdata: pgdata.to_string(),
-        pgbin: pgbin.to_string(),
-        spec,
-        ready: false,
-        last_active: Utc::now(),
-    };
-    let compute_state = Arc::new(RwLock::new(compute_state));
-
-    // Launch service threads first, so we were able to serve availability
-    // requests, while configuration is still in progress.
-    let mut _threads = vec![
-        launch_http_server(&compute_state).expect("cannot launch compute monitor thread"),
-        launch_monitor(&compute_state).expect("cannot launch http endpoint thread"),
-    ];
-
-    prepare_pgdata(&compute_state)?;
-
-    // Run compute (Postgres) and hang waiting on it. Panic if any error happens,
-    // it will help us to trigger unwind and kill postmaster as well.
-    match run_compute(&compute_state) {
-        Ok(ec) => exit(ec.success() as i32),
-        Err(error) => panic!("cannot start compute node, error: {}", error),
-    }
-}

compute_tools/src/config.rs

@@ -1,51 +0,0 @@
-use std::fs::{File, OpenOptions};
-use std::io;
-use std::io::prelude::*;
-use std::path::Path;
-
-use anyhow::Result;
-
-use crate::pg_helpers::PgOptionsSerialize;
-use crate::zenith::ClusterSpec;
-
-/// Check that `line` is inside a text file and put it there if it is not.
-/// Create file if it doesn't exist.
-pub fn line_in_file(path: &Path, line: &str) -> Result<bool> {
-    let mut file = OpenOptions::new()
-        .read(true)
-        .write(true)
-        .create(true)
-        .append(false)
-        .open(path)?;
-    let buf = io::BufReader::new(&file);
-    let mut count: usize = 0;
-
-    for l in buf.lines() {
-        if l? == line {
-            return Ok(false);
-        }
-        count = 1;
-    }
-
-    write!(file, "{}{}", "\n".repeat(count), line)?;
-    Ok(true)
-}
-
-/// Create or completely rewrite configuration file specified by `path`
-pub fn write_postgres_conf(path: &Path, spec: &ClusterSpec) -> Result<()> {
-    // File::create() destroys the file content if it exists.
-    let mut postgres_conf = File::create(path)?;
-
-    write_zenith_managed_block(&mut postgres_conf, &spec.cluster.settings.as_pg_settings())?;
-
-    Ok(())
-}
-
-// Write Postgres config block wrapped with generated comment section
-fn write_zenith_managed_block(file: &mut File, buf: &str) -> Result<()> {
-    writeln!(file, "# Managed by Zenith: begin")?;
-    writeln!(file, "{}", buf)?;
-    writeln!(file, "# Managed by Zenith: end")?;
-
-    Ok(())
-}

compute_tools/src/http_api.rs

@@ -1,73 +0,0 @@
-use std::convert::Infallible;
-use std::net::SocketAddr;
-use std::sync::{Arc, RwLock};
-use std::thread;
-
-use anyhow::Result;
-use hyper::service::{make_service_fn, service_fn};
-use hyper::{Body, Method, Request, Response, Server, StatusCode};
-use log::{error, info};
-
-use crate::zenith::*;
-
-// Service function to handle all available routes.
-fn routes(req: Request<Body>, state: Arc<RwLock<ComputeState>>) -> Response<Body> {
-    match (req.method(), req.uri().path()) {
-        // Timestamp of the last Postgres activity in the plain text.
-        (&Method::GET, "/last_activity") => {
-            info!("serving /last_active GET request");
-            let state = state.read().unwrap();
-
-            // Use RFC3339 format for consistency.
-            Response::new(Body::from(state.last_active.to_rfc3339()))
-        }
-
-        // Has compute setup process finished? -> true/false
-        (&Method::GET, "/ready") => {
-            info!("serving /ready GET request");
-            let state = state.read().unwrap();
-            Response::new(Body::from(format!("{}", state.ready)))
-        }
-
-        // Return the `404 Not Found` for any other routes.
-        _ => {
-            let mut not_found = Response::new(Body::from("404 Not Found"));
-            *not_found.status_mut() = StatusCode::NOT_FOUND;
-            not_found
-        }
-    }
-}
-
-// Main Hyper HTTP server function that runs it and blocks waiting on it forever.
-#[tokio::main]
-async fn serve(state: Arc<RwLock<ComputeState>>) {
-    let addr = SocketAddr::from(([0, 0, 0, 0], 3080));
-
-    let make_service = make_service_fn(move |_conn| {
-        let state = state.clone();
-        async move {
-            Ok::<_, Infallible>(service_fn(move |req: Request<Body>| {
-                let state = state.clone();
-                async move { Ok::<_, Infallible>(routes(req, state)) }
-            }))
-        }
-    });
-
-    info!("starting HTTP server on {}", addr);
-
-    let server = Server::bind(&addr).serve(make_service);
-
-    // Run this server forever
-    if let Err(e) = server.await {
-        error!("server error: {}", e);
-    }
-}
-
-/// Launch a separate Hyper HTTP API server thread and return its `JoinHandle`.
-pub fn launch_http_server(state: &Arc<RwLock<ComputeState>>) -> Result<thread::JoinHandle<()>> {
-    let state = Arc::clone(state);
-
-    Ok(thread::Builder::new()
-        .name("http-endpoint".into())
-        .spawn(move || serve(state))?)
-}

compute_tools/src/lib.rs

@@ -1,13 +0,0 @@
-//!
-//! Various tools and helpers to handle cluster / compute node (Postgres)
-//! configuration.
-//!
-pub mod config;
-pub mod http_api;
-#[macro_use]
-pub mod logger;
-pub mod monitor;
-pub mod params;
-pub mod pg_helpers;
-pub mod spec;
-pub mod zenith;

compute_tools/src/logger.rs

@@ -1,43 +0,0 @@
-use std::io::Write;
-
-use anyhow::Result;
-use chrono::Utc;
-use env_logger::{Builder, Env};
-
-macro_rules! info_println {
-    ($($tts:tt)*) => {
-        if log_enabled!(Level::Info) {
-            println!($($tts)*);
-        }
-    }
-}
-
-macro_rules! info_print {
-    ($($tts:tt)*) => {
-        if log_enabled!(Level::Info) {
-            print!($($tts)*);
-        }
-    }
-}
-
-/// Initialize `env_logger` using either `default_level` or
-/// `RUST_LOG` environment variable as default log level.
-pub fn init_logger(default_level: &str) -> Result<()> {
-    let env = Env::default().filter_or("RUST_LOG", default_level);
-
-    Builder::from_env(env)
-        .format(|buf, record| {
-            let thread_handle = std::thread::current();
-            writeln!(
-                buf,
-                "{} [{}] {}: {}",
-                Utc::now().format("%Y-%m-%d %H:%M:%S%.3f %Z"),
-                thread_handle.name().unwrap_or("main"),
-                record.level(),
-                record.args()
-            )
-        })
-        .init();
-
-    Ok(())
-}

compute_tools/src/monitor.rs

@@ -1,109 +0,0 @@
-use std::sync::{Arc, RwLock};
-use std::{thread, time};
-
-use anyhow::Result;
-use chrono::{DateTime, Utc};
-use log::{debug, info};
-use postgres::{Client, NoTls};
-
-use crate::zenith::ComputeState;
-
-const MONITOR_CHECK_INTERVAL: u64 = 500; // milliseconds
-
-// Spin in a loop and figure out the last activity time in the Postgres.
-// Then update it in the shared state. This function never errors out.
-// XXX: the only expected panic is at `RwLock` unwrap().
-fn watch_compute_activity(state: &Arc<RwLock<ComputeState>>) {
-    // Suppose that `connstr` doesn't change
-    let connstr = state.read().unwrap().connstr.clone();
-    // Define `client` outside of the loop to reuse existing connection if it's active.
-    let mut client = Client::connect(&connstr, NoTls);
-    let timeout = time::Duration::from_millis(MONITOR_CHECK_INTERVAL);
-
-    info!("watching Postgres activity at {}", connstr);
-
-    loop {
-        // Should be outside of the write lock to allow others to read while we sleep.
-        thread::sleep(timeout);
-
-        match &mut client {
-            Ok(cli) => {
-                if cli.is_closed() {
-                    info!("connection to postgres closed, trying to reconnect");
-
-                    // Connection is closed, reconnect and try again.
-                    client = Client::connect(&connstr, NoTls);
-                    continue;
-                }
-
-                // Get all running client backends except ourself, use RFC3339 DateTime format.
-                let backends = cli
-                    .query(
-                        "SELECT state, to_char(state_change, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS state_change
-                         FROM pg_stat_activity
-                         WHERE backend_type = 'client backend'
-                            AND pid != pg_backend_pid()
-                            AND usename != 'zenith_admin';", // XXX: find a better way to filter other monitors?
-                        &[],
-                    );
-                let mut last_active = state.read().unwrap().last_active;
-
-                if let Ok(backs) = backends {
-                    let mut idle_backs: Vec<DateTime<Utc>> = vec![];
-
-                    for b in backs.into_iter() {
-                        let state: String = b.get("state");
-                        let change: String = b.get("state_change");
-
-                        if state == "idle" {
-                            let change = DateTime::parse_from_rfc3339(&change);
-                            match change {
-                                Ok(t) => idle_backs.push(t.with_timezone(&Utc)),
-                                Err(e) => {
-                                    info!("cannot parse backend state_change DateTime: {}", e);
-                                    continue;
-                                }
-                            }
-                        } else {
-                            // Found non-idle backend, so the last activity is NOW.
-                            // Save it and exit the for loop. Also clear the idle backend
-                            // `state_change` timestamps array as it doesn't matter now.
-                            last_active = Utc::now();
-                            idle_backs.clear();
-                            break;
-                        }
-                    }
-
-                    // Sort idle backend `state_change` timestamps. The last one corresponds
-                    // to the last activity.
-                    idle_backs.sort();
-                    if let Some(last) = idle_backs.last() {
-                        last_active = *last;
-                    }
-                }
-
-                // Update the last activity in the shared state if we got a more recent one.
-                let mut state = state.write().unwrap();
-                if last_active > state.last_active {
-                    state.last_active = last_active;
-                    debug!("set the last compute activity time to: {}", last_active);
-                }
-            }
-            Err(e) => {
-                info!("cannot connect to postgres: {}, retrying", e);
-
-                // Establish a new connection and try again.
-                client = Client::connect(&connstr, NoTls);
-            }
-        }
-    }
-}
-
-/// Launch a separate compute monitor thread and return its `JoinHandle`.
-pub fn launch_monitor(state: &Arc<RwLock<ComputeState>>) -> Result<thread::JoinHandle<()>> {
-    let state = Arc::clone(state);
-
-    Ok(thread::Builder::new()
-        .name("compute-monitor".into())
-        .spawn(move || watch_compute_activity(&state))?)
-}

compute_tools/src/params.rs

@@ -1,3 +0,0 @@
-pub const DEFAULT_LOG_LEVEL: &str = "info";
-pub const DEFAULT_CONNSTRING: &str = "host=localhost user=postgres";
-pub const PG_HBA_ALL_MD5: &str = "host\tall\t\tall\t\t0.0.0.0/0\t\tmd5";

compute_tools/src/pg_helpers.rs

@@ -1,264 +0,0 @@
-use std::net::{SocketAddr, TcpStream};
-use std::os::unix::fs::PermissionsExt;
-use std::path::Path;
-use std::process::Command;
-use std::str::FromStr;
-use std::{fs, thread, time};
-
-use anyhow::{anyhow, Result};
-use postgres::{Client, Transaction};
-use serde::Deserialize;
-
-const POSTGRES_WAIT_TIMEOUT: u64 = 60 * 1000; // milliseconds
-
-/// Rust representation of Postgres role info with only those fields
-/// that matter for us.
-#[derive(Clone, Deserialize)]
-pub struct Role {
-    pub name: PgIdent,
-    pub encrypted_password: Option<String>,
-    pub options: GenericOptions,
-}
-
-/// Rust representation of Postgres database info with only those fields
-/// that matter for us.
-#[derive(Clone, Deserialize)]
-pub struct Database {
-    pub name: PgIdent,
-    pub owner: PgIdent,
-    pub options: GenericOptions,
-}
-
-/// Common type representing both SQL statement params with or without value,
-/// like `LOGIN` or `OWNER username` in the `CREATE/ALTER ROLE`, and config
-/// options like `wal_level = logical`.
-#[derive(Clone, Deserialize)]
-pub struct GenericOption {
-    pub name: String,
-    pub value: Option<String>,
-    pub vartype: String,
-}
-
-/// Optional collection of `GenericOption`'s. Type alias allows us to
-/// declare a `trait` on it.
-pub type GenericOptions = Option<Vec<GenericOption>>;
-
-impl GenericOption {
-    /// Represent `GenericOption` as SQL statement parameter.
-    pub fn to_pg_option(&self) -> String {
-        if let Some(val) = &self.value {
-            match self.vartype.as_ref() {
-                "string" => format!("{} '{}'", self.name, val),
-                _ => format!("{} {}", self.name, val),
-            }
-        } else {
-            self.name.to_owned()
-        }
-    }
-
-    /// Represent `GenericOption` as configuration option.
-    pub fn to_pg_setting(&self) -> String {
-        if let Some(val) = &self.value {
-            match self.vartype.as_ref() {
-                "string" => format!("{} = '{}'", self.name, val),
-                _ => format!("{} = {}", self.name, val),
-            }
-        } else {
-            self.name.to_owned()
-        }
-    }
-}
-
-pub trait PgOptionsSerialize {
-    fn as_pg_options(&self) -> String;
-    fn as_pg_settings(&self) -> String;
-}
-
-impl PgOptionsSerialize for GenericOptions {
-    /// Serialize an optional collection of `GenericOption`'s to
-    /// Postgres SQL statement arguments.
-    fn as_pg_options(&self) -> String {
-        if let Some(ops) = &self {
-            ops.iter()
-                .map(|op| op.to_pg_option())
-                .collect::<Vec<String>>()
-                .join(" ")
-        } else {
-            "".to_string()
-        }
-    }
-
-    /// Serialize an optional collection of `GenericOption`'s to
-    /// `postgresql.conf` compatible format.
-    fn as_pg_settings(&self) -> String {
-        if let Some(ops) = &self {
-            ops.iter()
-                .map(|op| op.to_pg_setting())
-                .collect::<Vec<String>>()
-                .join("\n")
-        } else {
-            "".to_string()
-        }
-    }
-}
-
-pub trait GenericOptionsSearch {
-    fn find(&self, name: &str) -> Option<String>;
-}
-
-impl GenericOptionsSearch for GenericOptions {
-    /// Lookup option by name
-    fn find(&self, name: &str) -> Option<String> {
-        match &self {
-            Some(ops) => {
-                let op = ops.iter().find(|s| s.name == name);
-                match op {
-                    Some(op) => op.value.clone(),
-                    None => None,
-                }
-            }
-            None => None,
-        }
-    }
-}
-
-impl Role {
-    /// Serialize a list of role parameters into a Postgres-acceptable
-    /// string of arguments.
-    pub fn to_pg_options(&self) -> String {
-        // XXX: consider putting LOGIN as a default option somewhere higher, e.g. in Rails.
-        // For now we do not use generic `options` for roles. Once used, add
-        // `self.options.as_pg_options()` somewhere here.
-        let mut params: String = "LOGIN".to_string();
-
-        if let Some(pass) = &self.encrypted_password {
-            params.push_str(&format!(" PASSWORD 'md5{}'", pass));
-        } else {
-            params.push_str(" PASSWORD NULL");
-        }
-
-        params
-    }
-}
-
-impl Database {
-    /// Serialize a list of database parameters into a Postgres-acceptable
-    /// string of arguments.
-    /// NB: `TEMPLATE` is actually also an identifier, but so far we only need
-    /// to use `template0` and `template1`, so it is not a problem. Yet in the future
-    /// it may require a proper quoting too.
-    pub fn to_pg_options(&self) -> String {
-        let mut params: String = self.options.as_pg_options();
-        params.push_str(&format!(" OWNER {}", &self.owner.quote()));
-
-        params
-    }
-}
-
-/// String type alias representing Postgres identifier and
-/// intended to be used for DB / role names.
-pub type PgIdent = String;
-
-/// Generic trait used to provide quoting for strings used in the
-/// Postgres SQL queries. Currently used only to implement quoting
-/// of identifiers, but could be used for literals in the future.
-pub trait PgQuote {
-    fn quote(&self) -> String;
-}
-
-impl PgQuote for PgIdent {
-    /// This is intended to mimic Postgres quote_ident(), but for simplicity it
-    /// always quotes provided string with `""` and escapes every `"`. Not idempotent,
-    /// i.e. if string is already escaped it will be escaped again.
-    fn quote(&self) -> String {
-        let result = format!("\"{}\"", self.replace("\"", "\"\""));
-        result
-    }
-}
-
-/// Build a list of existing Postgres roles
-pub fn get_existing_roles(xact: &mut Transaction<'_>) -> Result<Vec<Role>> {
-    let postgres_roles = xact
-        .query("SELECT rolname, rolpassword FROM pg_catalog.pg_authid", &[])?
-        .iter()
-        .map(|row| Role {
-            name: row.get("rolname"),
-            encrypted_password: row.get("rolpassword"),
-            options: None,
-        })
-        .collect();
-
-    Ok(postgres_roles)
-}
-
-/// Build a list of existing Postgres databases
-pub fn get_existing_dbs(client: &mut Client) -> Result<Vec<Database>> {
-    let postgres_dbs = client
-        .query(
-            "SELECT datname, datdba::regrole::text as owner
-               FROM pg_catalog.pg_database;",
-            &[],
-        )?
-        .iter()
-        .map(|row| Database {
-            name: row.get("datname"),
-            owner: row.get("owner"),
-            options: None,
-        })
-        .collect();
-
-    Ok(postgres_dbs)
-}
-
-/// Wait for Postgres to become ready to accept connections:
-/// - state should be `ready` in the `pgdata/postmaster.pid`
-/// - and we should be able to connect to 127.0.0.1:5432
-pub fn wait_for_postgres(port: &str, pgdata: &Path) -> Result<()> {
-    let pid_path = pgdata.join("postmaster.pid");
-    let mut slept: u64 = 0; // ms
-    let pause = time::Duration::from_millis(100);
-
-    let timeout = time::Duration::from_millis(200);
-    let addr = SocketAddr::from_str(&format!("127.0.0.1:{}", port)).unwrap();
-
-    loop {
-        // Sleep POSTGRES_WAIT_TIMEOUT at max (a bit longer actually if consider a TCP timeout,
-        // but postgres starts listening almost immediately, even if it is not really
-        // ready to accept connections).
-        if slept >= POSTGRES_WAIT_TIMEOUT {
-            return Err(anyhow!("timed out while waiting for Postgres to start"));
-        }
-
-        if pid_path.exists() {
-            // XXX: dumb and the simplest way to get the last line in a text file
-            // TODO: better use `.lines().last()` later
-            let stdout = Command::new("tail")
-                .args(&["-n1", pid_path.to_str().unwrap()])
-                .output()?
-                .stdout;
-            let status = String::from_utf8(stdout)?;
-            let can_connect = TcpStream::connect_timeout(&addr, timeout).is_ok();
-
-            // Now Postgres is ready to accept connections
-            if status.trim() == "ready" && can_connect {
-                break;
-            }
-        }
-
-        thread::sleep(pause);
-        slept += 100;
-    }
-
-    Ok(())
-}
-
-/// Remove `pgdata` directory and create it again with right permissions.
-pub fn create_pgdata(pgdata: &str) -> Result<()> {
-    // Ignore removal error, likely it is a 'No such file or directory (os error 2)'.
-    // If it is something different then create_dir() will error out anyway.
-    let _ok = fs::remove_dir_all(pgdata);
-    fs::create_dir(pgdata)?;
-    fs::set_permissions(pgdata, fs::Permissions::from_mode(0o700))?;
-
-    Ok(())
-}

compute_tools/src/spec.rs

@@ -1,246 +0,0 @@
-use std::path::Path;
-
-use anyhow::Result;
-use log::{info, log_enabled, warn, Level};
-use postgres::Client;
-
-use crate::config;
-use crate::params::PG_HBA_ALL_MD5;
-use crate::pg_helpers::*;
-use crate::zenith::ClusterSpec;
-
-/// It takes cluster specification and does the following:
-/// - Serialize cluster config and put it into `postgresql.conf` completely rewriting the file.
-/// - Update `pg_hba.conf` to allow external connections.
-pub fn handle_configuration(spec: &ClusterSpec, pgdata_path: &Path) -> Result<()> {
-    // File `postgresql.conf` is no longer included into `basebackup`, so just
-    // always write all config into it creating new file.
-    config::write_postgres_conf(&pgdata_path.join("postgresql.conf"), spec)?;
-
-    update_pg_hba(pgdata_path)?;
-
-    Ok(())
-}
-
-/// Check `pg_hba.conf` and update if needed to allow external connections.
-pub fn update_pg_hba(pgdata_path: &Path) -> Result<()> {
-    // XXX: consider making it a part of spec.json
-    info!("checking pg_hba.conf");
-    let pghba_path = pgdata_path.join("pg_hba.conf");
-
-    if config::line_in_file(&pghba_path, PG_HBA_ALL_MD5)? {
-        info!("updated pg_hba.conf to allow external connections");
-    } else {
-        info!("pg_hba.conf is up-to-date");
-    }
-
-    Ok(())
-}
-
-/// Given a cluster spec json and open transaction it handles roles creation,
-/// deletion and update.
-pub fn handle_roles(spec: &ClusterSpec, client: &mut Client) -> Result<()> {
-    let mut xact = client.transaction()?;
-    let existing_roles: Vec<Role> = get_existing_roles(&mut xact)?;
-
-    // Print a list of existing Postgres roles (only in debug mode)
-    info!("postgres roles:");
-    for r in &existing_roles {
-        info_println!(
-            "{} - {}:{}",
-            " ".repeat(27 + 5),
-            r.name,
-            if r.encrypted_password.is_some() {
-                "[FILTERED]"
-            } else {
-                "(null)"
-            }
-        );
-    }
-
-    // Process delta operations first
-    if let Some(ops) = &spec.delta_operations {
-        info!("processing delta operations on roles");
-        for op in ops {
-            match op.action.as_ref() {
-                // We do not check either role exists or not,
-                // Postgres will take care of it for us
-                "delete_role" => {
-                    let query: String = format!("DROP ROLE IF EXISTS {}", &op.name.quote());
-
-                    warn!("deleting role '{}'", &op.name);
-                    xact.execute(query.as_str(), &[])?;
-                }
-                // Renaming role drops its password, since tole name is
-                // used as a salt there.  It is important that this role
-                // is recorded with a new `name` in the `roles` list.
-                // Follow up roles update will set the new password.
-                "rename_role" => {
-                    let new_name = op.new_name.as_ref().unwrap();
-
-                    // XXX: with a limited number of roles it is fine, but consider making it a HashMap
-                    if existing_roles.iter().any(|r| r.name == op.name) {
-                        let query: String = format!(
-                            "ALTER ROLE {} RENAME TO {}",
-                            op.name.quote(),
-                            new_name.quote()
-                        );
-
-                        warn!("renaming role '{}' to '{}'", op.name, new_name);
-                        xact.execute(query.as_str(), &[])?;
-                    }
-                }
-                _ => {}
-            }
-        }
-    }
-
-    // Refresh Postgres roles info to handle possible roles renaming
-    let existing_roles: Vec<Role> = get_existing_roles(&mut xact)?;
-
-    info!("cluster spec roles:");
-    for role in &spec.cluster.roles {
-        let name = &role.name;
-
-        info_print!(
-            "{} - {}:{}",
-            " ".repeat(27 + 5),
-            name,
-            if role.encrypted_password.is_some() {
-                "[FILTERED]"
-            } else {
-                "(null)"
-            }
-        );
-
-        // XXX: with a limited number of roles it is fine, but consider making it a HashMap
-        let pg_role = existing_roles.iter().find(|r| r.name == *name);
-
-        if let Some(r) = pg_role {
-            let mut update_role = false;
-
-            if (r.encrypted_password.is_none() && role.encrypted_password.is_some())
-                || (r.encrypted_password.is_some() && role.encrypted_password.is_none())
-            {
-                update_role = true;
-            } else if let Some(pg_pwd) = &r.encrypted_password {
-                // Check whether password changed or not (trim 'md5:' prefix first)
-                update_role = pg_pwd[3..] != *role.encrypted_password.as_ref().unwrap();
-            }
-
-            if update_role {
-                let mut query: String = format!("ALTER ROLE {} ", name.quote());
-                info_print!(" -> update");
-
-                query.push_str(&role.to_pg_options());
-                xact.execute(query.as_str(), &[])?;
-            }
-        } else {
-            info!("role name {}", &name);
-            let mut query: String = format!("CREATE ROLE {} ", name.quote());
-            info!("role create query {}", &query);
-            info_print!(" -> create");
-
-            query.push_str(&role.to_pg_options());
-            xact.execute(query.as_str(), &[])?;
-        }
-
-        info_print!("\n");
-    }
-
-    xact.commit()?;
-
-    Ok(())
-}
-
-/// It follows mostly the same logic as `handle_roles()` excepting that we
-/// does not use an explicit transactions block, since major database operations
-/// like `CREATE DATABASE` and `DROP DATABASE` do not support it. Statement-level
-/// atomicity should be enough here due to the order of operations and various checks,
-/// which together provide us idempotency.
-pub fn handle_databases(spec: &ClusterSpec, client: &mut Client) -> Result<()> {
-    let existing_dbs: Vec<Database> = get_existing_dbs(client)?;
-
-    // Print a list of existing Postgres databases (only in debug mode)
-    info!("postgres databases:");
-    for r in &existing_dbs {
-        info_println!("{} - {}:{}", " ".repeat(27 + 5), r.name, r.owner);
-    }
-
-    // Process delta operations first
-    if let Some(ops) = &spec.delta_operations {
-        info!("processing delta operations on databases");
-        for op in ops {
-            match op.action.as_ref() {
-                // We do not check either DB exists or not,
-                // Postgres will take care of it for us
-                "delete_db" => {
-                    let query: String = format!("DROP DATABASE IF EXISTS {}", &op.name.quote());
-
-                    warn!("deleting database '{}'", &op.name);
-                    client.execute(query.as_str(), &[])?;
-                }
-                "rename_db" => {
-                    let new_name = op.new_name.as_ref().unwrap();
-
-                    // XXX: with a limited number of roles it is fine, but consider making it a HashMap
-                    if existing_dbs.iter().any(|r| r.name == op.name) {
-                        let query: String = format!(
-                            "ALTER DATABASE {} RENAME TO {}",
-                            op.name.quote(),
-                            new_name.quote()
-                        );
-
-                        warn!("renaming database '{}' to '{}'", op.name, new_name);
-                        client.execute(query.as_str(), &[])?;
-                    }
-                }
-                _ => {}
-            }
-        }
-    }
-
-    // Refresh Postgres databases info to handle possible renames
-    let existing_dbs: Vec<Database> = get_existing_dbs(client)?;
-
-    info!("cluster spec databases:");
-    for db in &spec.cluster.databases {
-        let name = &db.name;
-
-        info_print!("{} - {}:{}", " ".repeat(27 + 5), db.name, db.owner);
-
-        // XXX: with a limited number of databases it is fine, but consider making it a HashMap
-        let pg_db = existing_dbs.iter().find(|r| r.name == *name);
-
-        if let Some(r) = pg_db {
-            // XXX: db owner name is returned as quoted string from Postgres,
-            // when quoting is needed.
-            let new_owner = if r.owner.starts_with('\"') {
-                db.owner.quote()
-            } else {
-                db.owner.clone()
-            };
-
-            if new_owner != r.owner {
-                let query: String = format!(
-                    "ALTER DATABASE {} OWNER TO {}",
-                    name.quote(),
-                    db.owner.quote()
-                );
-                info_print!(" -> update");
-
-                client.execute(query.as_str(), &[])?;
-            }
-        } else {
-            let mut query: String = format!("CREATE DATABASE {} ", name.quote());
-            info_print!(" -> create");
-
-            query.push_str(&db.to_pg_options());
-            client.execute(query.as_str(), &[])?;
-        }
-
-        info_print!("\n");
-    }
-
-    Ok(())
-}

compute_tools/src/zenith.rs

@@ -1,107 +0,0 @@
-use std::process::{Command, Stdio};
-
-use anyhow::Result;
-use chrono::{DateTime, Utc};
-use postgres::{Client, NoTls};
-use serde::Deserialize;
-
-use crate::pg_helpers::*;
-
-/// Compute node state shared across several `zenith_ctl` threads.
-/// Should be used under `RwLock` to allow HTTP API server to serve
-/// status requests, while configuration is in progress.
-pub struct ComputeState {
-    pub connstr: String,
-    pub pgdata: String,
-    pub pgbin: String,
-    pub spec: ClusterSpec,
-    /// Compute setup process has finished
-    pub ready: bool,
-    /// Timestamp of the last Postgres activity
-    pub last_active: DateTime<Utc>,
-}
-
-/// Cluster spec or configuration represented as an optional number of
-/// delta operations + final cluster state description.
-#[derive(Clone, Deserialize)]
-pub struct ClusterSpec {
-    pub format_version: f32,
-    pub timestamp: String,
-    pub operation_uuid: Option<String>,
-    /// Expected cluster state at the end of transition process.
-    pub cluster: Cluster,
-    pub delta_operations: Option<Vec<DeltaOp>>,
-}
-
-/// Cluster state seen from the perspective of the external tools
-/// like Rails web console.
-#[derive(Clone, Deserialize)]
-pub struct Cluster {
-    pub cluster_id: String,
-    pub name: String,
-    pub state: Option<String>,
-    pub roles: Vec<Role>,
-    pub databases: Vec<Database>,
-    pub settings: GenericOptions,
-}
-
-/// Single cluster state changing operation that could not be represented as
-/// a static `Cluster` structure. For example:
-/// - DROP DATABASE
-/// - DROP ROLE
-/// - ALTER ROLE name RENAME TO new_name
-/// - ALTER DATABASE name RENAME TO new_name
-#[derive(Clone, Deserialize)]
-pub struct DeltaOp {
-    pub action: String,
-    pub name: PgIdent,
-    pub new_name: Option<PgIdent>,
-}
-
-/// Get basebackup from the libpq connection to pageserver using `connstr` and
-/// unarchive it to `pgdata` directory overriding all its previous content.
-pub fn get_basebackup(
-    pgdata: &str,
-    connstr: &str,
-    tenant: &str,
-    timeline: &str,
-    lsn: &str,
-) -> Result<()> {
-    let mut client = Client::connect(connstr, NoTls)?;
-    let basebackup_cmd = match lsn {
-        "0/0" => format!("basebackup {} {}", tenant, timeline), // First start of the compute
-        _ => format!("basebackup {} {} {}", tenant, timeline, lsn),
-    };
-    let copyreader = client.copy_out(basebackup_cmd.as_str())?;
-    let mut ar = tar::Archive::new(copyreader);
-
-    ar.unpack(&pgdata)?;
-
-    Ok(())
-}
-
-/// Run `postgres` in a special mode with `--sync-safekeepers` argument
-/// and return the reported LSN back to the caller.
-pub fn sync_safekeepers(pgdata: &str, pgbin: &str) -> Result<String> {
-    let sync_handle = Command::new(&pgbin)
-        .args(&["--sync-safekeepers"])
-        .env("PGDATA", &pgdata) // we cannot use -D in this mode
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .spawn()
-        .expect("postgres --sync-safekeepers failed to start");
-
-    let sync_output = sync_handle
-        .wait_with_output()
-        .expect("postgres --sync-safekeepers failed");
-    if !sync_output.status.success() {
-        anyhow::bail!(
-            "postgres --sync-safekeepers failed: '{}'",
-            String::from_utf8_lossy(&sync_output.stderr)
-        );
-    }
-
-    let lsn = String::from(String::from_utf8(sync_output.stdout)?.trim());
-
-    Ok(lsn)
-}

compute_tools/tests/cluster_spec.json

@@ -1,205 +0,0 @@
-{
-    "format_version": 1.0,
-
-    "timestamp": "2021-05-23T18:25:43.511Z",
-    "operation_uuid": "0f657b36-4b0f-4a2d-9c2e-1dcd615e7d8b",
-
-    "cluster": {
-        "cluster_id": "test-cluster-42",
-        "name": "Zenith Test",
-        "state": "restarted",
-        "roles": [
-            {
-                "name": "postgres",
-                "encrypted_password": "6b1d16b78004bbd51fa06af9eda75972",
-                "options": null
-            },
-            {
-                "name": "alexk",
-                "encrypted_password": null,
-                "options": null
-            },
-            {
-                "name": "zenith \"new\"",
-                "encrypted_password": "5b1d16b78004bbd51fa06af9eda75972",
-                "options": null
-            },
-            {
-                "name": "zen",
-                "encrypted_password": "9b1d16b78004bbd51fa06af9eda75972"
-            },
-            {
-                "name": "\"name\";\\n select 1;",
-                "encrypted_password": "5b1d16b78004bbd51fa06af9eda75972"
-            },
-            {
-                "name": "MyRole",
-                "encrypted_password": "5b1d16b78004bbd51fa06af9eda75972"
-            }
-        ],
-        "databases": [
-            {
-                "name": "DB2",
-                "owner": "alexk",
-                "options": [
-                    {
-                        "name": "LC_COLLATE",
-                        "value": "C",
-                        "vartype": "string"
-                    },
-                    {
-                        "name": "LC_CTYPE",
-                        "value": "C",
-                        "vartype": "string"
-                    },
-                    {
-                        "name": "TEMPLATE",
-                        "value": "template0",
-                        "vartype": "enum"
-                    }
-                ]
-            },
-            {
-                "name": "zenith",
-                "owner": "MyRole"
-            },
-            {
-                "name": "zen",
-                "owner": "zen"
-            }
-        ],
-        "settings": [
-            {
-                "name": "fsync",
-                "value": "off",
-                "vartype": "bool"
-            },
-            {
-                "name": "wal_level",
-                "value": "replica",
-                "vartype": "enum"
-            },
-            {
-                "name": "hot_standby",
-                "value": "on",
-                "vartype": "bool"
-            },
-            {
-                "name": "wal_acceptors",
-                "value": "127.0.0.1:6502,127.0.0.1:6503,127.0.0.1:6501",
-                "vartype": "string"
-            },
-            {
-                "name": "wal_log_hints",
-                "value": "on",
-                "vartype": "bool"
-            },
-            {
-                "name": "log_connections",
-                "value": "on",
-                "vartype": "bool"
-            },
-            {
-                "name": "shared_buffers",
-                "value": "32768",
-                "vartype": "integer"
-            },
-            {
-                "name": "port",
-                "value": "55432",
-                "vartype": "integer"
-            },
-            {
-                "name": "max_connections",
-                "value": "100",
-                "vartype": "integer"
-            },
-            {
-                "name": "max_wal_senders",
-                "value": "10",
-                "vartype": "integer"
-            },
-            {
-                "name": "listen_addresses",
-                "value": "0.0.0.0",
-                "vartype": "string"
-            },
-            {
-                "name": "wal_sender_timeout",
-                "value": "0",
-                "vartype": "integer"
-            },
-            {
-                "name": "password_encryption",
-                "value": "md5",
-                "vartype": "enum"
-            },
-            {
-                "name": "maintenance_work_mem",
-                "value": "65536",
-                "vartype": "integer"
-            },
-            {
-                "name": "max_parallel_workers",
-                "value": "8",
-                "vartype": "integer"
-            },
-            {
-                "name": "max_worker_processes",
-                "value": "8",
-                "vartype": "integer"
-            },
-            {
-                "name": "zenith.zenith_tenant",
-                "value": "b0554b632bd4d547a63b86c3630317e8",
-                "vartype": "string"
-            },
-            {
-                "name": "max_replication_slots",
-                "value": "10",
-                "vartype": "integer"
-            },
-            {
-                "name": "zenith.zenith_timeline",
-                "value": "2414a61ffc94e428f14b5758fe308e13",
-                "vartype": "string"
-            },
-            {
-                "name": "shared_preload_libraries",
-                "value": "zenith",
-                "vartype": "string"
-            },
-            {
-                "name": "synchronous_standby_names",
-                "value": "walproposer",
-                "vartype": "string"
-            },
-            {
-                "name": "zenith.page_server_connstring",
-                "value": "host=127.0.0.1 port=6400",
-                "vartype": "string"
-            }
-        ]
-    },
-
-    "delta_operations": [
-        {
-            "action": "delete_db",
-            "name": "zenith_test"
-        },
-        {
-            "action": "rename_db",
-            "name": "DB",
-            "new_name": "DB2"
-        },
-        {
-            "action": "delete_role",
-            "name": "zenith2"
-        },
-        {
-            "action": "rename_role",
-            "name": "zenith new",
-            "new_name": "zenith \"new\""
-        }
-    ]
-}

compute_tools/tests/config_test.rs

@@ -1,48 +0,0 @@
-#[cfg(test)]
-mod config_tests {
-
-    use std::fs::{remove_file, File};
-    use std::io::{Read, Write};
-    use std::path::Path;
-
-    use compute_tools::config::*;
-
-    fn write_test_file(path: &Path, content: &str) {
-        let mut file = File::create(path).unwrap();
-        file.write_all(content.as_bytes()).unwrap();
-    }
-
-    fn check_file_content(path: &Path, expected_content: &str) {
-        let mut file = File::open(path).unwrap();
-        let mut content = String::new();
-
-        file.read_to_string(&mut content).unwrap();
-        assert_eq!(content, expected_content);
-    }
-
-    #[test]
-    fn test_line_in_file() {
-        let path = Path::new("./tests/tmp/config_test.txt");
-        write_test_file(path, "line1\nline2.1\t line2.2\nline3");
-
-        let line = "line2.1\t line2.2";
-        let result = line_in_file(path, line).unwrap();
-        assert!(!result);
-        check_file_content(path, "line1\nline2.1\t line2.2\nline3");
-
-        let line = "line4";
-        let result = line_in_file(path, line).unwrap();
-        assert!(result);
-        check_file_content(path, "line1\nline2.1\t line2.2\nline3\nline4");
-
-        remove_file(path).unwrap();
-
-        let path = Path::new("./tests/tmp/new_config_test.txt");
-        let line = "line4";
-        let result = line_in_file(path, line).unwrap();
-        assert!(result);
-        check_file_content(path, "line4");
-
-        remove_file(path).unwrap();
-    }
-}

compute_tools/tests/pg_helpers_tests.rs

@@ -1,41 +0,0 @@
-#[cfg(test)]
-mod pg_helpers_tests {
-
-    use std::fs::File;
-
-    use compute_tools::pg_helpers::*;
-    use compute_tools::zenith::ClusterSpec;
-
-    #[test]
-    fn params_serialize() {
-        let file = File::open("tests/cluster_spec.json").unwrap();
-        let spec: ClusterSpec = serde_json::from_reader(file).unwrap();
-
-        assert_eq!(
-            spec.cluster.databases.first().unwrap().to_pg_options(),
-            "LC_COLLATE 'C' LC_CTYPE 'C' TEMPLATE template0 OWNER \"alexk\""
-        );
-        assert_eq!(
-            spec.cluster.roles.first().unwrap().to_pg_options(),
-            "LOGIN PASSWORD 'md56b1d16b78004bbd51fa06af9eda75972'"
-        );
-    }
-
-    #[test]
-    fn settings_serialize() {
-        let file = File::open("tests/cluster_spec.json").unwrap();
-        let spec: ClusterSpec = serde_json::from_reader(file).unwrap();
-
-        assert_eq!(
-            spec.cluster.settings.as_pg_settings(),
-            "fsync = off\nwal_level = replica\nhot_standby = on\nwal_acceptors = '127.0.0.1:6502,127.0.0.1:6503,127.0.0.1:6501'\nwal_log_hints = on\nlog_connections = on\nshared_buffers = 32768\nport = 55432\nmax_connections = 100\nmax_wal_senders = 10\nlisten_addresses = '0.0.0.0'\nwal_sender_timeout = 0\npassword_encryption = md5\nmaintenance_work_mem = 65536\nmax_parallel_workers = 8\nmax_worker_processes = 8\nzenith.zenith_tenant = 'b0554b632bd4d547a63b86c3630317e8'\nmax_replication_slots = 10\nzenith.zenith_timeline = '2414a61ffc94e428f14b5758fe308e13'\nshared_preload_libraries = 'zenith'\nsynchronous_standby_names = 'walproposer'\nzenith.page_server_connstring = 'host=127.0.0.1 port=6400'"
-        );
-    }
-
-    #[test]
-    fn quote_ident() {
-        let ident: PgIdent = PgIdent::from("\"name\";\\n select 1;");
-
-        assert_eq!(ident.quote(), "\"\"\"name\"\";\\n select 1;\"");
-    }
-}

compute_tools/tests/tmp/.gitignore

@@ -1 +0,0 @@
-**/*

control_plane/Cargo.toml

@@ -7,18 +7,24 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+rand = "0.8.3"
 tar = "0.4.33"
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 serde = { version = "1.0", features = ["derive"] }
+serde_json = "1"
 toml = "0.5"
 lazy_static = "1.4"
 regex = "1"
 anyhow = "1.0"
 thiserror = "1"
+bytes = "1.0.1"
 nix = "0.23"
 url = "2.2.2"
-reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] }
+hex = { version = "0.4.3", features = ["serde"] }
+reqwest = { version = "0.11", features = ["blocking", "json"] }
 
 pageserver = { path = "../pageserver" }
+walkeeper = { path = "../walkeeper" }
+postgres_ffi = { path = "../postgres_ffi" }
 zenith_utils = { path = "../zenith_utils" }
 workspace_hack = { path = "../workspace_hack" }

control_plane/safekeepers.conf

@@ -1,7 +1,7 @@
 # Page server and three safekeepers.
 [pageserver]
-listen_pg_addr = 'localhost:64000'
-listen_http_addr = 'localhost:9898'
+pg_port = 64000
+http_port = 9898
 auth_type = 'Trust'
 
 [[safekeepers]]

control_plane/simple.conf

@@ -1,8 +1,8 @@
 # Minimal zenith environment with one safekeeper. This is equivalent to the built-in
 # defaults that you get with no --config
 [pageserver]
-listen_pg_addr = 'localhost:64000'
-listen_http_addr = 'localhost:9898'
+pg_port = 64000
+http_port = 9898
 auth_type = 'Trust'
 
 [[safekeepers]]

control_plane/src/compute.rs

@@ -199,24 +199,17 @@ impl PostgresNode {
         })
     }
 
-    fn sync_safekeepers(&self, auth_token: &Option<String>) -> Result<Lsn> {
+    fn sync_safekeepers(&self) -> Result<Lsn> {
         let pg_path = self.env.pg_bin_dir().join("postgres");
-        let mut cmd = Command::new(&pg_path);
-
-        cmd.arg("--sync-safekeepers")
+        let sync_handle = Command::new(pg_path)
+            .arg("--sync-safekeepers")
             .env_clear()
             .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
             .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
             .env("PGDATA", self.pgdata().to_str().unwrap())
             .stdout(Stdio::piped())
             // Comment this to avoid capturing stderr (useful if command hangs)
-            .stderr(Stdio::piped());
-
-        if let Some(token) = auth_token {
-            cmd.env("ZENITH_AUTH_TOKEN", token);
-        }
-
-        let sync_handle = cmd
+            .stderr(Stdio::piped())
             .spawn()
             .expect("postgres --sync-safekeepers failed to start");
 
@@ -296,10 +289,8 @@ impl PostgresNode {
         conf.append("shared_buffers", "1MB");
         conf.append("fsync", "off");
         conf.append("max_connections", "100");
+        conf.append("wal_sender_timeout", "0");
         conf.append("wal_level", "replica");
-        // wal_sender_timeout is the maximum time to wait for WAL replication.
-        // It also defines how often the walreciever will send a feedback message to the wal sender.
-        conf.append("wal_sender_timeout", "5s");
         conf.append("listen_addresses", &self.address.ip().to_string());
         conf.append("port", &self.address.port().to_string());
 
@@ -324,11 +315,8 @@ impl PostgresNode {
             } else {
                 ""
             };
-            // NOTE avoiding spaces in connection string, because it is less error prone if we forward it somewhere.
-            // Also note that not all parameters are supported here. Because in compute we substitute $ZENITH_AUTH_TOKEN
-            // We parse this string and build it back with token from env var, and for simplicity rebuild
-            // uses only needed variables namely host, port, user, password.
-            format!("postgresql://no_user:{}@{}:{}", password, host, port)
+
+            format!("host={} port={} password={}", host, port, password)
         };
         conf.append("shared_preload_libraries", "zenith");
         conf.append_line("");
@@ -341,11 +329,6 @@ impl PostgresNode {
         conf.append_line("");
 
         if !self.env.safekeepers.is_empty() {
-            // Configure backpressure
-            // In setup with safekeepers apply_lag depends on
-            // speed of data checkpointing on pageserver (see disk_consistent_lsn).
-            conf.append("max_replication_apply_lag", "1500MB");
-
             // Configure the node to connect to the safekeepers
             conf.append("synchronous_standby_names", "walproposer");
 
@@ -358,16 +341,6 @@ impl PostgresNode {
                 .join(",");
             conf.append("wal_acceptors", &wal_acceptors);
         } else {
-            // Configure backpressure
-            // In setup without safekeepers, flush_lag depends on
-            // speed of of data checkpointing on pageserver (see disk_consistent_lsn)
-            conf.append("max_replication_flush_lag", "1500MB");
-
-            // We only use setup without safekeepers for tests,
-            // and don't care about data durability on pageserver,
-            // so set more relaxed synchronous_commit.
-            conf.append("synchronous_commit", "remote_write");
-
             // Configure the node to stream WAL directly to the pageserver
             // This isn't really a supported configuration, but can be useful for
             // testing.
@@ -381,7 +354,7 @@ impl PostgresNode {
         Ok(())
     }
 
-    fn load_basebackup(&self, auth_token: &Option<String>) -> Result<()> {
+    fn load_basebackup(&self) -> Result<()> {
         let backup_lsn = if let Some(lsn) = self.lsn {
             Some(lsn)
         } else if self.uses_wal_proposer {
@@ -389,7 +362,7 @@ impl PostgresNode {
             // latest data from the pageserver. That is a bit clumsy but whole bootstrap
             // procedure evolves quite actively right now, so let's think about it again
             // when things would be more stable (TODO).
-            let lsn = self.sync_safekeepers(auth_token)?;
+            let lsn = self.sync_safekeepers()?;
             if lsn == Lsn(0) {
                 None
             } else {
@@ -440,6 +413,7 @@ impl PostgresNode {
         .env_clear()
         .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap())
         .env("DYLD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap());
+
         if let Some(token) = auth_token {
             cmd.env("ZENITH_AUTH_TOKEN", token);
         }
@@ -473,7 +447,7 @@ impl PostgresNode {
         fs::write(&postgresql_conf_path, postgresql_conf)?;
 
         // 3. Load basebackup
-        self.load_basebackup(auth_token)?;
+        self.load_basebackup()?;
 
         if self.lsn.is_some() {
             File::create(self.pgdata().join("standby.signal"))?;

control_plane/src/local_env.rs

@@ -1,9 +1,10 @@
-//! This module is responsible for locating and loading paths in a local setup.
-//!
-//! Now it also provides init method which acts like a stub for proper installation
-//! script which will use local paths.
-
-use anyhow::{bail, Context};
+//
+// This module is responsible for locating and loading paths in a local setup.
+//
+// Now it also provides init method which acts like a stub for proper installation
+// script which will use local paths.
+//
+use anyhow::{Context, Result};
 use serde::{Deserialize, Serialize};
 use std::env;
 use std::fmt::Write;
@@ -12,7 +13,7 @@ use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
 use zenith_utils::auth::{encode_from_key_file, Claims, Scope};
 use zenith_utils::postgres_backend::AuthType;
-use zenith_utils::zid::{opt_display_serde, ZTenantId};
+use zenith_utils::zid::ZTenantId;
 
 //
 // This data structures represents zenith CLI config
@@ -45,7 +46,7 @@ pub struct LocalEnv {
 
     // Default tenant ID to use with the 'zenith' command line utility, when
     // --tenantid is not explicitly specified.
-    #[serde(with = "opt_display_serde")]
+    #[serde(with = "opt_tenantid_serde")]
     #[serde(default)]
     pub default_tenantid: Option<ZTenantId>,
 
@@ -63,8 +64,8 @@ pub struct LocalEnv {
 #[serde(default)]
 pub struct PageServerConf {
     // Pageserver connection settings
-    pub listen_pg_addr: String,
-    pub listen_http_addr: String,
+    pub pg_port: u16,
+    pub http_port: u16,
 
     // used to determine which auth type is used
     pub auth_type: AuthType,
@@ -76,10 +77,10 @@ pub struct PageServerConf {
 impl Default for PageServerConf {
     fn default() -> Self {
         Self {
-            listen_pg_addr: String::new(),
-            listen_http_addr: String::new(),
+            pg_port: 0,
+            http_port: 0,
             auth_type: AuthType::Trust,
-            auth_token: String::new(),
+            auth_token: "".to_string(),
         }
     }
 }
@@ -96,7 +97,7 @@ pub struct SafekeeperConf {
 impl Default for SafekeeperConf {
     fn default() -> Self {
         Self {
-            name: String::new(),
+            name: "".to_string(),
             pg_port: 0,
             http_port: 0,
             sync: true,
@@ -113,11 +114,11 @@ impl LocalEnv {
         self.pg_distrib_dir.join("lib")
     }
 
-    pub fn pageserver_bin(&self) -> anyhow::Result<PathBuf> {
+    pub fn pageserver_bin(&self) -> Result<PathBuf> {
         Ok(self.zenith_distrib_dir.join("pageserver"))
     }
 
-    pub fn safekeeper_bin(&self) -> anyhow::Result<PathBuf> {
+    pub fn safekeeper_bin(&self) -> Result<PathBuf> {
         Ok(self.zenith_distrib_dir.join("safekeeper"))
     }
 
@@ -144,7 +145,7 @@ impl LocalEnv {
     ///
     /// Unlike 'load_config', this function fills in any defaults that are missing
     /// from the config file.
-    pub fn create_config(toml: &str) -> anyhow::Result<Self> {
+    pub fn create_config(toml: &str) -> Result<LocalEnv> {
         let mut env: LocalEnv = toml::from_str(toml)?;
 
         // Find postgres binaries.
@@ -158,7 +159,7 @@ impl LocalEnv {
             }
         }
         if !env.pg_distrib_dir.join("bin/postgres").exists() {
-            bail!(
+            anyhow::bail!(
                 "Can't find postgres binary at {}",
                 env.pg_distrib_dir.display()
             );
@@ -168,14 +169,11 @@ impl LocalEnv {
         if env.zenith_distrib_dir == Path::new("") {
             env.zenith_distrib_dir = env::current_exe()?.parent().unwrap().to_owned();
         }
-        for binary in ["pageserver", "safekeeper"] {
-            if !env.zenith_distrib_dir.join(binary).exists() {
-                bail!(
-                    "Can't find binary '{}' in zenith distrib dir '{}'",
-                    binary,
-                    env.zenith_distrib_dir.display()
-                );
-            }
+        if !env.zenith_distrib_dir.join("pageserver").exists() {
+            anyhow::bail!("Can't find pageserver binary.");
+        }
+        if !env.zenith_distrib_dir.join("safekeeper").exists() {
+            anyhow::bail!("Can't find safekeeper binary.");
         }
 
         // If no initial tenant ID was given, generate it.
@@ -189,11 +187,11 @@ impl LocalEnv {
     }
 
     /// Locate and load config
-    pub fn load_config() -> anyhow::Result<Self> {
+    pub fn load_config() -> Result<LocalEnv> {
         let repopath = base_path();
 
         if !repopath.exists() {
-            bail!(
+            anyhow::bail!(
                 "Zenith config is not found in {}. You need to run 'zenith init' first",
                 repopath.to_str().unwrap()
             );
@@ -211,7 +209,7 @@ impl LocalEnv {
     }
 
     // this function is used only for testing purposes in CLI e g generate tokens during init
-    pub fn generate_auth_token(&self, claims: &Claims) -> anyhow::Result<String> {
+    pub fn generate_auth_token(&self, claims: &Claims) -> Result<String> {
         let private_key_path = if self.private_key_path.is_absolute() {
             self.private_key_path.to_path_buf()
         } else {
@@ -225,14 +223,14 @@ impl LocalEnv {
     //
     // Initialize a new Zenith repository
     //
-    pub fn init(&mut self) -> anyhow::Result<()> {
+    pub fn init(&mut self) -> Result<()> {
         // check if config already exists
         let base_path = &self.base_data_dir;
         if base_path == Path::new("") {
-            bail!("repository base path is missing");
+            anyhow::bail!("repository base path is missing");
         }
         if base_path.exists() {
-            bail!(
+            anyhow::bail!(
                 "directory '{}' already exists. Perhaps already initialized?",
                 base_path.to_str().unwrap()
             );
@@ -253,12 +251,12 @@ impl LocalEnv {
                 .output()
                 .with_context(|| "failed to generate auth private key")?;
             if !keygen_output.status.success() {
-                bail!(
+                anyhow::bail!(
                     "openssl failed: '{}'",
                     String::from_utf8_lossy(&keygen_output.stderr)
                 );
             }
-            self.private_key_path = PathBuf::from("auth_private_key.pem");
+            self.private_key_path = Path::new("auth_private_key.pem").to_path_buf();
 
             let public_key_path = base_path.join("auth_public_key.pem");
             // openssl rsa -in private_key.pem -pubout -outform PEM -out public_key.pem
@@ -272,7 +270,7 @@ impl LocalEnv {
                 .output()
                 .with_context(|| "failed to generate auth private key")?;
             if !keygen_output.status.success() {
-                bail!(
+                anyhow::bail!(
                     "openssl failed: '{}'",
                     String::from_utf8_lossy(&keygen_output.stderr)
                 );
@@ -284,7 +282,7 @@ impl LocalEnv {
 
         fs::create_dir_all(self.pg_data_dirs_path())?;
 
-        for safekeeper in &self.safekeepers {
+        for safekeeper in self.safekeepers.iter() {
             fs::create_dir_all(self.safekeeper_data_dir(&safekeeper.name))?;
         }
 
@@ -327,3 +325,30 @@ fn base_path() -> PathBuf {
         None => ".zenith".into(),
     }
 }
+
+/// Serde routines for Option<ZTenantId>. The serialized form is a hex string.
+mod opt_tenantid_serde {
+    use serde::{Deserialize, Deserializer, Serialize, Serializer};
+    use std::str::FromStr;
+    use zenith_utils::zid::ZTenantId;
+
+    pub fn serialize<S>(tenantid: &Option<ZTenantId>, ser: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        tenantid.map(|t| t.to_string()).serialize(ser)
+    }
+
+    pub fn deserialize<'de, D>(des: D) -> Result<Option<ZTenantId>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s: Option<String> = Option::deserialize(des)?;
+        if let Some(s) = s {
+            return Ok(Some(
+                ZTenantId::from_str(&s).map_err(serde::de::Error::custom)?,
+            ));
+        }
+        Ok(None)
+    }
+}

control_plane/src/safekeeper.rs

@@ -15,11 +15,13 @@ use reqwest::blocking::{Client, RequestBuilder, Response};
 use reqwest::{IntoUrl, Method};
 use thiserror::Error;
 use zenith_utils::http::error::HttpErrorBody;
+use zenith_utils::postgres_backend::AuthType;
 
 use crate::local_env::{LocalEnv, SafekeeperConf};
 use crate::read_pidfile;
 use crate::storage::PageServerNode;
 use zenith_utils::connstring::connection_address;
+use zenith_utils::connstring::connection_host_port;
 
 #[derive(Error, Debug)]
 pub enum SafekeeperHttpError {
@@ -114,24 +116,36 @@ impl SafekeeperNode {
         );
         io::stdout().flush().unwrap();
 
+        // Configure connection to page server
+        //
+        // FIXME: We extract the host and port from the connection string instead of using
+        // the connection string directly, because the 'safekeeper' binary expects
+        // host:port format. That's a bit silly when we already have a full libpq connection
+        // string at hand.
+        let pageserver_conn = {
+            let (host, port) = connection_host_port(&self.pageserver.pg_connection_config);
+            format!("{}:{}", host, port)
+        };
+
         let listen_pg = format!("localhost:{}", self.conf.pg_port);
         let listen_http = format!("localhost:{}", self.conf.http_port);
 
-        let mut cmd = Command::new(self.env.safekeeper_bin()?);
-        cmd.args(&["-D", self.datadir_path().to_str().unwrap()])
+        let mut cmd: &mut Command = &mut Command::new(self.env.safekeeper_bin()?);
+        cmd = cmd
+            .args(&["-D", self.datadir_path().to_str().unwrap()])
             .args(&["--listen-pg", &listen_pg])
             .args(&["--listen-http", &listen_http])
+            .args(&["--pageserver", &pageserver_conn])
             .args(&["--recall", "1 second"])
             .arg("--daemonize")
             .env_clear()
             .env("RUST_BACKTRACE", "1");
         if !self.conf.sync {
-            cmd.arg("--no-sync");
+            cmd = cmd.arg("--no-sync");
         }
 
-        let var = "LLVM_PROFILE_FILE";
-        if let Some(val) = std::env::var_os(var) {
-            cmd.env(var, val);
+        if self.env.pageserver.auth_type == AuthType::ZenithJWT {
+            cmd.env("PAGESERVER_AUTH_TOKEN", &self.env.pageserver.auth_token);
         }
 
         if !cmd.status()?.success() {

control_plane/src/storage.rs

@@ -5,7 +5,7 @@ use std::process::Command;
 use std::time::Duration;
 use std::{io, result, thread};
 
-use anyhow::bail;
+use anyhow::{anyhow, bail};
 use nix::errno::Errno;
 use nix::sys::signal::{kill, Signal};
 use nix::unistd::Pid;
@@ -78,58 +78,48 @@ impl PageServerNode {
             ""
         };
 
-        Self {
+        PageServerNode {
             pg_connection_config: Self::pageserver_connection_config(
                 password,
-                &env.pageserver.listen_pg_addr,
+                env.pageserver.pg_port,
             ),
             env: env.clone(),
             http_client: Client::new(),
-            http_base_url: format!("http://{}/v1", env.pageserver.listen_http_addr),
+            http_base_url: format!("http://localhost:{}/v1", env.pageserver.http_port),
         }
     }
 
     /// Construct libpq connection string for connecting to the pageserver.
-    fn pageserver_connection_config(password: &str, listen_addr: &str) -> Config {
-        format!("postgresql://no_user:{}@{}/no_db", password, listen_addr)
+    fn pageserver_connection_config(password: &str, port: u16) -> Config {
+        format!("postgresql://no_user:{}@localhost:{}/no_db", password, port)
             .parse()
             .unwrap()
     }
 
     pub fn init(&self, create_tenant: Option<&str>) -> anyhow::Result<()> {
         let mut cmd = Command::new(self.env.pageserver_bin()?);
-        let var = "LLVM_PROFILE_FILE";
-        if let Some(val) = std::env::var_os(var) {
-            cmd.env(var, val);
-        }
-
-        // FIXME: the paths should be shell-escaped to handle paths with spaces, quotas etc.
+        let listen_pg = format!("localhost:{}", self.env.pageserver.pg_port);
+        let listen_http = format!("localhost:{}", self.env.pageserver.http_port);
         let mut args = vec![
-            "--init".to_string(),
-            "-D".to_string(),
-            self.env.base_data_dir.display().to_string(),
-            "-c".to_string(),
-            format!("pg_distrib_dir='{}'", self.env.pg_distrib_dir.display()),
-            "-c".to_string(),
-            format!("auth_type='{}'", self.env.pageserver.auth_type),
-            "-c".to_string(),
-            format!(
-                "listen_http_addr='{}'",
-                self.env.pageserver.listen_http_addr
-            ),
-            "-c".to_string(),
-            format!("listen_pg_addr='{}'", self.env.pageserver.listen_pg_addr),
+            "--init",
+            "-D",
+            self.env.base_data_dir.to_str().unwrap(),
+            "--postgres-distrib",
+            self.env.pg_distrib_dir.to_str().unwrap(),
+            "--listen-pg",
+            &listen_pg,
+            "--listen-http",
+            &listen_http,
         ];
 
+        let auth_type_str = &self.env.pageserver.auth_type.to_string();
         if self.env.pageserver.auth_type != AuthType::Trust {
-            args.extend([
-                "-c".to_string(),
-                "auth_validation_public_key_path='auth_public_key.pem'".to_string(),
-            ]);
+            args.extend(&["--auth-validation-public-key-path", "auth_public_key.pem"]);
         }
+        args.extend(&["--auth-type", auth_type_str]);
 
         if let Some(tenantid) = create_tenant {
-            args.extend(["--create-tenant".to_string(), tenantid.to_string()])
+            args.extend(&["--create-tenant", tenantid])
         }
 
         let status = cmd
@@ -139,11 +129,11 @@ impl PageServerNode {
             .status()
             .expect("pageserver init failed");
 
-        if !status.success() {
-            bail!("pageserver init failed");
+        if status.success() {
+            Ok(())
+        } else {
+            Err(anyhow!("pageserver init failed"))
         }
-
-        Ok(())
     }
 
     pub fn repo_path(&self) -> PathBuf {
@@ -168,11 +158,6 @@ impl PageServerNode {
             .env_clear()
             .env("RUST_BACKTRACE", "1");
 
-        let var = "LLVM_PROFILE_FILE";
-        if let Some(val) = std::env::var_os(var) {
-            cmd.env(var, val);
-        }
-
         if !cmd.status()?.success() {
             bail!(
                 "Pageserver failed to start. See '{}' for details.",

docker-entrypoint.sh

@@ -4,10 +4,10 @@ set -eux
 if [ "$1" = 'pageserver' ]; then
     if [ ! -d "/data/tenants" ]; then
         echo "Initializing pageserver data directory"
-        pageserver --init -D /data -c "pg_distrib_dir='/usr/local'"
+        pageserver --init -D /data --postgres-distrib /usr/local
     fi
     echo "Staring pageserver at 0.0.0.0:6400"
-    pageserver -c "listen_pg_addr='0.0.0.0:6400'" -c "listen_http_addr='0.0.0.0:9898'" -D /data
+    pageserver -l 0.0.0.0:6400 --listen-http 0.0.0.0:9898 -D /data
 else
     "$@"
 fi

docs/glossary.md

@@ -51,14 +51,11 @@ Each PostgreSQL fork is considered a separate relish.
 
 ### Layer
 
-A layer contains data needed to reconstruct any page versions within the
-layer's Segment and range of LSNs.
-
+Each layer corresponds to the specific version of a relish Segment in a range of LSNs.
 There are two kinds of layers, in-memory and on-disk layers. In-memory
 layers are used to ingest incoming WAL, and provide fast access
 to the recent page versions. On-disk layers are stored as files on disk, and
-are immutable. See pageserver/src/layered_repository/README.md for more.
-
+are immutable.
 ### Layer file (on-disk layer)
 
 Layered repository on-disk format is based on immutable files.  The

docs/settings.md

@@ -1,176 +0,0 @@
-## Pageserver
-
-Pageserver is mainly configured via a `pageserver.toml` config file.
-If there's no such file during `init` phase of the server, it creates the file itself. Without 'init', the file is read.
-
-There's a possibility to pass an arbitrary config value to the pageserver binary as an argument: such values override
-the values in the config file, if any are specified for the same key and get into the final config during init phase.
-
-
-### Config example
-
-```toml
-# Initial configuration file created by 'pageserver --init'
-
-listen_pg_addr = '127.0.0.1:64000'
-listen_http_addr = '127.0.0.1:9898'
-
-checkpoint_distance = '268435456' # in bytes
-checkpoint_period = '1 s'
-
-gc_period = '100 s'
-gc_horizon = '67108864'
-
-max_file_descriptors = '100'
-
-# initial superuser role name to use when creating a new tenant
-initial_superuser_name = 'zenith_admin'
-
-# [remote_storage]
-```
-
-The config above shows default values for all basic pageserver settings.
-Pageserver uses default values for all files that are missing in the config, so it's not a hard error to leave the config blank.
-Yet, it validates the config values it can (e.g. postgres install dir) and errors if the validation fails, refusing to start.
-
-Note the `[remote_storage]` section: it's a [table](https://toml.io/en/v1.0.0#table) in TOML specification and
-
-* either has to be placed in the config after the table-less values such as `initial_superuser_name = 'zenith_admin'`
-
-* or can be placed anywhere if rewritten in identical form as [inline table](https://toml.io/en/v1.0.0#inline-table): `remote_storage = {foo = 2}`
-
-### Config values
-
-All values can be passed as an argument to the pageserver binary, using the `-c` parameter and specified as a valid TOML string. All tables should be passed in the inline form.
-
-Example: `${PAGESERVER_BIN} -c "checkpoint_period = '100 s'" -c "remote_storage={local_path='/some/local/path/'}"`
-
-Note that TOML distinguishes between strings and integers, the former require single or double quotes around them.
-
-#### checkpoint_distance
-
-`checkpoint_distance` is the amount of incoming WAL that is held in
-the open layer, before it's flushed to local disk. It puts an upper
-bound on how much WAL needs to be re-processed after a pageserver
-crash. It is a soft limit, the pageserver can momentarily go above it,
-but it will trigger a checkpoint operation to get it back below the
-limit.
-
-`checkpoint_distance` also determines how much WAL needs to be kept
-durable in the safekeeper.  The safekeeper must have capacity to hold
-this much WAL, with some headroom, otherwise you can get stuck in a
-situation where the safekeeper is full and stops accepting new WAL,
-but the pageserver is not flushing out and releasing the space in the
-safekeeper because it hasn't reached checkpoint_distance yet.
-
-`checkpoint_distance` also controls how often the WAL is uploaded to
-S3.
-
-The unit is # of bytes.
-
-#### checkpoint_period
-
-The pageserver checks whether `checkpoint_distance` has been reached
-every `checkpoint_period` seconds. Default is 1 s, which should be
-fine.
-
-#### gc_horizon
-
-`gz_horizon` determines how much history is retained, to allow
-branching and read replicas at an older point in time. The unit is #
-of bytes of WAL. Page versions older than this are garbage collected
-away.
-
-#### gc_period
-
-Interval at which garbage collection is triggered. Default is 100 s.
-
-#### initial_superuser_name
-
-Name of the initial superuser role, passed to initdb when a new tenant
-is initialized. It doesn't affect anything after initialization. The
-default is Note: The default is 'zenith_admin', and the console
-depends on that, so if you change it, bad things will happen.
-
-#### page_cache_size
-
-Size of the page cache, to hold materialized page versions. Unit is
-number of 8 kB blocks. The default is 8192, which means 64 MB.
-
-#### max_file_descriptors
-
-Max number of file descriptors to hold open concurrently for accessing
-layer files. This should be kept well below the process/container/OS
-limit (see `ulimit -n`), as the pageserver also needs file descriptors
-for other files and for sockets for incoming connections.
-
-#### pg_distrib_dir
-
-A directory with Postgres installation to use during pageserver activities.
-Inside that dir, a `bin/postgres` binary should be present.
-
-The default distrib dir is `./tmp_install/`.
-
-#### workdir (-D)
-
-A directory in the file system, where pageserver will store its files.
-The default is `./.zenith/`.
-
-This parameter has a special CLI alias (`-D`) and can not be overridden with regular `-c` way.
-
-##### Remote storage
-
-There's a way to automatically back up and restore some of the pageserver's data from working dir to the remote storage.
-The backup system is disabled by default and can be enabled for either of the currently available storages:
-
-###### Local FS storage
-
-Pageserver can back up and restore some of its workdir contents to another directory.
-For that, only a path to that directory needs to be specified as a parameter:
-
-```toml
-[remote_storage]
-local_path = '/some/local/path/'
-```
-
-###### S3 storage
-
-Pageserver can back up and restore some of its workdir contents to S3.
-Full set of S3 credentials is needed for that as parameters.
-Configuration example:
-
-```toml
-[remote_storage]
-# Name of the bucket to connect to
-bucket_name = 'some-sample-bucket'
-
-# Name of the region where the bucket is located at
-bucket_region = 'eu-north-1'
-
-# Access key to connect to the bucket ("login" part of the credentials)
-access_key_id = 'SOMEKEYAAAAASADSAH*#'
-
-# Secret access key to connect to the bucket ("password" part of the credentials)
-secret_access_key = 'SOMEsEcReTsd292v'
-```
-
-###### General remote storage configuration
-
-Pagesever allows only one remote storage configured concurrently and errors if parameters from multiple different remote configurations are used.
-No default values are used for the remote storage configuration parameters.
-
-Besides, there are parameters common for all types of remote storage that can be configured, those have defaults:
-
-```toml
-[remote_storage]
-# Max number of concurrent connections to open for uploading to or downloading from the remote storage.
-max_concurrent_sync = 100
-
-# Max number of errors a single task can have before it's considered failed and not attempted to run anymore.
-max_sync_errors = 10
-```
-
-
-## safekeeper
-
-TODO

docs/sourcetree.md

@@ -79,61 +79,3 @@ Helpers for exposing Prometheus metrics from the server.
 `/zenith_utils`:
 
 Helpers that are shared between other crates in this repository.
-
-## Using Python
-Note that Debian/Ubuntu Python packages are stale, as it commonly happens,
-so manual installation of dependencies is not recommended.
-
-A single virtual environment with all dependencies is described in the single `Pipfile`.
-
-### Prerequisites
-- Install Python 3.7 (the minimal supported version)
-    - Later version (e.g. 3.8) is ok if you don't write Python code
-    - You can install Python 3.7 separately, e.g.:
-      ```bash
-      # In Ubuntu
-      sudo add-apt-repository ppa:deadsnakes/ppa
-      sudo apt update
-      sudo apt install python3.7
-      ```
-- Install `pipenv`
-    - Exact version of `pipenv` is not important, you can use Debian/Ubuntu package `pipenv`.
-- Install dependencies via either
-  * `pipenv --python 3.7 install --dev` if you will write Python code, or
-  * `pipenv install` if you only want to run Python scripts and don't have Python 3.7.
-
-Run `pipenv shell` to activate the virtual environment.
-Alternatively, use `pipenv run` to run a single command in the venv, e.g. `pipenv run pytest`.
-
-### Obligatory checks
-We force code formatting via `yapf` and type hints via `mypy`.
-Run the following commands in the repository's root (next to `setup.cfg`):
-
-```bash
-pipenv run yapf -ri .  # All code is reformatted
-pipenv run mypy .  # Ensure there are no typing errors
-```
-
-**WARNING**: do not run `mypy` from a directory other than the root of the repository.
-Otherwise it will not find its configuration.
-
-Also consider:
-
-* Running `flake8` (or a linter of your choice, e.g. `pycodestyle`) and fixing possible defects, if any.
-* Adding more type hints to your code to avoid `Any`.
-
-### Changing dependencies
-You have to update `Pipfile.lock` if you have changed `Pipfile`:
-
-```bash
-pipenv --python 3.7 install --dev  # Re-create venv for Python 3.7 and install recent pipenv inside
-pipenv run pipenv --version  # Should be at least 2021.5.29
-pipenv run pipenv lock  # Regenerate Pipfile.lock
-```
-
-As the minimal supported version is Python 3.7 and we use it in CI,
-you have to use a Python 3.7 environment when updating `Pipfile.lock`.
-Otherwise some back-compatibility packages will be missing.
-
-It is also important to run recent `pipenv`.
-Older versions remove markers from `Pipfile.lock`.

pageserver/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Stas Kelvich <stas@zenith.tech>"]
 edition = "2018"
 
 [dependencies]
-bookfile = { git = "https://github.com/zenithdb/bookfile.git", branch="generic-readext" }
+bookfile = "^0.3"
 chrono = "0.4.19"
 rand = "0.8.3"
 regex = "1.4.5"
@@ -17,12 +17,10 @@ lazy_static = "1.4.0"
 log = "0.4.14"
 clap = "2.33.0"
 daemonize = "0.4.1"
-tokio = { version = "1.11", features = ["process", "sync", "macros", "fs", "rt", "io-util", "time"] }
+tokio = { version = "1.11", features = ["process", "macros", "fs", "rt", "io-util"] }
 postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
-tokio-stream = "0.1.8"
 routerify = "2"
 anyhow = "1.0"
 crc32c = "0.6.0"
@@ -32,21 +30,14 @@ tar = "0.4.33"
 humantime = "2.1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
-toml_edit = { version = "0.12", features = ["easy"] }
+toml = "0.5"
 scopeguard = "1.1.0"
+rust-s3 = { version = "0.27.0-rc4", features = ["no-verify-ssl"] }
 async-trait = "0.1"
 const_format = "0.2.21"
 tracing = "0.1.27"
-tracing-futures = "0.2"
-signal-hook = "0.3.10"
+signal-hook = {version = "0.3.10", features = ["extended-siginfo"] }
 url = "2"
-nix = "0.23"
-once_cell = "1.8.0"
-parking_lot = "0.11.2"
-crossbeam-utils = "0.8.5"
-
-rust-s3 = { version = "0.28", default-features = false, features = ["no-verify-ssl", "tokio-rustls-tls"] }
-async-compression = {version = "0.3", features = ["zstd", "tokio"]}
 
 postgres_ffi = { path = "../postgres_ffi" }
 zenith_metrics = { path = "../zenith_metrics" }

pageserver/README.md

@@ -9,7 +9,7 @@ The Page Server has a few different duties:
 
 S3 is the main fault-tolerant storage of all data, as there are no Page Server
 replicas. We use a separate fault-tolerant WAL service to reduce latency. It
-keeps track of WAL records which are not synced to S3 yet.
+keeps track of WAL records which are not syncted to S3 yet.
 
 The Page Server consists of multiple threads that operate on a shared
 repository of page versions:
@@ -129,29 +129,29 @@ There are the following implementations present:
 * local filesystem — to use in tests mainly
 * AWS S3           - to use in production
 
-Implementation details are covered in the [backup readme](./src/remote_storage/README.md) and corresponding Rust file docs.
+Implementation details are covered in the [storage readme](./src/relish_storage/README.md) and corresponding Rust file docs.
 
 The backup service is disabled by default and can be enabled to interact with a single remote storage.
 
 CLI examples:
-* Local FS: `${PAGESERVER_BIN} -c "remote_storage={local_path='/some/local/path/'}"`
-* AWS S3  : `${PAGESERVER_BIN} -c "remote_storage={bucket_name='some-sample-bucket',bucket_region='eu-north-1',access_key_id='SOMEKEYAAAAASADSAH*#',secret_access_key='SOMEsEcReTsd292v'}"`
+* Local FS: `${PAGESERVER_BIN} --relish-storage-local-path="/some/local/path/"`
+* AWS S3  : `${PAGESERVER_BIN} --relish-storage-s3-bucket="some-sample-bucket" --relish-storage-region="eu-north-1" --relish-storage-access-key="SOMEKEYAAAAASADSAH*#" --relish-storage-secret-access-key="SOMEsEcReTsd292v"`
 
 For Amazon AWS S3, a key id and secret access key could be located in `~/.aws/credentials` if awscli was ever configured to work with the desired bucket, on the AWS Settings page for a certain user. Also note, that the bucket names does not contain any protocols when used on AWS.
 For local S3 installations, refer to the their documentation for name format and credentials.
 
-Similar to other pageserver settings, toml config file can be used to configure either of the storages as backup targets.
+Similar to other pageserver settings, toml config file can be used to configure either of the storages as backup backup targets.
 Required sections are:
 
 ```toml
-[remote_storage]
+[relish_storage]
 local_path = '/Users/someonetoignore/Downloads/tmp_dir/'
 ```
 
 or
 
 ```toml
-[remote_storage]
+[relish_storage]
 bucket_name = 'some-sample-bucket'
 bucket_region = 'eu-north-1'
 access_key_id = 'SOMEKEYAAAAASADSAH*#'

pageserver/src/basebackup.rs

@@ -10,7 +10,7 @@
 //! This module is responsible for creation of such tarball
 //! from data stored in object storage.
 //!
-use anyhow::{Context, Result};
+use anyhow::Result;
 use bytes::{BufMut, BytesMut};
 use log::*;
 use std::fmt::Write as FmtWrite;
@@ -242,12 +242,10 @@ impl<'a> Basebackup<'a> {
     fn add_pgcontrol_file(&mut self) -> anyhow::Result<()> {
         let checkpoint_bytes = self
             .timeline
-            .get_page_at_lsn(RelishTag::Checkpoint, 0, self.lsn)
-            .context("failed to get checkpoint bytes")?;
-        let pg_control_bytes = self
-            .timeline
-            .get_page_at_lsn(RelishTag::ControlFile, 0, self.lsn)
-            .context("failed get control bytes")?;
+            .get_page_at_lsn(RelishTag::Checkpoint, 0, self.lsn)?;
+        let pg_control_bytes =
+            self.timeline
+                .get_page_at_lsn(RelishTag::ControlFile, 0, self.lsn)?;
         let mut pg_control = ControlFileData::decode(&pg_control_bytes)?;
         let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
 
@@ -287,7 +285,11 @@ impl<'a> Basebackup<'a> {
 
         //send wal segment
         let segno = self.lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE);
-        let wal_file_name = XLogFileName(PG_TLI, segno, pg_constants::WAL_SEGMENT_SIZE);
+        let wal_file_name = XLogFileName(
+            1, // FIXME: always use Postgres timeline 1
+            segno,
+            pg_constants::WAL_SEGMENT_SIZE,
+        );
         let wal_file_path = format!("pg_wal/{}", wal_file_name);
         let header = new_tar_header(&wal_file_path, pg_constants::WAL_SEGMENT_SIZE as u64)?;
         let wal_seg = generate_wal_segment(segno, pg_control.system_identifier);

pageserver/src/bin/dump_layerfile.rs

@@ -4,14 +4,11 @@
 use anyhow::Result;
 use clap::{App, Arg};
 use pageserver::layered_repository::dump_layerfile_from_path;
-use pageserver::virtual_file;
 use std::path::PathBuf;
-use zenith_utils::GIT_VERSION;
 
 fn main() -> Result<()> {
     let arg_matches = App::new("Zenith dump_layerfile utility")
         .about("Dump contents of one layer file, for debugging")
-        .version(GIT_VERSION)
         .arg(
             Arg::with_name("path")
                 .help("Path to file to dump")
@@ -22,9 +19,6 @@ fn main() -> Result<()> {
 
     let path = PathBuf::from(arg_matches.value_of("path").unwrap());
 
-    // Basic initialization of things that don't change after startup
-    virtual_file::init(10);
-
     dump_layerfile_from_path(&path)?;
 
     Ok(())

pageserver/src/bin/pageserver.rs

@@ -1,29 +1,275 @@
-//! Main entry point for the Page Server executable.
+//
+// Main entry point for the Page Server executable
+//
 
-use std::{env, path::Path, str::FromStr, thread};
+use serde::{Deserialize, Serialize};
+use std::{
+    env,
+    path::{Path, PathBuf},
+    str::FromStr,
+    thread,
+};
 use tracing::*;
-use zenith_utils::{auth::JwtAuth, logging, postgres_backend::AuthType, tcp_listener, GIT_VERSION};
+use zenith_utils::{auth::JwtAuth, logging, postgres_backend::AuthType, tcp_listener};
 
-use anyhow::{bail, Context, Result};
+use anyhow::{bail, ensure, Context, Result};
+use signal_hook::consts::signal::*;
+use signal_hook::consts::TERM_SIGNALS;
+use signal_hook::flag;
+use signal_hook::iterator::exfiltrator::WithOrigin;
+use signal_hook::iterator::SignalsInfo;
+use std::process::exit;
+use std::sync::atomic::AtomicBool;
+use std::sync::Arc;
 
-use clap::{App, Arg};
+use clap::{App, Arg, ArgMatches};
 use daemonize::Daemonize;
 
 use pageserver::{
-    branches,
-    config::{defaults::*, PageServerConf},
-    http, page_cache, page_service, remote_storage, tenant_mgr, virtual_file, LOG_FILE_NAME,
+    branches, defaults::*, http, page_service, relish_storage, tenant_mgr, PageServerConf,
+    RelishStorageConfig, RelishStorageKind, S3Config, LOG_FILE_NAME,
 };
 use zenith_utils::http::endpoint;
 use zenith_utils::postgres_backend;
-use zenith_utils::shutdown::exit_now;
-use zenith_utils::signals::{self, Signal};
+
+use const_format::formatcp;
+
+/// String arguments that can be declared via CLI or config file
+#[derive(Serialize, Deserialize, PartialEq, Eq, Clone)]
+struct CfgFileParams {
+    listen_pg_addr: Option<String>,
+    listen_http_addr: Option<String>,
+    checkpoint_distance: Option<String>,
+    checkpoint_period: Option<String>,
+    gc_horizon: Option<String>,
+    gc_period: Option<String>,
+    pg_distrib_dir: Option<String>,
+    auth_validation_public_key_path: Option<String>,
+    auth_type: Option<String>,
+    relish_storage_max_concurrent_sync: Option<String>,
+    /////////////////////////////////
+    //// Don't put `Option<String>` and other "simple" values below.
+    ////
+    /// `Option<RelishStorage>` is a <a href='https://toml.io/en/v1.0.0#table'>table</a> in TOML.
+    /// Values in TOML cannot be defined after tables (other tables can),
+    /// and [`toml`] crate serializes all fields in the order of their appearance.
+    ////////////////////////////////
+    relish_storage: Option<RelishStorage>,
+}
+
+#[derive(Serialize, Deserialize, PartialEq, Eq, Clone)]
+// Without this attribute, enums with values won't be serialized by the `toml` library (but can be deserialized nonetheless!).
+// See https://github.com/alexcrichton/toml-rs/blob/6c162e6562c3e432bf04c82a3d1d789d80761a86/examples/enum_external.rs for the examples
+#[serde(untagged)]
+enum RelishStorage {
+    Local {
+        local_path: String,
+    },
+    AwsS3 {
+        bucket_name: String,
+        bucket_region: String,
+        #[serde(skip_serializing)]
+        access_key_id: Option<String>,
+        #[serde(skip_serializing)]
+        secret_access_key: Option<String>,
+    },
+}
+
+impl CfgFileParams {
+    /// Extract string arguments from CLI
+    fn from_args(arg_matches: &ArgMatches) -> Self {
+        let get_arg = |arg_name: &str| -> Option<String> {
+            arg_matches.value_of(arg_name).map(str::to_owned)
+        };
+
+        let relish_storage = if let Some(local_path) = get_arg("relish-storage-local-path") {
+            Some(RelishStorage::Local { local_path })
+        } else if let Some((bucket_name, bucket_region)) =
+            get_arg("relish-storage-s3-bucket").zip(get_arg("relish-storage-region"))
+        {
+            Some(RelishStorage::AwsS3 {
+                bucket_name,
+                bucket_region,
+                access_key_id: get_arg("relish-storage-access-key"),
+                secret_access_key: get_arg("relish-storage-secret-access-key"),
+            })
+        } else {
+            None
+        };
+
+        Self {
+            listen_pg_addr: get_arg("listen-pg"),
+            listen_http_addr: get_arg("listen-http"),
+            checkpoint_distance: get_arg("checkpoint_distance"),
+            checkpoint_period: get_arg("checkpoint_period"),
+            gc_horizon: get_arg("gc_horizon"),
+            gc_period: get_arg("gc_period"),
+            pg_distrib_dir: get_arg("postgres-distrib"),
+            auth_validation_public_key_path: get_arg("auth-validation-public-key-path"),
+            auth_type: get_arg("auth-type"),
+            relish_storage,
+            relish_storage_max_concurrent_sync: get_arg("relish-storage-max-concurrent-sync"),
+        }
+    }
+
+    /// Fill missing values in `self` with `other`
+    fn or(self, other: CfgFileParams) -> Self {
+        // TODO cleaner way to do this
+        Self {
+            listen_pg_addr: self.listen_pg_addr.or(other.listen_pg_addr),
+            listen_http_addr: self.listen_http_addr.or(other.listen_http_addr),
+            checkpoint_distance: self.checkpoint_distance.or(other.checkpoint_distance),
+            checkpoint_period: self.checkpoint_period.or(other.checkpoint_period),
+            gc_horizon: self.gc_horizon.or(other.gc_horizon),
+            gc_period: self.gc_period.or(other.gc_period),
+            pg_distrib_dir: self.pg_distrib_dir.or(other.pg_distrib_dir),
+            auth_validation_public_key_path: self
+                .auth_validation_public_key_path
+                .or(other.auth_validation_public_key_path),
+            auth_type: self.auth_type.or(other.auth_type),
+            relish_storage: self.relish_storage.or(other.relish_storage),
+            relish_storage_max_concurrent_sync: self
+                .relish_storage_max_concurrent_sync
+                .or(other.relish_storage_max_concurrent_sync),
+        }
+    }
+
+    /// Create a PageServerConf from these string parameters
+    fn try_into_config(&self) -> Result<PageServerConf> {
+        let workdir = PathBuf::from(".");
+
+        let listen_pg_addr = match self.listen_pg_addr.as_ref() {
+            Some(addr) => addr.clone(),
+            None => DEFAULT_PG_LISTEN_ADDR.to_owned(),
+        };
+
+        let listen_http_addr = match self.listen_http_addr.as_ref() {
+            Some(addr) => addr.clone(),
+            None => DEFAULT_HTTP_LISTEN_ADDR.to_owned(),
+        };
+
+        let checkpoint_distance: u64 = match self.checkpoint_distance.as_ref() {
+            Some(checkpoint_distance_str) => checkpoint_distance_str.parse()?,
+            None => DEFAULT_CHECKPOINT_DISTANCE,
+        };
+        let checkpoint_period = match self.checkpoint_period.as_ref() {
+            Some(checkpoint_period_str) => humantime::parse_duration(checkpoint_period_str)?,
+            None => DEFAULT_CHECKPOINT_PERIOD,
+        };
+
+        let gc_horizon: u64 = match self.gc_horizon.as_ref() {
+            Some(horizon_str) => horizon_str.parse()?,
+            None => DEFAULT_GC_HORIZON,
+        };
+        let gc_period = match self.gc_period.as_ref() {
+            Some(period_str) => humantime::parse_duration(period_str)?,
+            None => DEFAULT_GC_PERIOD,
+        };
+
+        let pg_distrib_dir = match self.pg_distrib_dir.as_ref() {
+            Some(pg_distrib_dir_str) => PathBuf::from(pg_distrib_dir_str),
+            None => env::current_dir()?.join("tmp_install"),
+        };
+
+        let auth_validation_public_key_path = self
+            .auth_validation_public_key_path
+            .as_ref()
+            .map(PathBuf::from);
+
+        let auth_type = self
+            .auth_type
+            .as_ref()
+            .map_or(Ok(AuthType::Trust), |auth_type| {
+                AuthType::from_str(auth_type)
+            })?;
+
+        if !pg_distrib_dir.join("bin/postgres").exists() {
+            bail!("Can't find postgres binary at {:?}", pg_distrib_dir);
+        }
+
+        if auth_type == AuthType::ZenithJWT {
+            ensure!(
+                auth_validation_public_key_path.is_some(),
+                "Missing auth_validation_public_key_path when auth_type is ZenithJWT"
+            );
+            let path_ref = auth_validation_public_key_path.as_ref().unwrap();
+            ensure!(
+                path_ref.exists(),
+                format!("Can't find auth_validation_public_key at {:?}", path_ref)
+            );
+        }
+
+        let max_concurrent_sync = match self.relish_storage_max_concurrent_sync.as_deref() {
+            Some(relish_storage_max_concurrent_sync) => {
+                relish_storage_max_concurrent_sync.parse()?
+            }
+            None => DEFAULT_RELISH_STORAGE_MAX_CONCURRENT_SYNC_LIMITS,
+        };
+        let relish_storage_config = self.relish_storage.as_ref().map(|storage_params| {
+            let storage = match storage_params.clone() {
+                RelishStorage::Local { local_path } => {
+                    RelishStorageKind::LocalFs(PathBuf::from(local_path))
+                }
+                RelishStorage::AwsS3 {
+                    bucket_name,
+                    bucket_region,
+                    access_key_id,
+                    secret_access_key,
+                } => RelishStorageKind::AwsS3(S3Config {
+                    bucket_name,
+                    bucket_region,
+                    access_key_id,
+                    secret_access_key,
+                }),
+            };
+            RelishStorageConfig {
+                max_concurrent_sync,
+                storage,
+            }
+        });
+
+        Ok(PageServerConf {
+            daemonize: false,
+
+            listen_pg_addr,
+            listen_http_addr,
+            checkpoint_distance,
+            checkpoint_period,
+            gc_horizon,
+            gc_period,
+
+            superuser: String::from(DEFAULT_SUPERUSER),
+
+            workdir,
+
+            pg_distrib_dir,
+
+            auth_validation_public_key_path,
+            auth_type,
+            relish_storage_config,
+        })
+    }
+}
 
 fn main() -> Result<()> {
     zenith_metrics::set_common_metrics_prefix("pageserver");
     let arg_matches = App::new("Zenith page server")
         .about("Materializes WAL stream to pages and serves them to the postgres")
-        .version(GIT_VERSION)
+        .arg(
+            Arg::with_name("listen-pg")
+                .short("l")
+                .long("listen-pg")
+                .alias("listen") // keep some compatibility
+                .takes_value(true)
+                .help(formatcp!("listen for incoming page requests on ip:port (default: {DEFAULT_PG_LISTEN_ADDR})")),
+        )
+        .arg(
+            Arg::with_name("listen-http")
+                .long("listen-http")
+                .alias("http_endpoint") // keep some compatibility
+                .takes_value(true)
+                .help(formatcp!("http endpoint address for metrics and management API calls on ip:port (default: {DEFAULT_HTTP_LISTEN_ADDR})")),
+        )
         .arg(
             Arg::with_name("daemonize")
                 .short("d")
@@ -37,6 +283,30 @@ fn main() -> Result<()> {
                 .takes_value(false)
                 .help("Initialize pageserver repo"),
         )
+        .arg(
+            Arg::with_name("checkpoint_distance")
+                .long("checkpoint_distance")
+                .takes_value(true)
+                .help("Distance from current LSN to perform checkpoint of in-memory layers"),
+        )
+        .arg(
+            Arg::with_name("checkpoint_period")
+                .long("checkpoint_period")
+                .takes_value(true)
+                .help("Interval between checkpoint iterations"),
+        )
+        .arg(
+            Arg::with_name("gc_horizon")
+                .long("gc_horizon")
+                .takes_value(true)
+                .help("Distance from current LSN to perform all wal records cleanup"),
+        )
+        .arg(
+            Arg::with_name("gc_period")
+                .long("gc_period")
+                .takes_value(true)
+                .help("Interval between garbage collector iterations"),
+        )
         .arg(
             Arg::with_name("workdir")
                 .short("D")
@@ -44,6 +314,12 @@ fn main() -> Result<()> {
                 .takes_value(true)
                 .help("Working directory for the pageserver"),
         )
+        .arg(
+            Arg::with_name("postgres-distrib")
+                .long("postgres-distrib")
+                .takes_value(true)
+                .help("Postgres distribution directory"),
+        )
         .arg(
             Arg::with_name("create-tenant")
                 .long("create-tenant")
@@ -51,27 +327,90 @@ fn main() -> Result<()> {
                 .help("Create tenant during init")
                 .requires("init"),
         )
-        // See `settings.md` for more details on the extra configuration patameters pageserver can process
         .arg(
-            Arg::with_name("config-option")
-                .short("c")
+            Arg::with_name("auth-validation-public-key-path")
+                .long("auth-validation-public-key-path")
                 .takes_value(true)
-                .number_of_values(1)
-                .multiple(true)
-                .help("Additional configuration options or overrides of the ones from the toml config file.
-                Any option has to be a valid toml document, example: `-c \"foo='hey'\"` `-c \"foo={value=1}\"`"),
+                .help("Path to public key used to validate jwt signature"),
+        )
+        .arg(
+            Arg::with_name("auth-type")
+                .long("auth-type")
+                .takes_value(true)
+                .help("Authentication scheme type. One of: Trust, MD5, ZenithJWT"),
+        )
+        .arg(
+            Arg::with_name("relish-storage-local-path")
+                .long("relish-storage-local-path")
+                .takes_value(true)
+                .help("Path to the local directory, to be used as an external relish storage")
+                .conflicts_with_all(&[
+                    "relish-storage-s3-bucket",
+                    "relish-storage-region",
+                    "relish-storage-access-key",
+                    "relish-storage-secret-access-key",
+                ]),
+        )
+        .arg(
+            Arg::with_name("relish-storage-s3-bucket")
+                .long("relish-storage-s3-bucket")
+                .takes_value(true)
+                .help("Name of the AWS S3 bucket to use an external relish storage")
+                .requires("relish-storage-region"),
+        )
+        .arg(
+            Arg::with_name("relish-storage-region")
+                .long("relish-storage-region")
+                .takes_value(true)
+                .help("Region of the AWS S3 bucket"),
+        )
+        .arg(
+            Arg::with_name("relish-storage-access-key")
+                .long("relish-storage-access-key")
+                .takes_value(true)
+                .help("Credentials to access the AWS S3 bucket"),
+        )
+        .arg(
+            Arg::with_name("relish-storage-secret-access-key")
+                .long("relish-storage-secret-access-key")
+                .takes_value(true)
+                .help("Credentials to access the AWS S3 bucket"),
+        )
+        .arg(
+            Arg::with_name("relish-storage-max-concurrent-sync")
+                .long("relish-storage-max-concurrent-sync")
+                .takes_value(true)
+                .help("Maximum allowed concurrent synchronisations with storage"),
         )
         .get_matches();
 
     let workdir = Path::new(arg_matches.value_of("workdir").unwrap_or(".zenith"));
-    let workdir = workdir
+    let cfg_file_path = workdir
         .canonicalize()
-        .with_context(|| format!("Error opening workdir '{}'", workdir.display()))?;
-    let cfg_file_path = workdir.join("pageserver.toml");
+        .with_context(|| format!("Error opening workdir '{}'", workdir.display()))?
+        .join("pageserver.toml");
+
+    let args_params = CfgFileParams::from_args(&arg_matches);
 
     let init = arg_matches.is_present("init");
     let create_tenant = arg_matches.value_of("create-tenant");
 
+    let params = if init {
+        // We're initializing the repo, so there's no config file yet
+        args_params
+    } else {
+        // Supplement the CLI arguments with the config file
+        let cfg_file_contents = std::fs::read_to_string(&cfg_file_path)
+            .with_context(|| format!("No pageserver config at '{}'", cfg_file_path.display()))?;
+        let file_params: CfgFileParams = toml::from_str(&cfg_file_contents).with_context(|| {
+            format!(
+                "Failed to read '{}' as pageserver config",
+                cfg_file_path.display()
+            )
+        })?;
+        args_params.or(file_params)
+    };
+
     // Set CWD to workdir for non-daemon modes
     env::set_current_dir(&workdir).with_context(|| {
         format!(
@@ -80,63 +419,33 @@ fn main() -> Result<()> {
         )
     })?;
 
-    let daemonize = arg_matches.is_present("daemonize");
-    if init && daemonize {
+    // Ensure the config is valid, even if just init-ing
+    let mut conf = params.try_into_config().with_context(|| {
+        format!(
+            "Pageserver config at '{}' is not valid",
+            cfg_file_path.display()
+        )
+    })?;
+
+    conf.daemonize = arg_matches.is_present("daemonize");
+
+    if init && conf.daemonize {
         bail!("--daemonize cannot be used with --init")
     }
 
-    let mut toml = if init {
-        // We're initializing the repo, so there's no config file yet
-        DEFAULT_CONFIG_FILE
-            .parse::<toml_edit::Document>()
-            .expect("could not parse built-in config file")
-    } else {
-        // Supplement the CLI arguments with the config file
-        let cfg_file_contents = std::fs::read_to_string(&cfg_file_path)
-            .with_context(|| format!("No pageserver config at '{}'", cfg_file_path.display()))?;
-        cfg_file_contents
-            .parse::<toml_edit::Document>()
-            .with_context(|| {
-                format!(
-                    "Failed to read '{}' as pageserver config",
-                    cfg_file_path.display()
-                )
-            })?
-    };
-
-    // Process any extra options given with -c
-    if let Some(values) = arg_matches.values_of("config-option") {
-        for option_line in values {
-            let doc = toml_edit::Document::from_str(option_line).with_context(|| {
-                format!(
-                    "Option '{}' could not be parsed as a toml document",
-                    option_line
-                )
-            })?;
-            for (key, item) in doc.iter() {
-                toml.insert(key, item.clone());
-            }
-        }
-    }
-    trace!("Resulting toml: {}", toml);
-    let conf = PageServerConf::parse_and_validate(&toml, &workdir)
-        .context("Failed to parse pageserver configuration")?;
-
     // The configuration is all set up now. Turn it into a 'static
     // that can be freely stored in structs and passed across threads
     // as a ref.
     let conf: &'static PageServerConf = Box::leak(Box::new(conf));
 
-    // Basic initialization of things that don't change after startup
-    virtual_file::init(conf.max_file_descriptors);
-
-    page_cache::init(conf);
-
     // Create repo and exit if init was requested
     if init {
         branches::init_pageserver(conf, create_tenant).context("Failed to init pageserver")?;
         // write the config file
-        std::fs::write(&cfg_file_path, toml.to_string()).with_context(|| {
+        let cfg_file_contents = toml::to_string_pretty(&params)
+            .context("Failed to create pageserver config contents for initialisation")?;
+        // TODO support enable-auth flag
+        std::fs::write(&cfg_file_path, cfg_file_contents).with_context(|| {
             format!(
                 "Failed to initialize pageserver config at '{}'",
                 cfg_file_path.display()
@@ -144,15 +453,24 @@ fn main() -> Result<()> {
         })?;
         Ok(())
     } else {
-        start_pageserver(conf, daemonize).context("Failed to start pageserver")
+        start_pageserver(conf).context("Failed to start pageserver")
     }
 }
 
-fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()> {
+fn start_pageserver(conf: &'static PageServerConf) -> Result<()> {
     // Initialize logger
-    let log_file = logging::init(LOG_FILE_NAME, daemonize)?;
+    let log_file = logging::init(LOG_FILE_NAME, conf.daemonize)?;
 
-    info!("version: {}", GIT_VERSION);
+    let term_now = Arc::new(AtomicBool::new(false));
+    for sig in TERM_SIGNALS {
+        // When terminated by a second term signal, exit with exit code 1.
+        // This will do nothing the first time (because term_now is false).
+        flag::register_conditional_shutdown(*sig, 1, Arc::clone(&term_now))?;
+        // But this will "arm" the above for the second time, by setting it to true.
+        // The order of registering these is important, if you put this one first, it will
+        // first arm and then terminate ‒ all in the first round.
+        flag::register(*sig, Arc::clone(&term_now))?;
+    }
 
     // TODO: Check that it looks like a valid repository before going further
 
@@ -169,8 +487,7 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
     );
     let pageserver_listener = tcp_listener::bind(conf.listen_pg_addr.clone())?;
 
-    // XXX: Don't spawn any threads before daemonizing!
-    if daemonize {
+    if conf.daemonize {
         info!("daemonizing...");
 
         // There shouldn't be any logging to stdin/stdout. Redirect it to the main log so
@@ -184,28 +501,21 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
             .stdout(stdout)
             .stderr(stderr);
 
-        // XXX: The parent process should exit abruptly right after
-        // it has spawned a child to prevent coverage machinery from
-        // dumping stats into a `profraw` file now owned by the child.
-        // Otherwise, the coverage data will be damaged.
-        match daemonize.exit_action(|| exit_now(0)).start() {
+        match daemonize.start() {
             Ok(_) => info!("Success, daemonized"),
             Err(err) => error!(%err, "could not daemonize"),
         }
     }
 
-    let signals = signals::install_shutdown_handlers()?;
-    let mut threads = Vec::new();
+    // keep join handles for spawned threads
+    // don't spawn threads before daemonizing
+    let mut join_handles = Vec::new();
 
-    let sync_startup = remote_storage::start_local_timeline_sync(conf)
-        .context("Failed to set up local files sync with external storage")?;
-
-    if let Some(handle) = sync_startup.sync_loop_handle {
-        threads.push(handle);
+    if let Some(handle) = relish_storage::run_storage_sync_thread(conf)? {
+        join_handles.push(handle);
     }
-
     // Initialize tenant manager.
-    tenant_mgr::set_timeline_states(conf, sync_startup.initial_timeline_states);
+    tenant_mgr::init(conf);
 
     // initialize authentication for incoming connections
     let auth = match &conf.auth_type {
@@ -221,53 +531,190 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
     // Spawn a new thread for the http endpoint
     // bind before launching separate thread so the error reported before startup exits
     let cloned = auth.clone();
-    threads.push(
-        thread::Builder::new()
-            .name("http_endpoint_thread".into())
-            .spawn(move || {
-                let router = http::make_router(conf, cloned);
-                endpoint::serve_thread_main(router, http_listener)
-            })?,
-    );
+    let http_endpoint_thread = thread::Builder::new()
+        .name("http_endpoint_thread".into())
+        .spawn(move || {
+            let router = http::make_router(conf, cloned);
+            endpoint::serve_thread_main(router, http_listener)
+        })?;
+
+    join_handles.push(http_endpoint_thread);
 
     // Spawn a thread to listen for connections. It will spawn further threads
     // for each connection.
-    threads.push(
-        thread::Builder::new()
-            .name("Page Service thread".into())
-            .spawn(move || {
-                page_service::thread_main(conf, auth, pageserver_listener, conf.auth_type)
-            })?,
-    );
+    let page_service_thread = thread::Builder::new()
+        .name("Page Service thread".into())
+        .spawn(move || {
+            page_service::thread_main(conf, auth, pageserver_listener, conf.auth_type)
+        })?;
 
-    signals.handle(|signal| match signal {
-        Signal::Quit => {
-            info!(
-                "Got {}. Terminating in immediate shutdown mode",
-                signal.name()
-            );
-            std::process::exit(111);
-        }
-
-        Signal::Interrupt | Signal::Terminate => {
-            info!(
-                "Got {}. Terminating gracefully in fast shutdown mode",
-                signal.name()
-            );
-
-            postgres_backend::set_pgbackend_shutdown_requested();
-            tenant_mgr::shutdown_all_tenants()?;
-            endpoint::shutdown();
-
-            for handle in std::mem::take(&mut threads) {
-                handle
+    for info in SignalsInfo::<WithOrigin>::new(TERM_SIGNALS)?.into_iter() {
+        match info.signal {
+            SIGQUIT => {
+                info!("Got SIGQUIT. Terminate pageserver in immediate shutdown mode");
+                exit(111);
+            }
+            SIGINT | SIGTERM => {
+                info!("Got SIGINT/SIGTERM. Terminate gracefully in fast shutdown mode");
+                // Terminate postgres backends
+                postgres_backend::set_pgbackend_shutdown_requested();
+                // Stop all tenants and flush their data
+                tenant_mgr::shutdown_all_tenants()?;
+                // Wait for pageservice thread to complete the job
+                page_service_thread
                     .join()
                     .expect("thread panicked")
                     .expect("thread exited with an error");
-            }
 
-            info!("Shut down successfully completed");
-            std::process::exit(0);
+                // Shut down http router
+                endpoint::shutdown();
+
+                // Wait for all threads
+                for handle in join_handles.into_iter() {
+                    handle
+                        .join()
+                        .expect("thread panicked")
+                        .expect("thread exited with an error");
+                }
+                info!("Pageserver shut down successfully completed");
+                exit(0);
+            }
+            unknown_signal => {
+                debug!("Unknown signal {}", unknown_signal);
+            }
         }
-    })
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn page_server_conf_toml_serde() {
+        let params = CfgFileParams {
+            listen_pg_addr: Some("listen_pg_addr_VALUE".to_string()),
+            listen_http_addr: Some("listen_http_addr_VALUE".to_string()),
+            checkpoint_distance: Some("checkpoint_distance_VALUE".to_string()),
+            checkpoint_period: Some("checkpoint_period_VALUE".to_string()),
+            gc_horizon: Some("gc_horizon_VALUE".to_string()),
+            gc_period: Some("gc_period_VALUE".to_string()),
+            pg_distrib_dir: Some("pg_distrib_dir_VALUE".to_string()),
+            auth_validation_public_key_path: Some(
+                "auth_validation_public_key_path_VALUE".to_string(),
+            ),
+            auth_type: Some("auth_type_VALUE".to_string()),
+            relish_storage: Some(RelishStorage::Local {
+                local_path: "relish_storage_local_VALUE".to_string(),
+            }),
+            relish_storage_max_concurrent_sync: Some(
+                "relish_storage_max_concurrent_sync_VALUE".to_string(),
+            ),
+        };
+
+        let toml_string = toml::to_string(&params).expect("Failed to serialize correct config");
+        let toml_pretty_string =
+            toml::to_string_pretty(&params).expect("Failed to serialize correct config");
+        assert_eq!(
+            r#"listen_pg_addr = 'listen_pg_addr_VALUE'
+listen_http_addr = 'listen_http_addr_VALUE'
+checkpoint_distance = 'checkpoint_distance_VALUE'
+checkpoint_period = 'checkpoint_period_VALUE'
+gc_horizon = 'gc_horizon_VALUE'
+gc_period = 'gc_period_VALUE'
+pg_distrib_dir = 'pg_distrib_dir_VALUE'
+auth_validation_public_key_path = 'auth_validation_public_key_path_VALUE'
+auth_type = 'auth_type_VALUE'
+relish_storage_max_concurrent_sync = 'relish_storage_max_concurrent_sync_VALUE'
+
+[relish_storage]
+local_path = 'relish_storage_local_VALUE'
+"#,
+            toml_pretty_string
+        );
+
+        let params_from_serialized: CfgFileParams = toml::from_str(&toml_string)
+            .expect("Failed to deserialize the serialization result of the config");
+        let params_from_serialized_pretty: CfgFileParams = toml::from_str(&toml_pretty_string)
+            .expect("Failed to deserialize the prettified serialization result of the config");
+        assert!(
+            params_from_serialized == params,
+            "Expected the same config in the end of config -> serialize -> deserialize chain"
+        );
+        assert!(
+            params_from_serialized_pretty == params,
+            "Expected the same config in the end of config -> serialize pretty -> deserialize chain"
+        );
+    }
+
+    #[test]
+    fn credentials_omitted_during_serialization() {
+        let params = CfgFileParams {
+            listen_pg_addr: Some("listen_pg_addr_VALUE".to_string()),
+            listen_http_addr: Some("listen_http_addr_VALUE".to_string()),
+            checkpoint_distance: Some("checkpoint_distance_VALUE".to_string()),
+            checkpoint_period: Some("checkpoint_period_VALUE".to_string()),
+            gc_horizon: Some("gc_horizon_VALUE".to_string()),
+            gc_period: Some("gc_period_VALUE".to_string()),
+            pg_distrib_dir: Some("pg_distrib_dir_VALUE".to_string()),
+            auth_validation_public_key_path: Some(
+                "auth_validation_public_key_path_VALUE".to_string(),
+            ),
+            auth_type: Some("auth_type_VALUE".to_string()),
+            relish_storage: Some(RelishStorage::AwsS3 {
+                bucket_name: "bucket_name_VALUE".to_string(),
+                bucket_region: "bucket_region_VALUE".to_string(),
+                access_key_id: Some("access_key_id_VALUE".to_string()),
+                secret_access_key: Some("secret_access_key_VALUE".to_string()),
+            }),
+            relish_storage_max_concurrent_sync: Some(
+                "relish_storage_max_concurrent_sync_VALUE".to_string(),
+            ),
+        };
+
+        let toml_string = toml::to_string(&params).expect("Failed to serialize correct config");
+        let toml_pretty_string =
+            toml::to_string_pretty(&params).expect("Failed to serialize correct config");
+        assert_eq!(
+            r#"listen_pg_addr = 'listen_pg_addr_VALUE'
+listen_http_addr = 'listen_http_addr_VALUE'
+checkpoint_distance = 'checkpoint_distance_VALUE'
+checkpoint_period = 'checkpoint_period_VALUE'
+gc_horizon = 'gc_horizon_VALUE'
+gc_period = 'gc_period_VALUE'
+pg_distrib_dir = 'pg_distrib_dir_VALUE'
+auth_validation_public_key_path = 'auth_validation_public_key_path_VALUE'
+auth_type = 'auth_type_VALUE'
+relish_storage_max_concurrent_sync = 'relish_storage_max_concurrent_sync_VALUE'
+
+[relish_storage]
+bucket_name = 'bucket_name_VALUE'
+bucket_region = 'bucket_region_VALUE'
+"#,
+            toml_pretty_string
+        );
+
+        let params_from_serialized: CfgFileParams = toml::from_str(&toml_string)
+            .expect("Failed to deserialize the serialization result of the config");
+        let params_from_serialized_pretty: CfgFileParams = toml::from_str(&toml_pretty_string)
+            .expect("Failed to deserialize the prettified serialization result of the config");
+
+        let mut expected_params = params;
+        expected_params.relish_storage = Some(RelishStorage::AwsS3 {
+            bucket_name: "bucket_name_VALUE".to_string(),
+            bucket_region: "bucket_region_VALUE".to_string(),
+            access_key_id: None,
+            secret_access_key: None,
+        });
+        assert!(
+            params_from_serialized == expected_params,
+            "Expected the config without credentials in the end of a 'config -> serialize -> deserialize' chain"
+        );
+        assert!(
+            params_from_serialized_pretty == expected_params,
+            "Expected the config without credentials in the end of a 'config -> serialize pretty -> deserialize' chain"
+        );
+    }
 }

pageserver/src/bin/update_metadata.rs

@@ -1,72 +0,0 @@
-//! Main entry point for the edit_metadata executable
-//!
-//! A handy tool for debugging, that's all.
-use anyhow::Result;
-use clap::{App, Arg};
-use pageserver::layered_repository::metadata::TimelineMetadata;
-use std::path::PathBuf;
-use std::str::FromStr;
-use zenith_utils::lsn::Lsn;
-use zenith_utils::GIT_VERSION;
-
-fn main() -> Result<()> {
-    let arg_matches = App::new("Zenith update metadata utility")
-        .about("Dump or update metadata file")
-        .version(GIT_VERSION)
-        .arg(
-            Arg::with_name("path")
-                .help("Path to metadata file")
-                .required(true),
-        )
-        .arg(
-            Arg::with_name("disk_lsn")
-                .short("d")
-                .long("disk_lsn")
-                .takes_value(true)
-                .help("Replace disk constistent lsn"),
-        )
-        .arg(
-            Arg::with_name("prev_lsn")
-                .short("p")
-                .long("prev_lsn")
-                .takes_value(true)
-                .help("Previous record LSN"),
-        )
-        .get_matches();
-
-    let path = PathBuf::from(arg_matches.value_of("path").unwrap());
-    let metadata_bytes = std::fs::read(&path)?;
-    let mut meta = TimelineMetadata::from_bytes(&metadata_bytes)?;
-    println!("Current metadata:\n{:?}", &meta);
-
-    let mut update_meta = false;
-
-    if let Some(disk_lsn) = arg_matches.value_of("disk_lsn") {
-        meta = TimelineMetadata::new(
-            Lsn::from_str(disk_lsn)?,
-            meta.prev_record_lsn(),
-            meta.ancestor_timeline(),
-            meta.ancestor_lsn(),
-            meta.latest_gc_cutoff_lsn(),
-            meta.initdb_lsn(),
-        );
-        update_meta = true;
-    }
-
-    if let Some(prev_lsn) = arg_matches.value_of("prev_lsn") {
-        meta = TimelineMetadata::new(
-            meta.disk_consistent_lsn(),
-            Some(Lsn::from_str(prev_lsn)?),
-            meta.ancestor_timeline(),
-            meta.ancestor_lsn(),
-            meta.latest_gc_cutoff_lsn(),
-            meta.initdb_lsn(),
-        );
-        update_meta = true;
-    }
-    if update_meta {
-        let metadata_bytes = meta.to_bytes()?;
-        std::fs::write(&path, &metadata_bytes)?;
-    }
-    Ok(())
-}

pageserver/src/branches.rs

@@ -4,7 +4,7 @@
 // TODO: move all paths construction to conf impl
 //
 
-use anyhow::{anyhow, bail, Context, Result};
+use anyhow::{bail, Context, Result};
 use postgres_ffi::ControlFileData;
 use serde::{Deserialize, Serialize};
 use std::{
@@ -21,11 +21,11 @@ use zenith_utils::logging;
 use zenith_utils::lsn::Lsn;
 use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
+use crate::tenant_mgr;
 use crate::walredo::WalRedoManager;
 use crate::CheckpointConfig;
-use crate::{config::PageServerConf, repository::Repository};
-use crate::{import_datadir, LOG_FILE_NAME};
-use crate::{repository::RepositoryTimeline, tenant_mgr};
+use crate::{repository::Repository, PageServerConf};
+use crate::{restore_local_repo, LOG_FILE_NAME};
 
 #[derive(Serialize, Deserialize, Clone)]
 pub struct BranchInfo {
@@ -42,6 +42,8 @@ pub struct BranchInfo {
 impl BranchInfo {
     pub fn from_path<T: AsRef<Path>>(
         path: T,
+        conf: &PageServerConf,
+        tenantid: &ZTenantId,
         repo: &Arc<dyn Repository>,
         include_non_incremental_logical_size: bool,
     ) -> Result<Self> {
@@ -54,21 +56,29 @@ impl BranchInfo {
             .to_string();
         let timeline_id = std::fs::read_to_string(path)?.parse::<ZTimelineId>()?;
 
-        let timeline = match repo.get_timeline(timeline_id)? {
-            RepositoryTimeline::Local(local_entry) => local_entry,
-            RepositoryTimeline::Remote { .. } => {
-                bail!("Timeline {} is remote, no branches to display", timeline_id)
-            }
-        };
+        let timeline = repo.get_timeline(timeline_id)?;
 
-        // we use ancestor lsn zero if we don't have an ancestor, so turn this into an option based on timeline id
-        let (ancestor_id, ancestor_lsn) = match timeline.get_ancestor_timeline_id() {
-            Some(ancestor_id) => (
-                Some(ancestor_id.to_string()),
-                Some(timeline.get_ancestor_lsn().to_string()),
-            ),
-            None => (None, None),
-        };
+        let ancestor_path = conf.ancestor_path(&timeline_id, tenantid);
+        let mut ancestor_id: Option<String> = None;
+        let mut ancestor_lsn: Option<String> = None;
+
+        if ancestor_path.exists() {
+            let ancestor = std::fs::read_to_string(ancestor_path)?;
+            let mut strings = ancestor.split('@');
+
+            ancestor_id = Some(
+                strings
+                    .next()
+                    .with_context(|| "wrong branch ancestor point in time format")?
+                    .to_owned(),
+            );
+            ancestor_lsn = Some(
+                strings
+                    .next()
+                    .with_context(|| "wrong branch ancestor point in time format")?
+                    .to_owned(),
+            );
+        }
 
         // non incremental size calculation can be heavy, so let it be optional
         // needed for tests to check size calculation
@@ -144,23 +154,19 @@ pub fn create_repo(
 
     info!("created directory structure in {}", repo_dir.display());
 
-    // create a new timeline directory
-    let timeline_id = ZTimelineId::generate();
-    let timelinedir = conf.timeline_path(&timeline_id, &tenantid);
-
-    crashsafe_dir::create_dir(&timelinedir)?;
+    let tli = create_timeline(conf, None, &tenantid)?;
 
     let repo = Arc::new(crate::layered_repository::LayeredRepository::new(
         conf,
         wal_redo_manager,
         tenantid,
-        conf.remote_storage_config.is_some(),
+        false,
     ));
 
     // Load data into pageserver
     // TODO To implement zenith import we need to
     //      move data loading out of create_repo()
-    bootstrap_timeline(conf, tenantid, timeline_id, repo.as_ref())?;
+    bootstrap_timeline(conf, tenantid, tli, repo.as_ref())?;
 
     Ok(repo)
 }
@@ -185,7 +191,6 @@ fn run_initdb(conf: &'static PageServerConf, initdbpath: &Path) -> Result<()> {
     let initdb_output = Command::new(initdb_path)
         .args(&["-D", initdbpath.to_str().unwrap()])
         .args(&["-U", &conf.superuser])
-        .args(&["-E", "utf8"])
         .arg("--no-instructions")
         // This is only used for a temporary installation that is deleted shortly after,
         // so no need to fsync it
@@ -228,10 +233,8 @@ fn bootstrap_timeline(
 
     // Import the contents of the data directory at the initial checkpoint
     // LSN, and any WAL after that.
-    // Initdb lsn will be equal to last_record_lsn which will be set after import.
-    // Because we know it upfront avoid having an option or dummy zero value by passing it to create_empty_timeline.
-    let timeline = repo.create_empty_timeline(tli, lsn)?;
-    import_datadir::import_timeline_from_postgres_datadir(
+    let timeline = repo.create_empty_timeline(tli)?;
+    restore_local_repo::import_timeline_from_postgres_datadir(
         &pgdata_path,
         timeline.writer().as_ref(),
         lsn,
@@ -265,24 +268,13 @@ pub(crate) fn get_branches(
     // with timeline_id.
     let branches_dir = conf.branches_path(tenantid);
 
-    std::fs::read_dir(&branches_dir)
-        .with_context(|| {
-            format!(
-                "Found no branches directory '{}' for tenant {}",
-                branches_dir.display(),
-                tenantid
-            )
-        })?
+    std::fs::read_dir(&branches_dir)?
         .map(|dir_entry_res| {
-            let dir_entry = dir_entry_res.with_context(|| {
-                format!(
-                    "Failed to list branches directory '{}' content for tenant {}",
-                    branches_dir.display(),
-                    tenantid
-                )
-            })?;
+            let dir_entry = dir_entry_res?;
             BranchInfo::from_path(
                 dir_entry.path(),
+                conf,
+                tenantid,
                 &repo,
                 include_non_incremental_logical_size,
             )
@@ -303,10 +295,7 @@ pub(crate) fn create_branch(
     }
 
     let mut startpoint = parse_point_in_time(conf, startpoint_str, tenantid)?;
-    let timeline = repo
-        .get_timeline(startpoint.timelineid)?
-        .local_timeline()
-        .ok_or_else(|| anyhow!("Cannot branch off the timeline that's not present locally"))?;
+    let timeline = repo.get_timeline(startpoint.timelineid)?;
     if startpoint.lsn == Lsn(0) {
         // Find end of WAL on the old timeline
         let end_of_wal = timeline.get_last_record_lsn();
@@ -331,24 +320,24 @@ pub(crate) fn create_branch(
         );
     }
 
-    let new_timeline_id = ZTimelineId::generate();
+    // create a new timeline directory for it
+    let newtli = create_timeline(conf, Some(startpoint), tenantid)?;
 
-    // Forward entire timeline creation routine to repository
-    // backend, so it can do all needed initialization
-    repo.branch_timeline(startpoint.timelineid, new_timeline_id, startpoint.lsn)?;
+    // Let the Repository backend do its initialization
+    repo.branch_timeline(startpoint.timelineid, newtli, startpoint.lsn)?;
 
     // Remember the human-readable branch name for the new timeline.
     // FIXME: there's a race condition, if you create a branch with the same
     // name concurrently.
-    let data = new_timeline_id.to_string();
+    let data = newtli.to_string();
     fs::write(conf.branch_path(branchname, tenantid), data)?;
 
     Ok(BranchInfo {
         name: branchname.to_string(),
-        timeline_id: new_timeline_id,
+        timeline_id: newtli,
         latest_valid_lsn: startpoint.lsn,
-        ancestor_id: Some(startpoint.timelineid.to_string()),
-        ancestor_lsn: Some(startpoint.lsn.to_string()),
+        ancestor_id: None,
+        ancestor_lsn: None,
         current_logical_size: 0,
         current_logical_size_non_incremental: Some(0),
     })
@@ -426,3 +415,24 @@ fn parse_point_in_time(
 
     bail!("could not parse point-in-time {}", s);
 }
+
+fn create_timeline(
+    conf: &PageServerConf,
+    ancestor: Option<PointInTime>,
+    tenantid: &ZTenantId,
+) -> Result<ZTimelineId> {
+    // Create initial timeline
+
+    let timelineid = ZTimelineId::generate();
+
+    let timelinedir = conf.timeline_path(&timelineid, tenantid);
+
+    fs::create_dir(&timelinedir)?;
+
+    if let Some(ancestor) = ancestor {
+        let data = format!("{}@{}", ancestor.timelineid, ancestor.lsn);
+        fs::write(timelinedir.join("ancestor"), data)?;
+    }
+
+    Ok(timelineid)
+}

pageserver/src/config.rs

@@ -1,662 +0,0 @@
-//! Functions for handling page server configuration options
-//!
-//! Configuration options can be set in the pageserver.toml configuration
-//! file, or on the command line.
-//! See also `settings.md` for better description on every parameter.
-
-use anyhow::{anyhow, bail, ensure, Context, Result};
-use toml_edit;
-use toml_edit::{Document, Item};
-use zenith_utils::postgres_backend::AuthType;
-use zenith_utils::zid::{ZTenantId, ZTimelineId};
-
-use std::convert::TryInto;
-use std::env;
-use std::num::{NonZeroU32, NonZeroUsize};
-use std::path::{Path, PathBuf};
-use std::str::FromStr;
-use std::time::Duration;
-
-use crate::layered_repository::TIMELINES_SEGMENT_NAME;
-
-pub mod defaults {
-    use const_format::formatcp;
-
-    pub const DEFAULT_PG_LISTEN_PORT: u16 = 64000;
-    pub const DEFAULT_PG_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_PG_LISTEN_PORT}");
-    pub const DEFAULT_HTTP_LISTEN_PORT: u16 = 9898;
-    pub const DEFAULT_HTTP_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_HTTP_LISTEN_PORT}");
-
-    // FIXME: This current value is very low. I would imagine something like 1 GB or 10 GB
-    // would be more appropriate. But a low value forces the code to be exercised more,
-    // which is good for now to trigger bugs.
-    pub const DEFAULT_CHECKPOINT_DISTANCE: u64 = 256 * 1024 * 1024;
-    pub const DEFAULT_CHECKPOINT_PERIOD: &str = "1 s";
-
-    pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
-    pub const DEFAULT_GC_PERIOD: &str = "100 s";
-
-    pub const DEFAULT_SUPERUSER: &str = "zenith_admin";
-    pub const DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNC: usize = 100;
-    pub const DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS: u32 = 10;
-
-    pub const DEFAULT_PAGE_CACHE_SIZE: usize = 8192;
-    pub const DEFAULT_MAX_FILE_DESCRIPTORS: usize = 100;
-
-    ///
-    /// Default built-in configuration file.
-    ///
-    pub const DEFAULT_CONFIG_FILE: &str = formatcp!(
-        r###"
-# Initial configuration file created by 'pageserver --init'
-
-#listen_pg_addr = '{DEFAULT_PG_LISTEN_ADDR}'
-#listen_http_addr = '{DEFAULT_HTTP_LISTEN_ADDR}'
-
-#checkpoint_distance = {DEFAULT_CHECKPOINT_DISTANCE} # in bytes
-#checkpoint_period = '{DEFAULT_CHECKPOINT_PERIOD}'
-
-#gc_period = '{DEFAULT_GC_PERIOD}'
-#gc_horizon = {DEFAULT_GC_HORIZON}
-
-#max_file_descriptors = {DEFAULT_MAX_FILE_DESCRIPTORS}
-
-# initial superuser role name to use when creating a new tenant
-#initial_superuser_name = '{DEFAULT_SUPERUSER}'
-
-# [remote_storage]
-
-"###
-    );
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct PageServerConf {
-    /// Example (default): 127.0.0.1:64000
-    pub listen_pg_addr: String,
-    /// Example (default): 127.0.0.1:9898
-    pub listen_http_addr: String,
-
-    // Flush out an inmemory layer, if it's holding WAL older than this
-    // This puts a backstop on how much WAL needs to be re-digested if the
-    // page server crashes.
-    pub checkpoint_distance: u64,
-    pub checkpoint_period: Duration,
-
-    pub gc_horizon: u64,
-    pub gc_period: Duration,
-    pub superuser: String,
-
-    pub page_cache_size: usize,
-    pub max_file_descriptors: usize,
-
-    // Repository directory, relative to current working directory.
-    // Normally, the page server changes the current working directory
-    // to the repository, and 'workdir' is always '.'. But we don't do
-    // that during unit testing, because the current directory is global
-    // to the process but different unit tests work on different
-    // repositories.
-    pub workdir: PathBuf,
-
-    pub pg_distrib_dir: PathBuf,
-
-    pub auth_type: AuthType,
-
-    pub auth_validation_public_key_path: Option<PathBuf>,
-    pub remote_storage_config: Option<RemoteStorageConfig>,
-}
-
-/// External backup storage configuration, enough for creating a client for that storage.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct RemoteStorageConfig {
-    /// Max allowed number of concurrent sync operations between pageserver and the remote storage.
-    pub max_concurrent_sync: NonZeroUsize,
-    /// Max allowed errors before the sync task is considered failed and evicted.
-    pub max_sync_errors: NonZeroU32,
-    /// The storage connection configuration.
-    pub storage: RemoteStorageKind,
-}
-
-/// A kind of a remote storage to connect to, with its connection configuration.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum RemoteStorageKind {
-    /// Storage based on local file system.
-    /// Specify a root folder to place all stored relish data into.
-    LocalFs(PathBuf),
-    /// AWS S3 based storage, storing all relishes into the root
-    /// of the S3 bucket from the config.
-    AwsS3(S3Config),
-}
-
-/// AWS S3 bucket coordinates and access credentials to manage the bucket contents (read and write).
-#[derive(Clone, PartialEq, Eq)]
-pub struct S3Config {
-    /// Name of the bucket to connect to.
-    pub bucket_name: String,
-    /// The region where the bucket is located at.
-    pub bucket_region: String,
-    /// "Login" to use when connecting to bucket.
-    /// Can be empty for cases like AWS k8s IAM
-    /// where we can allow certain pods to connect
-    /// to the bucket directly without any credentials.
-    pub access_key_id: Option<String>,
-    /// "Password" to use when connecting to bucket.
-    pub secret_access_key: Option<String>,
-}
-
-impl std::fmt::Debug for S3Config {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("S3Config")
-            .field("bucket_name", &self.bucket_name)
-            .field("bucket_region", &self.bucket_region)
-            .finish()
-    }
-}
-
-impl PageServerConf {
-    //
-    // Repository paths, relative to workdir.
-    //
-
-    pub fn tenants_path(&self) -> PathBuf {
-        self.workdir.join("tenants")
-    }
-
-    pub fn tenant_path(&self, tenantid: &ZTenantId) -> PathBuf {
-        self.tenants_path().join(tenantid.to_string())
-    }
-
-    pub fn tags_path(&self, tenantid: &ZTenantId) -> PathBuf {
-        self.tenant_path(tenantid).join("refs").join("tags")
-    }
-
-    pub fn tag_path(&self, tag_name: &str, tenantid: &ZTenantId) -> PathBuf {
-        self.tags_path(tenantid).join(tag_name)
-    }
-
-    pub fn branches_path(&self, tenantid: &ZTenantId) -> PathBuf {
-        self.tenant_path(tenantid).join("refs").join("branches")
-    }
-
-    pub fn branch_path(&self, branch_name: &str, tenantid: &ZTenantId) -> PathBuf {
-        self.branches_path(tenantid).join(branch_name)
-    }
-
-    pub fn timelines_path(&self, tenantid: &ZTenantId) -> PathBuf {
-        self.tenant_path(tenantid).join(TIMELINES_SEGMENT_NAME)
-    }
-
-    pub fn timeline_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
-        self.timelines_path(tenantid).join(timelineid.to_string())
-    }
-
-    pub fn ancestor_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
-        self.timeline_path(timelineid, tenantid).join("ancestor")
-    }
-
-    //
-    // Postgres distribution paths
-    //
-
-    pub fn pg_bin_dir(&self) -> PathBuf {
-        self.pg_distrib_dir.join("bin")
-    }
-
-    pub fn pg_lib_dir(&self) -> PathBuf {
-        self.pg_distrib_dir.join("lib")
-    }
-
-    /// Parse a configuration file (pageserver.toml) into a PageServerConf struct,
-    /// validating the input and failing on errors.
-    ///
-    /// This leaves any options not present in the file in the built-in defaults.
-    pub fn parse_and_validate(toml: &Document, workdir: &Path) -> Result<Self> {
-        use defaults::*;
-
-        let mut conf = PageServerConf {
-            workdir: workdir.to_path_buf(),
-
-            listen_pg_addr: DEFAULT_PG_LISTEN_ADDR.to_string(),
-            listen_http_addr: DEFAULT_HTTP_LISTEN_ADDR.to_string(),
-            checkpoint_distance: DEFAULT_CHECKPOINT_DISTANCE,
-            checkpoint_period: humantime::parse_duration(DEFAULT_CHECKPOINT_PERIOD)?,
-            gc_horizon: DEFAULT_GC_HORIZON,
-            gc_period: humantime::parse_duration(DEFAULT_GC_PERIOD)?,
-            page_cache_size: DEFAULT_PAGE_CACHE_SIZE,
-            max_file_descriptors: DEFAULT_MAX_FILE_DESCRIPTORS,
-
-            pg_distrib_dir: PathBuf::new(),
-            auth_validation_public_key_path: None,
-            auth_type: AuthType::Trust,
-
-            remote_storage_config: None,
-
-            superuser: DEFAULT_SUPERUSER.to_string(),
-        };
-
-        for (key, item) in toml.iter() {
-            match key {
-                "listen_pg_addr" => conf.listen_pg_addr = parse_toml_string(key, item)?,
-                "listen_http_addr" => conf.listen_http_addr = parse_toml_string(key, item)?,
-                "checkpoint_distance" => conf.checkpoint_distance = parse_toml_u64(key, item)?,
-                "checkpoint_period" => conf.checkpoint_period = parse_toml_duration(key, item)?,
-                "gc_horizon" => conf.gc_horizon = parse_toml_u64(key, item)?,
-                "gc_period" => conf.gc_period = parse_toml_duration(key, item)?,
-                "initial_superuser_name" => conf.superuser = parse_toml_string(key, item)?,
-                "page_cache_size" => conf.page_cache_size = parse_toml_u64(key, item)? as usize,
-                "max_file_descriptors" => {
-                    conf.max_file_descriptors = parse_toml_u64(key, item)? as usize
-                }
-                "pg_distrib_dir" => {
-                    conf.pg_distrib_dir = PathBuf::from(parse_toml_string(key, item)?)
-                }
-                "auth_validation_public_key_path" => {
-                    conf.auth_validation_public_key_path =
-                        Some(PathBuf::from(parse_toml_string(key, item)?))
-                }
-                "auth_type" => conf.auth_type = parse_toml_auth_type(key, item)?,
-                "remote_storage" => {
-                    conf.remote_storage_config = Some(Self::parse_remote_storage_config(item)?)
-                }
-                _ => bail!("unrecognized pageserver option '{}'", key),
-            }
-        }
-
-        if conf.auth_type == AuthType::ZenithJWT {
-            let auth_validation_public_key_path = conf
-                .auth_validation_public_key_path
-                .get_or_insert_with(|| workdir.join("auth_public_key.pem"));
-            ensure!(
-                auth_validation_public_key_path.exists(),
-                format!(
-                    "Can't find auth_validation_public_key at '{}'",
-                    auth_validation_public_key_path.display()
-                )
-            );
-        }
-
-        if conf.pg_distrib_dir == PathBuf::new() {
-            conf.pg_distrib_dir = env::current_dir()?.join("tmp_install")
-        };
-        if !conf.pg_distrib_dir.join("bin/postgres").exists() {
-            bail!(
-                "Can't find postgres binary at {}",
-                conf.pg_distrib_dir.display()
-            );
-        }
-
-        Ok(conf)
-    }
-
-    /// subroutine of parse_config(), to parse the `[remote_storage]` table.
-    fn parse_remote_storage_config(toml: &toml_edit::Item) -> anyhow::Result<RemoteStorageConfig> {
-        let local_path = toml.get("local_path");
-        let bucket_name = toml.get("bucket_name");
-        let bucket_region = toml.get("bucket_region");
-
-        let max_concurrent_sync: NonZeroUsize = if let Some(s) = toml.get("max_concurrent_sync") {
-            parse_toml_u64("max_concurrent_sync", s)
-                .and_then(|toml_u64| {
-                    toml_u64.try_into().with_context(|| {
-                        format!("'max_concurrent_sync' value {} is too large", toml_u64)
-                    })
-                })
-                .ok()
-                .and_then(NonZeroUsize::new)
-                .ok_or_else(|| {
-                    anyhow!("'max_concurrent_sync' must be a non-zero positive integer")
-                })?
-        } else {
-            NonZeroUsize::new(defaults::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNC).unwrap()
-        };
-        let max_sync_errors: NonZeroU32 = if let Some(s) = toml.get("max_sync_errors") {
-            parse_toml_u64("max_sync_errors", s)
-                .and_then(|toml_u64| {
-                    toml_u64.try_into().with_context(|| {
-                        format!("'max_sync_errors' value {} is too large", toml_u64)
-                    })
-                })
-                .ok()
-                .and_then(NonZeroU32::new)
-                .ok_or_else(|| anyhow!("'max_sync_errors' must be a non-zero positive integer"))?
-        } else {
-            NonZeroU32::new(defaults::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS).unwrap()
-        };
-
-        let storage = match (local_path, bucket_name, bucket_region) {
-            (None, None, None) => bail!("no 'local_path' nor 'bucket_name' option"),
-            (_, Some(_), None) => {
-                bail!("'bucket_region' option is mandatory if 'bucket_name' is given ")
-            }
-            (_, None, Some(_)) => {
-                bail!("'bucket_name' option is mandatory if 'bucket_region' is given ")
-            }
-            (None, Some(bucket_name), Some(bucket_region)) => RemoteStorageKind::AwsS3(S3Config {
-                bucket_name: bucket_name.as_str().unwrap().to_string(),
-                bucket_region: bucket_region.as_str().unwrap().to_string(),
-                access_key_id: toml
-                    .get("access_key_id")
-                    .map(|x| x.as_str().unwrap().to_string()),
-                secret_access_key: toml
-                    .get("secret_access_key")
-                    .map(|x| x.as_str().unwrap().to_string()),
-            }),
-            (Some(local_path), None, None) => {
-                RemoteStorageKind::LocalFs(PathBuf::from(local_path.as_str().unwrap()))
-            }
-            (Some(_), Some(_), _) => bail!("local_path and bucket_name are mutually exclusive"),
-        };
-
-        Ok(RemoteStorageConfig {
-            max_concurrent_sync,
-            max_sync_errors,
-            storage,
-        })
-    }
-
-    #[cfg(test)]
-    pub fn test_repo_dir(test_name: &str) -> PathBuf {
-        PathBuf::from(format!("../tmp_check/test_{}", test_name))
-    }
-
-    #[cfg(test)]
-    pub fn dummy_conf(repo_dir: PathBuf) -> Self {
-        PageServerConf {
-            checkpoint_distance: defaults::DEFAULT_CHECKPOINT_DISTANCE,
-            checkpoint_period: Duration::from_secs(10),
-            gc_horizon: defaults::DEFAULT_GC_HORIZON,
-            gc_period: Duration::from_secs(10),
-            page_cache_size: defaults::DEFAULT_PAGE_CACHE_SIZE,
-            max_file_descriptors: defaults::DEFAULT_MAX_FILE_DESCRIPTORS,
-            listen_pg_addr: defaults::DEFAULT_PG_LISTEN_ADDR.to_string(),
-            listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(),
-            superuser: "zenith_admin".to_string(),
-            workdir: repo_dir,
-            pg_distrib_dir: PathBuf::new(),
-            auth_type: AuthType::Trust,
-            auth_validation_public_key_path: None,
-            remote_storage_config: None,
-        }
-    }
-}
-
-// Helper functions to parse a toml Item
-
-fn parse_toml_string(name: &str, item: &Item) -> Result<String> {
-    let s = item
-        .as_str()
-        .ok_or_else(|| anyhow!("configure option {} is not a string", name))?;
-    Ok(s.to_string())
-}
-
-fn parse_toml_u64(name: &str, item: &Item) -> Result<u64> {
-    // A toml integer is signed, so it cannot represent the full range of an u64. That's OK
-    // for our use, though.
-    let i: i64 = item
-        .as_integer()
-        .ok_or_else(|| anyhow!("configure option {} is not an integer", name))?;
-    if i < 0 {
-        bail!("configure option {} cannot be negative", name);
-    }
-    Ok(i as u64)
-}
-
-fn parse_toml_duration(name: &str, item: &Item) -> Result<Duration> {
-    let s = item
-        .as_str()
-        .ok_or_else(|| anyhow!("configure option {} is not a string", name))?;
-
-    Ok(humantime::parse_duration(s)?)
-}
-
-fn parse_toml_auth_type(name: &str, item: &Item) -> Result<AuthType> {
-    let v = item
-        .as_str()
-        .ok_or_else(|| anyhow!("configure option {} is not a string", name))?;
-    AuthType::from_str(v)
-}
-
-#[cfg(test)]
-mod tests {
-    use std::fs;
-
-    use tempfile::{tempdir, TempDir};
-
-    use super::*;
-
-    const ALL_BASE_VALUES_TOML: &str = r#"
-# Initial configuration file created by 'pageserver --init'
-
-listen_pg_addr = '127.0.0.1:64000'
-listen_http_addr = '127.0.0.1:9898'
-
-checkpoint_distance = 111 # in bytes
-checkpoint_period = '111 s'
-
-gc_period = '222 s'
-gc_horizon = 222
-
-page_cache_size = 444
-max_file_descriptors = 333
-
-# initial superuser role name to use when creating a new tenant
-initial_superuser_name = 'zzzz'
-
-    "#;
-
-    #[test]
-    fn parse_defaults() -> anyhow::Result<()> {
-        let tempdir = tempdir()?;
-        let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?;
-        // we have to create dummy pathes to overcome the validation errors
-        let config_string = format!("pg_distrib_dir='{}'", pg_distrib_dir.display());
-        let toml = config_string.parse()?;
-
-        let parsed_config =
-            PageServerConf::parse_and_validate(&toml, &workdir).unwrap_or_else(|e| {
-                panic!("Failed to parse config '{}', reason: {}", config_string, e)
-            });
-
-        assert_eq!(
-            parsed_config,
-            PageServerConf {
-                listen_pg_addr: defaults::DEFAULT_PG_LISTEN_ADDR.to_string(),
-                listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(),
-                checkpoint_distance: defaults::DEFAULT_CHECKPOINT_DISTANCE,
-                checkpoint_period: humantime::parse_duration(defaults::DEFAULT_CHECKPOINT_PERIOD)?,
-                gc_horizon: defaults::DEFAULT_GC_HORIZON,
-                gc_period: humantime::parse_duration(defaults::DEFAULT_GC_PERIOD)?,
-                superuser: defaults::DEFAULT_SUPERUSER.to_string(),
-                page_cache_size: defaults::DEFAULT_PAGE_CACHE_SIZE,
-                max_file_descriptors: defaults::DEFAULT_MAX_FILE_DESCRIPTORS,
-                workdir,
-                pg_distrib_dir,
-                auth_type: AuthType::Trust,
-                auth_validation_public_key_path: None,
-                remote_storage_config: None,
-            },
-            "Correct defaults should be used when no config values are provided"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn parse_basic_config() -> anyhow::Result<()> {
-        let tempdir = tempdir()?;
-        let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?;
-
-        let config_string = format!(
-            "{}pg_distrib_dir='{}'",
-            ALL_BASE_VALUES_TOML,
-            pg_distrib_dir.display()
-        );
-        let toml = config_string.parse()?;
-
-        let parsed_config =
-            PageServerConf::parse_and_validate(&toml, &workdir).unwrap_or_else(|e| {
-                panic!("Failed to parse config '{}', reason: {}", config_string, e)
-            });
-
-        assert_eq!(
-            parsed_config,
-            PageServerConf {
-                listen_pg_addr: "127.0.0.1:64000".to_string(),
-                listen_http_addr: "127.0.0.1:9898".to_string(),
-                checkpoint_distance: 111,
-                checkpoint_period: Duration::from_secs(111),
-                gc_horizon: 222,
-                gc_period: Duration::from_secs(222),
-                superuser: "zzzz".to_string(),
-                page_cache_size: 444,
-                max_file_descriptors: 333,
-                workdir,
-                pg_distrib_dir,
-                auth_type: AuthType::Trust,
-                auth_validation_public_key_path: None,
-                remote_storage_config: None,
-            },
-            "Should be able to parse all basic config values correctly"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn parse_remote_fs_storage_config() -> anyhow::Result<()> {
-        let tempdir = tempdir()?;
-        let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?;
-
-        let local_storage_path = tempdir.path().join("local_remote_storage");
-
-        let identical_toml_declarations = &[
-            format!(
-                r#"[remote_storage]
-local_path = '{}'"#,
-                local_storage_path.display()
-            ),
-            format!(
-                "remote_storage={{local_path='{}'}}",
-                local_storage_path.display()
-            ),
-        ];
-
-        for remote_storage_config_str in identical_toml_declarations {
-            let config_string = format!(
-                r#"{}
-pg_distrib_dir='{}'
-
-{}"#,
-                ALL_BASE_VALUES_TOML,
-                pg_distrib_dir.display(),
-                remote_storage_config_str,
-            );
-
-            let toml = config_string.parse()?;
-
-            let parsed_remote_storage_config = PageServerConf::parse_and_validate(&toml, &workdir)
-                .unwrap_or_else(|e| {
-                    panic!("Failed to parse config '{}', reason: {}", config_string, e)
-                })
-                .remote_storage_config
-                .expect("Should have remote storage config for the local FS");
-
-            assert_eq!(
-            parsed_remote_storage_config,
-            RemoteStorageConfig {
-                max_concurrent_sync: NonZeroUsize::new(
-                    defaults::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNC
-                )
-                .unwrap(),
-                max_sync_errors: NonZeroU32::new(defaults::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS)
-                    .unwrap(),
-                storage: RemoteStorageKind::LocalFs(local_storage_path.clone()),
-            },
-            "Remote storage config should correctly parse the local FS config and fill other storage defaults"
-        );
-        }
-        Ok(())
-    }
-
-    #[test]
-    fn parse_remote_s3_storage_config() -> anyhow::Result<()> {
-        let tempdir = tempdir()?;
-        let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?;
-
-        let bucket_name = "some-sample-bucket".to_string();
-        let bucket_region = "eu-north-1".to_string();
-        let access_key_id = "SOMEKEYAAAAASADSAH*#".to_string();
-        let secret_access_key = "SOMEsEcReTsd292v".to_string();
-        let max_concurrent_sync = NonZeroUsize::new(111).unwrap();
-        let max_sync_errors = NonZeroU32::new(222).unwrap();
-
-        let identical_toml_declarations = &[
-            format!(
-                r#"[remote_storage]
-max_concurrent_sync = {}
-max_sync_errors = {}
-bucket_name = '{}'
-bucket_region = '{}'
-access_key_id = '{}'
-secret_access_key = '{}'"#,
-                max_concurrent_sync, max_sync_errors, bucket_name, bucket_region, access_key_id, secret_access_key
-            ),
-            format!(
-                "remote_storage={{max_concurrent_sync = {}, max_sync_errors = {}, bucket_name='{}', bucket_region='{}', access_key_id='{}', secret_access_key='{}'}}",
-                max_concurrent_sync, max_sync_errors, bucket_name, bucket_region, access_key_id, secret_access_key
-            ),
-        ];
-
-        for remote_storage_config_str in identical_toml_declarations {
-            let config_string = format!(
-                r#"{}
-pg_distrib_dir='{}'
-
-{}"#,
-                ALL_BASE_VALUES_TOML,
-                pg_distrib_dir.display(),
-                remote_storage_config_str,
-            );
-
-            let toml = config_string.parse()?;
-
-            let parsed_remote_storage_config = PageServerConf::parse_and_validate(&toml, &workdir)
-                .unwrap_or_else(|e| {
-                    panic!("Failed to parse config '{}', reason: {}", config_string, e)
-                })
-                .remote_storage_config
-                .expect("Should have remote storage config for S3");
-
-            assert_eq!(
-                parsed_remote_storage_config,
-                RemoteStorageConfig {
-                    max_concurrent_sync,
-                    max_sync_errors,
-                    storage: RemoteStorageKind::AwsS3(S3Config {
-                        bucket_name: bucket_name.clone(),
-                        bucket_region: bucket_region.clone(),
-                        access_key_id: Some(access_key_id.clone()),
-                        secret_access_key: Some(secret_access_key.clone()),
-                    }),
-                },
-                "Remote storage config should correctly parse the S3 config"
-            );
-        }
-        Ok(())
-    }
-
-    fn prepare_fs(tempdir: &TempDir) -> anyhow::Result<(PathBuf, PathBuf)> {
-        let tempdir_path = tempdir.path();
-
-        let workdir = tempdir_path.join("workdir");
-        fs::create_dir_all(&workdir)?;
-
-        let pg_distrib_dir = tempdir_path.join("pg_distrib");
-        fs::create_dir_all(&pg_distrib_dir)?;
-        let postgres_bin_dir = pg_distrib_dir.join("bin");
-        fs::create_dir_all(&postgres_bin_dir)?;
-        fs::write(postgres_bin_dir.join("postgres"), "I'm postgres, trust me")?;
-
-        Ok((workdir, pg_distrib_dir))
-    }
-}

pageserver/src/http/openapi_spec.yml

@@ -17,98 +17,6 @@ paths:
             application/json:
               schema:
                 type: object
-  /v1/timeline/{tenant_id}:
-    parameters:
-      - name: tenant_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-    get:
-      description: List tenant timelines
-      responses:
-        "200":
-          description: array of brief timeline descriptions
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  # currently, just a timeline id string, but when remote index gets to be accessed
-                  # remote/local timeline field would be added at least
-                  type: string
-        "400":
-          description: Error when no tenant id found in path
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-  /v1/timeline/{tenant_id}/{timeline_id}:
-    parameters:
-      - name: tenant_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-      - name: timeline_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-    get:
-      description: Get timeline info for tenant's remote timeline
-      responses:
-        "200":
-          description: TimelineInfo
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TimelineInfo"
-        "400":
-          description: Error when no tenant id found in path or no branch name
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
   /v1/branch/{tenant_id}:
     parameters:
       - name: tenant_id
@@ -376,36 +284,6 @@ components:
           type: integer
         current_logical_size_non_incremental:
           type: integer
-    TimelineInfo:
-      type: object
-      required:
-        - timeline_id
-        - tenant_id
-        - last_record_lsn
-        - prev_record_lsn
-        - start_lsn
-        - disk_consistent_lsn
-      properties:
-        timeline_id:
-          type: string
-          format: hex
-        tenant_id:
-          type: string
-          format: hex
-        ancestor_timeline_id:
-          type: string
-          format: hex
-        last_record_lsn:
-          type: string
-        prev_record_lsn:
-          type: string
-        start_lsn:
-          type: string
-        disk_consistent_lsn:
-          type: string
-        timeline_state:
-          type: string
-
     Error:
       type: object
       required:

pageserver/src/http/routes.rs

@@ -1,11 +1,10 @@
 use std::sync::Arc;
 
-use anyhow::{bail, Context, Result};
+use anyhow::Result;
 use hyper::header;
 use hyper::StatusCode;
 use hyper::{Body, Request, Response, Uri};
 use routerify::{ext::RequestExt, RouterBuilder};
-use serde::Serialize;
 use tracing::*;
 use zenith_utils::auth::JwtAuth;
 use zenith_utils::http::endpoint::attach_openapi_ui;
@@ -19,14 +18,11 @@ use zenith_utils::http::{
     request::get_request_param,
     request::parse_request_param,
 };
-use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::{opt_display_serde, ZTimelineId};
 
 use super::models::BranchCreateRequest;
 use super::models::TenantCreateRequest;
 use crate::branches::BranchInfo;
-use crate::repository::TimelineSyncState;
-use crate::{branches, config::PageServerConf, tenant_mgr, ZTenantId};
+use crate::{branches, tenant_mgr, PageServerConf, ZTenantId};
 
 #[derive(Debug)]
 struct State {
@@ -136,98 +132,13 @@ async fn branch_detail_handler(request: Request<Body>) -> Result<Response<Body>,
     let response_data = tokio::task::spawn_blocking(move || {
         let _enter = info_span!("branch_detail", tenant = %tenantid, branch=%branch_name).entered();
         let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
-        BranchInfo::from_path(path, &repo, include_non_incremental_logical_size)
-    })
-    .await
-    .map_err(ApiError::from_err)??;
-
-    Ok(json_response(StatusCode::OK, response_data)?)
-}
-
-async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let tenant_id: ZTenantId = parse_request_param(&request, "tenant_id")?;
-    check_permission(&request, Some(tenant_id))?;
-
-    let conf = get_state(&request).conf;
-    let timelines_dir = conf.timelines_path(&tenant_id);
-
-    let mut timelines_dir_contents =
-        tokio::fs::read_dir(&timelines_dir).await.with_context(|| {
-            format!(
-                "Failed to list timelines dir '{}' contents",
-                timelines_dir.display()
-            )
-        })?;
-
-    let mut local_timelines = Vec::new();
-    while let Some(entry) = timelines_dir_contents.next_entry().await.with_context(|| {
-        format!(
-            "Failed to list timelines dir '{}' contents",
-            timelines_dir.display()
+        BranchInfo::from_path(
+            path,
+            conf,
+            &tenantid,
+            &repo,
+            include_non_incremental_logical_size,
         )
-    })? {
-        let entry_path = entry.path();
-        let entry_type = entry.file_type().await.with_context(|| {
-            format!(
-                "Failed to get file type of timeline dirs' entry '{}'",
-                entry_path.display()
-            )
-        })?;
-
-        if entry_type.is_dir() {
-            match entry.file_name().to_string_lossy().parse::<ZTimelineId>() {
-                Ok(timeline_id) => local_timelines.push(timeline_id.to_string()),
-                Err(e) => error!(
-                    "Failed to get parse timeline id from timeline dirs' entry '{}': {}",
-                    entry_path.display(),
-                    e
-                ),
-            }
-        }
-    }
-
-    Ok(json_response(StatusCode::OK, local_timelines)?)
-}
-
-#[derive(Debug, Serialize)]
-struct TimelineInfo {
-    #[serde(with = "hex")]
-    timeline_id: ZTimelineId,
-    #[serde(with = "hex")]
-    tenant_id: ZTenantId,
-    #[serde(with = "opt_display_serde")]
-    ancestor_timeline_id: Option<ZTimelineId>,
-    last_record_lsn: Lsn,
-    prev_record_lsn: Lsn,
-    start_lsn: Lsn,
-    disk_consistent_lsn: Lsn,
-    timeline_state: Option<TimelineSyncState>,
-}
-
-async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let tenant_id: ZTenantId = parse_request_param(&request, "tenant_id")?;
-    check_permission(&request, Some(tenant_id))?;
-
-    let timeline_id: ZTimelineId = parse_request_param(&request, "timeline_id")?;
-
-    let response_data = tokio::task::spawn_blocking(move || {
-        let _enter =
-            info_span!("timeline_detail_handler", tenant = %tenant_id, timeline = %timeline_id)
-                .entered();
-        let repo = tenant_mgr::get_repository_for_tenant(tenant_id)?;
-        match repo.get_timeline(timeline_id)?.local_timeline() {
-            None => bail!("Timeline with id {} is not present locally", timeline_id),
-            Some(timeline) => Ok::<_, anyhow::Error>(TimelineInfo {
-                timeline_id,
-                tenant_id,
-                ancestor_timeline_id: timeline.get_ancestor_timeline_id(),
-                disk_consistent_lsn: timeline.get_disk_consistent_lsn(),
-                last_record_lsn: timeline.get_last_record_lsn(),
-                prev_record_lsn: timeline.get_prev_record_lsn(),
-                start_lsn: timeline.get_start_lsn(),
-                timeline_state: repo.get_timeline_state(timeline_id),
-            }),
-        }
     })
     .await
     .map_err(ApiError::from_err)??;
@@ -291,11 +202,6 @@ pub fn make_router(
     router
         .data(Arc::new(State::new(conf, auth)))
         .get("/v1/status", status_handler)
-        .get("/v1/timeline/:tenant_id", timeline_list_handler)
-        .get(
-            "/v1/timeline/:tenant_id/:timeline_id",
-            timeline_detail_handler,
-        )
         .get("/v1/branch/:tenant_id", branch_list_handler)
         .get("/v1/branch/:tenant_id/:branch_name", branch_detail_handler)
         .post("/v1/branch", branch_create_handler)

pageserver/src/import_datadir.rs

@@ -1,380 +0,0 @@
-//!
-//! Import data and WAL from a PostgreSQL data directory and WAL segments into
-//! a zenith Timeline.
-//!
-use std::fs;
-use std::fs::File;
-use std::io::{Read, Seek, SeekFrom};
-use std::path::{Path, PathBuf};
-
-use anyhow::{anyhow, bail, ensure, Result};
-use bytes::Bytes;
-use tracing::*;
-
-use crate::relish::*;
-use crate::repository::*;
-use crate::walingest::WalIngest;
-use postgres_ffi::relfile_utils::*;
-use postgres_ffi::waldecoder::*;
-use postgres_ffi::xlog_utils::*;
-use postgres_ffi::Oid;
-use postgres_ffi::{pg_constants, ControlFileData, DBState_DB_SHUTDOWNED};
-use zenith_utils::lsn::Lsn;
-
-///
-/// Import all relation data pages from local disk into the repository.
-///
-/// This is currently only used to import a cluster freshly created by initdb.
-/// The code that deals with the checkpoint would not work right if the
-/// cluster was not shut down cleanly.
-pub fn import_timeline_from_postgres_datadir(
-    path: &Path,
-    writer: &dyn TimelineWriter,
-    lsn: Lsn,
-) -> Result<()> {
-    let mut pg_control: Option<ControlFileData> = None;
-
-    // Scan 'global'
-    for direntry in fs::read_dir(path.join("global"))? {
-        let direntry = direntry?;
-        match direntry.file_name().to_str() {
-            None => continue,
-
-            Some("pg_control") => {
-                pg_control = Some(import_control_file(writer, lsn, &direntry.path())?);
-            }
-            Some("pg_filenode.map") => import_nonrel_file(
-                writer,
-                lsn,
-                RelishTag::FileNodeMap {
-                    spcnode: pg_constants::GLOBALTABLESPACE_OID,
-                    dbnode: 0,
-                },
-                &direntry.path(),
-            )?,
-
-            // Load any relation files into the page server
-            _ => import_relfile(
-                &direntry.path(),
-                writer,
-                lsn,
-                pg_constants::GLOBALTABLESPACE_OID,
-                0,
-            )?,
-        }
-    }
-
-    // Scan 'base'. It contains database dirs, the database OID is the filename.
-    // E.g. 'base/12345', where 12345 is the database OID.
-    for direntry in fs::read_dir(path.join("base"))? {
-        let direntry = direntry?;
-
-        //skip all temporary files
-        if direntry.file_name().to_str().unwrap() == "pgsql_tmp" {
-            continue;
-        }
-
-        let dboid = direntry.file_name().to_str().unwrap().parse::<u32>()?;
-
-        for direntry in fs::read_dir(direntry.path())? {
-            let direntry = direntry?;
-            match direntry.file_name().to_str() {
-                None => continue,
-
-                Some("PG_VERSION") => continue,
-                Some("pg_filenode.map") => import_nonrel_file(
-                    writer,
-                    lsn,
-                    RelishTag::FileNodeMap {
-                        spcnode: pg_constants::DEFAULTTABLESPACE_OID,
-                        dbnode: dboid,
-                    },
-                    &direntry.path(),
-                )?,
-
-                // Load any relation files into the page server
-                _ => import_relfile(
-                    &direntry.path(),
-                    writer,
-                    lsn,
-                    pg_constants::DEFAULTTABLESPACE_OID,
-                    dboid,
-                )?,
-            }
-        }
-    }
-    for entry in fs::read_dir(path.join("pg_xact"))? {
-        let entry = entry?;
-        import_slru_file(writer, lsn, SlruKind::Clog, &entry.path())?;
-    }
-    for entry in fs::read_dir(path.join("pg_multixact").join("members"))? {
-        let entry = entry?;
-        import_slru_file(writer, lsn, SlruKind::MultiXactMembers, &entry.path())?;
-    }
-    for entry in fs::read_dir(path.join("pg_multixact").join("offsets"))? {
-        let entry = entry?;
-        import_slru_file(writer, lsn, SlruKind::MultiXactOffsets, &entry.path())?;
-    }
-    for entry in fs::read_dir(path.join("pg_twophase"))? {
-        let entry = entry?;
-        let xid = u32::from_str_radix(entry.path().to_str().unwrap(), 16)?;
-        import_nonrel_file(writer, lsn, RelishTag::TwoPhase { xid }, &entry.path())?;
-    }
-    // TODO: Scan pg_tblspc
-
-    // We're done importing all the data files.
-    writer.advance_last_record_lsn(lsn);
-
-    // We expect the Postgres server to be shut down cleanly.
-    let pg_control = pg_control.ok_or_else(|| anyhow!("pg_control file not found"))?;
-    ensure!(
-        pg_control.state == DBState_DB_SHUTDOWNED,
-        "Postgres cluster was not shut down cleanly"
-    );
-    ensure!(
-        pg_control.checkPointCopy.redo == lsn.0,
-        "unexpected checkpoint REDO pointer"
-    );
-
-    // Import WAL. This is needed even when starting from a shutdown checkpoint, because
-    // this reads the checkpoint record itself, advancing the tip of the timeline to
-    // *after* the checkpoint record. And crucially, it initializes the 'prev_lsn'.
-    import_wal(
-        &path.join("pg_wal"),
-        writer,
-        Lsn(pg_control.checkPointCopy.redo),
-        lsn,
-    )?;
-
-    Ok(())
-}
-
-// subroutine of import_timeline_from_postgres_datadir(), to load one relation file.
-fn import_relfile(
-    path: &Path,
-    timeline: &dyn TimelineWriter,
-    lsn: Lsn,
-    spcoid: Oid,
-    dboid: Oid,
-) -> Result<()> {
-    // Does it look like a relation file?
-    trace!("importing rel file {}", path.display());
-
-    let p = parse_relfilename(path.file_name().unwrap().to_str().unwrap());
-    if let Err(e) = p {
-        warn!("unrecognized file in postgres datadir: {:?} ({})", path, e);
-        return Err(e.into());
-    }
-    let (relnode, forknum, segno) = p.unwrap();
-
-    let mut file = File::open(path)?;
-    let mut buf: [u8; 8192] = [0u8; 8192];
-
-    let mut blknum: u32 = segno * (1024 * 1024 * 1024 / pg_constants::BLCKSZ as u32);
-    loop {
-        let r = file.read_exact(&mut buf);
-        match r {
-            Ok(_) => {
-                let rel = RelTag {
-                    spcnode: spcoid,
-                    dbnode: dboid,
-                    relnode,
-                    forknum,
-                };
-                let tag = RelishTag::Relation(rel);
-                timeline.put_page_image(tag, blknum, lsn, Bytes::copy_from_slice(&buf))?;
-            }
-
-            // TODO: UnexpectedEof is expected
-            Err(err) => match err.kind() {
-                std::io::ErrorKind::UnexpectedEof => {
-                    // reached EOF. That's expected.
-                    // FIXME: maybe check that we read the full length of the file?
-                    break;
-                }
-                _ => {
-                    bail!("error reading file {}: {:#}", path.display(), err);
-                }
-            },
-        };
-        blknum += 1;
-    }
-
-    Ok(())
-}
-
-///
-/// Import a "non-blocky" file into the repository
-///
-/// This is used for small files like the control file, twophase files etc. that
-/// are just slurped into the repository as one blob.
-///
-fn import_nonrel_file(
-    timeline: &dyn TimelineWriter,
-    lsn: Lsn,
-    tag: RelishTag,
-    path: &Path,
-) -> Result<()> {
-    let mut file = File::open(path)?;
-    let mut buffer = Vec::new();
-    // read the whole file
-    file.read_to_end(&mut buffer)?;
-
-    trace!("importing non-rel file {}", path.display());
-
-    timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buffer[..]))?;
-    Ok(())
-}
-
-///
-/// Import pg_control file into the repository.
-///
-/// The control file is imported as is, but we also extract the checkpoint record
-/// from it and store it separated.
-fn import_control_file(
-    timeline: &dyn TimelineWriter,
-    lsn: Lsn,
-    path: &Path,
-) -> Result<ControlFileData> {
-    let mut file = File::open(path)?;
-    let mut buffer = Vec::new();
-    // read the whole file
-    file.read_to_end(&mut buffer)?;
-
-    trace!("importing control file {}", path.display());
-
-    // Import it as ControlFile
-    timeline.put_page_image(
-        RelishTag::ControlFile,
-        0,
-        lsn,
-        Bytes::copy_from_slice(&buffer[..]),
-    )?;
-
-    // Extract the checkpoint record and import it separately.
-    let pg_control = ControlFileData::decode(&buffer)?;
-    let checkpoint_bytes = pg_control.checkPointCopy.encode();
-    timeline.put_page_image(RelishTag::Checkpoint, 0, lsn, checkpoint_bytes)?;
-
-    Ok(pg_control)
-}
-
-///
-/// Import an SLRU segment file
-///
-fn import_slru_file(
-    timeline: &dyn TimelineWriter,
-    lsn: Lsn,
-    slru: SlruKind,
-    path: &Path,
-) -> Result<()> {
-    // Does it look like an SLRU file?
-    let mut file = File::open(path)?;
-    let mut buf: [u8; 8192] = [0u8; 8192];
-    let segno = u32::from_str_radix(path.file_name().unwrap().to_str().unwrap(), 16)?;
-
-    trace!("importing slru file {}", path.display());
-
-    let mut rpageno = 0;
-    loop {
-        let r = file.read_exact(&mut buf);
-        match r {
-            Ok(_) => {
-                timeline.put_page_image(
-                    RelishTag::Slru { slru, segno },
-                    rpageno,
-                    lsn,
-                    Bytes::copy_from_slice(&buf),
-                )?;
-            }
-
-            // TODO: UnexpectedEof is expected
-            Err(err) => match err.kind() {
-                std::io::ErrorKind::UnexpectedEof => {
-                    // reached EOF. That's expected.
-                    // FIXME: maybe check that we read the full length of the file?
-                    break;
-                }
-                _ => {
-                    bail!("error reading file {}: {:#}", path.display(), err);
-                }
-            },
-        };
-        rpageno += 1;
-
-        // TODO: Check that the file isn't unexpectedly large, not larger than SLRU_PAGES_PER_SEGMENT pages
-    }
-
-    Ok(())
-}
-
-/// Scan PostgreSQL WAL files in given directory and load all records between
-/// 'startpoint' and 'endpoint' into the repository.
-fn import_wal(
-    walpath: &Path,
-    writer: &dyn TimelineWriter,
-    startpoint: Lsn,
-    endpoint: Lsn,
-) -> Result<()> {
-    let mut waldecoder = WalStreamDecoder::new(startpoint);
-
-    let mut segno = startpoint.segment_number(pg_constants::WAL_SEGMENT_SIZE);
-    let mut offset = startpoint.segment_offset(pg_constants::WAL_SEGMENT_SIZE);
-    let mut last_lsn = startpoint;
-
-    let mut walingest = WalIngest::new(writer.deref(), startpoint)?;
-
-    while last_lsn <= endpoint {
-        // FIXME: assume postgresql tli 1 for now
-        let filename = XLogFileName(1, segno, pg_constants::WAL_SEGMENT_SIZE);
-        let mut buf = Vec::new();
-
-        // Read local file
-        let mut path = walpath.join(&filename);
-
-        // It could be as .partial
-        if !PathBuf::from(&path).exists() {
-            path = walpath.join(filename + ".partial");
-        }
-
-        // Slurp the WAL file
-        let mut file = File::open(&path)?;
-
-        if offset > 0 {
-            file.seek(SeekFrom::Start(offset as u64))?;
-        }
-
-        let nread = file.read_to_end(&mut buf)?;
-        if nread != pg_constants::WAL_SEGMENT_SIZE - offset as usize {
-            // Maybe allow this for .partial files?
-            error!("read only {} bytes from WAL file", nread);
-        }
-
-        waldecoder.feed_bytes(&buf);
-
-        let mut nrecords = 0;
-        while last_lsn <= endpoint {
-            if let Some((lsn, recdata)) = waldecoder.poll_decode()? {
-                walingest.ingest_record(writer, recdata, lsn)?;
-                last_lsn = lsn;
-
-                nrecords += 1;
-
-                trace!("imported record at {} (end {})", lsn, endpoint);
-            }
-        }
-
-        debug!("imported {} records up to {}", nrecords, last_lsn);
-
-        segno += 1;
-        offset = 0;
-    }
-
-    if last_lsn != startpoint {
-        debug!("reached end of WAL at {}", last_lsn);
-    } else {
-        info!("no WAL to import at {}", last_lsn);
-    }
-
-    Ok(())
-}

pageserver/src/layered_repository/README.md

@@ -1,56 +1,12 @@
 # Overview
 
-The on-disk format is based on immutable files. The page server receives a
-stream of incoming WAL, parses the WAL records to determine which pages they
-apply to, and accumulates the incoming changes in memory. Every now and then,
-the accumulated changes are written out to new immutable files. This process is
-called checkpointing. Old versions of on-disk files that are not needed by any
-timeline are removed by GC process.
-
-The main responsibility of the Page Server is to process the incoming WAL, and
-reprocess it into a format that allows reasonably quick access to any page
-version.
-
-The incoming WAL contains updates to arbitrary pages in the system. The
-distribution depends on the workload: the updates could be totally random, or
-there could be a long stream of updates to a single relation when data is bulk
-loaded, for example, or something in between. The page server slices the
-incoming WAL per relation and page, and packages the sliced WAL into
-suitably-sized "layer files". The layer files contain all the history of the
-database, back to some reasonable retention period. This system replaces the
-base backups and the WAL archive used in a traditional PostgreSQL
-installation. The layer files are immutable, they are not modified in-place
-after creation. New layer files are created for new incoming WAL, and old layer
-files are removed when they are no longer needed. We could also replace layer
-files with new files that contain the same information, merging small files for
-example, but that hasn't been implemented yet.
-
-
-Cloud Storage                   Page Server                   Safekeeper
-                     Local disk                Memory            WAL
-
-|AAAA|               |AAAA|AAAA|               |AA
-|BBBB|               |BBBB|BBBB|               |
-|CCCC|CCCC|  <----   |CCCC|CCCC|CCCC|   <---   |CC     <----   ADEBAABED
-|DDDD|DDDD|          |DDDD|DDDD|               |DDD
-|EEEE|               |EEEE|EEEE|EEEE|          |E
-
-
-In this illustration, WAL is received as a stream from the Safekeeper, from the
-right.  It is immediately captured by the page server and stored quickly in
-memory. The page server memory can be thought of as a quick "reorder buffer",
-used to hold the incoming WAL and reorder it so that we keep the WAL records for
-the same page and relation close to each other.
-
-From the page server memory, whenever enough WAL has been accumulated for one
-relation segment, it is moved to local disk, as a new layer file, and the memory
-is released.
-
-From the local disk, the layers are further copied to Cloud Storage, for
-long-term archival. After a layer has been copied to Cloud Storage, it can be
-removed from local disk, although we currently keep everything locally for fast
-access. If a layer is needed that isn't found locally, it is fetched from Cloud
-Storage and stored in local disk.
+The on-disk format is based on immutable files. The page server
+receives a stream of incoming WAL, parses the WAL records to determine
+which pages they apply to, and accumulates the incoming changes in
+memory. Every now and then, the accumulated changes are written out to
+new immutable files. This process is called checkpointing. Old versions
+of on-disk files that are not needed by any timeline are removed by GC
+process.
 
 # Terms used in layered repository
 
@@ -58,9 +14,32 @@ Storage and stored in local disk.
 - Segment - one slice of a Relish that is stored in a LayeredTimeline.
 - Layer -  specific version of a relish Segment in a range of LSNs.
 
-# Layer map
+Layers can be InMemory or OnDisk:
+- InMemory layer is not durably stored and needs to rebuild from WAL on pageserver start.
+- OnDisk layer is durably stored.
 
-The LayerMap tracks what layers exist for all the relishes in a timeline.
+OnDisk layers can be Image or Delta:
+- ImageLayer represents an image or a snapshot of a segment at one particular LSN.
+- DeltaLayer represents a collection of WAL records or page images in a range of LSNs.
+
+Dropped segments are always represented on disk by DeltaLayer.
+
+LSN range defined by start_lsn and end_lsn:
+- start_lsn is inclusive.
+- end_lsn is exclusive.
+
+For an open in-memory layer, the end_lsn is MAX_LSN. For a frozen
+in-memory layer or a delta layer, it is a valid end bound. An image
+layer represents snapshot at one LSN, so end_lsn is always the
+snapshot LSN + 1
+
+Layers can be open or historical:
+- Open layer is a writeable one. Only InMemory layer can be open.
+FIXME: If open layer is dropped, it is not writeable, so it should be turned into historical, 
+but now it is not implemented - see bug #569.
+- Historical layer is the one that cannot be modified anymore. Now only OnDisk layers can be historical.
+
+- LayerMap - a map that tracks what layers exist for all the relishes in a timeline.
 
 LayerMap consists of two data structures:
 - segs - All the layers keyed by segment tag
@@ -75,55 +54,8 @@ TODO: Are there any exceptions to this?
 For example, timeline.list_rels(lsn) will return all segments that are visible in this timeline at the LSN,
 including ones that were not modified in this timeline and thus don't have a layer in the timeline's LayerMap.
 
-
-# Different kinds of layers
-
-A layer can be in different states:
-
-- Open - a layer where new WAL records can be appended to.
-- Closed - a layer that is read-only, no new WAL records can be appended to it
-- Historic: synonym for closed
-- InMemory: A layer that needs to be rebuilt from WAL on pageserver start.
-To avoid OOM errors, InMemory layers can be spilled to disk into ephemeral file.
-- OnDisk: A layer that is stored on disk. If its end-LSN is older than
-  disk_consistent_lsn, it is known to be fully flushed and fsync'd to local disk.
-- Frozen layer: an in-memory layer that is Closed.
-
-TODO: Clarify the difference between Closed, Historic and Frozen.
-
-There are two kinds of OnDisk layers:
-- ImageLayer represents an image or a snapshot of a 10 MB relish segment, at one particular LSN.
-- DeltaLayer represents a collection of WAL records or page images in a range of LSNs, for one
-  relish segment.
-
-Dropped segments are always represented on disk by DeltaLayer.
-
-# Layer life cycle
-
-LSN range defined by start_lsn and end_lsn:
-- start_lsn is inclusive.
-- end_lsn is exclusive.
-
-For an open in-memory layer, the end_lsn is MAX_LSN. For a frozen in-memory
-layer or a delta layer, it is a valid end bound. An image layer represents
-snapshot at one LSN, so end_lsn is always the snapshot LSN + 1
-
-Every layer starts its life as an Open In-Memory layer. When the page server
-receives the first WAL record for a segment, it creates a new In-Memory layer
-for it, and puts it to the layer map. Later, the layer is old enough, its
-contents are written to disk, as On-Disk layers. This process is called
-"evicting" a layer.
-
-Layer eviction is a two-step process: First, the layer is marked as closed, so
-that it no longer accepts new WAL records, and the layer map is updated
-accordingly. If a new WAL record for that segment arrives after this step, a new
-Open layer is created to hold it. After this first step, the layer is a Closed
-InMemory state. This first step is called "freezing" the layer.
-
-In the second step, new Delta and Image layers are created, containing all the
-data in the Frozen InMemory layer. When the new layers are ready, the original
-frozen layer is replaced with the new layers in the layer map, and the original
-frozen layer is dropped, releasing the memory.
+TODO:
+Describe GC and checkpoint interval settings.
 
 # Layer files (On-disk layers)
 
@@ -434,8 +366,6 @@ is a newer layer file there. TODO: This optimization hasn't been
 implemented! The GC algorithm will currently keep the file on the
 'main' branch anyway, for as long as the child branch exists.
 
-TODO:
-Describe GC and checkpoint interval settings.
 
 # TODO: On LSN ranges
 

pageserver/src/layered_repository/blob.rs

@@ -0,0 +1,45 @@
+use std::{fs::File, io::Write};
+
+use anyhow::Result;
+use bookfile::{BookWriter, BoundedReader, ChapterId, ChapterWriter};
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+pub struct BlobRange {
+    offset: u64,
+    size: usize,
+}
+
+pub fn read_blob(reader: &BoundedReader<&'_ File>, range: &BlobRange) -> Result<Vec<u8>> {
+    let mut buf = vec![0u8; range.size];
+    reader.read_exact_at(&mut buf, range.offset)?;
+    Ok(buf)
+}
+
+pub struct BlobWriter<W> {
+    writer: ChapterWriter<W>,
+    offset: u64,
+}
+
+impl<W: Write> BlobWriter<W> {
+    // This function takes a BookWriter and creates a new chapter to ensure offset is 0.
+    pub fn new(book_writer: BookWriter<W>, chapter_id: impl Into<ChapterId>) -> Self {
+        let writer = book_writer.new_chapter(chapter_id);
+        Self { writer, offset: 0 }
+    }
+
+    pub fn write_blob(&mut self, blob: &[u8]) -> Result<BlobRange> {
+        self.writer.write_all(blob)?;
+
+        let range = BlobRange {
+            offset: self.offset,
+            size: blob.len(),
+        };
+        self.offset += blob.len() as u64;
+        Ok(range)
+    }
+
+    pub fn close(self) -> bookfile::Result<BookWriter<W>> {
+        self.writer.close()
+    }
+}

pageserver/src/layered_repository/delta_layer.rs

@@ -11,7 +11,7 @@
 //! can happen when you create a new branch in the middle of a delta layer, and the WAL
 //! records on the new branch are put in a new delta layer.
 //!
-//! When a delta file needs to be accessed, we slurp the metadata and segsize chapters
+//! When a delta file needs to be accessed, we slurp the metadata and relsize chapters
 //! into memory, into the DeltaLayerInner struct. See load() and unload() functions.
 //! To access a page/WAL record, we search `page_version_metas` for the block # and LSN.
 //! The byte ranges in the metadata can be used to find the page/WAL record in
@@ -35,16 +35,15 @@
 //! file contents in any way.
 //!
 //! A detlta file is constructed using the 'bookfile' crate. Each file consists of two
-//! parts: the page versions and the segment sizes. They are stored as separate chapters.
+//! parts: the page versions and the relation sizes. They are stored as separate chapters.
 //!
-use crate::config::PageServerConf;
+use crate::layered_repository::blob::BlobWriter;
 use crate::layered_repository::filename::{DeltaFileName, PathOrConf};
 use crate::layered_repository::storage_layer::{
-    Layer, PageReconstructData, PageReconstructResult, PageVersion, SegmentBlk, SegmentTag,
-    RELISH_SEG_SIZE,
+    Layer, PageReconstructData, PageReconstructResult, PageVersion, SegmentTag,
 };
-use crate::virtual_file::VirtualFile;
-use crate::walrecord;
+use crate::waldecoder;
+use crate::PageServerConf;
 use crate::{ZTenantId, ZTimelineId};
 use anyhow::{bail, ensure, Result};
 use log::*;
@@ -54,17 +53,19 @@ use zenith_utils::vec_map::VecMap;
 // while being able to use std::fmt::Write's methods
 use std::fmt::Write as _;
 use std::fs;
+use std::fs::File;
 use std::io::{BufWriter, Write};
 use std::ops::Bound::Included;
-use std::os::unix::fs::FileExt;
 use std::path::{Path, PathBuf};
 use std::sync::{Mutex, MutexGuard};
 
-use bookfile::{Book, BookWriter, BoundedReader, ChapterWriter};
+use bookfile::{Book, BookWriter};
 
 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;
 
+use super::blob::{read_blob, BlobRange};
+
 // Magic constant to identify a Zenith delta file
 pub const DELTA_FILE_MAGIC: u32 = 0x5A616E01;
 
@@ -74,7 +75,7 @@ static PAGE_VERSION_METAS_CHAPTER: u64 = 1;
 /// Page/WAL bytes - cannot be interpreted
 /// without PAGE_VERSION_METAS_CHAPTER
 static PAGE_VERSIONS_CHAPTER: u64 = 2;
-static SEG_SIZES_CHAPTER: u64 = 3;
+static REL_SIZES_CHAPTER: u64 = 3;
 
 /// Contains the [`Summary`] struct
 static SUMMARY_CHAPTER: u64 = 4;
@@ -106,18 +107,6 @@ impl From<&DeltaLayer> for Summary {
     }
 }
 
-#[derive(Serialize, Deserialize)]
-struct BlobRange {
-    offset: u64,
-    size: usize,
-}
-
-fn read_blob<F: FileExt>(reader: &BoundedReader<&'_ F>, range: &BlobRange) -> Result<Vec<u8>> {
-    let mut buf = vec![0u8; range.size];
-    reader.read_exact_at(&mut buf, range.offset)?;
-    Ok(buf)
-}
-
 ///
 /// DeltaLayer is the in-memory data structure associated with an
 /// on-disk delta file.  We keep a DeltaLayer in memory for each
@@ -146,39 +135,19 @@ pub struct DeltaLayer {
 }
 
 pub struct DeltaLayerInner {
-    /// If false, the 'page_version_metas' and 'seg_sizes' have not been
+    /// If false, the 'page_version_metas' and 'relsizes' have not been
     /// loaded into memory yet.
     loaded: bool,
 
-    book: Option<Book<VirtualFile>>,
-
     /// All versions of all pages in the file are are kept here.
     /// Indexed by block number and LSN.
-    page_version_metas: VecMap<(SegmentBlk, Lsn), BlobRange>,
+    page_version_metas: VecMap<(u32, Lsn), BlobRange>,
 
-    /// `seg_sizes` tracks the size of the segment at different points in time.
-    seg_sizes: VecMap<Lsn, SegmentBlk>,
-}
-
-impl DeltaLayerInner {
-    fn get_seg_size(&self, lsn: Lsn) -> Result<SegmentBlk> {
-        // Scan the VecMap backwards, starting from the given entry.
-        let slice = self
-            .seg_sizes
-            .slice_range((Included(&Lsn(0)), Included(&lsn)));
-        if let Some((_entry_lsn, entry)) = slice.last() {
-            Ok(*entry)
-        } else {
-            Err(anyhow::anyhow!("could not find seg size in delta layer"))
-        }
-    }
+    /// `relsizes` tracks the size of the relation at different points in time.
+    relsizes: VecMap<Lsn, u32>,
 }
 
 impl Layer for DeltaLayer {
-    fn get_tenant_id(&self) -> ZTenantId {
-        self.tenantid
-    }
-
     fn get_timeline_id(&self) -> ZTimelineId {
         self.timelineid
     }
@@ -206,32 +175,22 @@ impl Layer for DeltaLayer {
     /// Look up given page in the cache.
     fn get_page_reconstruct_data(
         &self,
-        blknum: SegmentBlk,
+        blknum: u32,
         lsn: Lsn,
-        cached_img_lsn: Option<Lsn>,
         reconstruct_data: &mut PageReconstructData,
     ) -> Result<PageReconstructResult> {
         let mut need_image = true;
 
-        assert!((0..RELISH_SEG_SIZE).contains(&blknum));
-
-        match &cached_img_lsn {
-            Some(cached_lsn) if &self.end_lsn <= cached_lsn => {
-                return Ok(PageReconstructResult::Cached)
-            }
-            _ => {}
-        }
+        assert!(self.seg.blknum_in_seg(blknum));
 
         {
             // Open the file and lock the metadata in memory
+            // TODO: avoid opening the file for each read
+            let (_path, book) = self.open_book()?;
+            let page_version_reader = book.chapter_reader(PAGE_VERSIONS_CHAPTER)?;
             let inner = self.load()?;
-            let page_version_reader = inner
-                .book
-                .as_ref()
-                .expect("should be loaded in load call above")
-                .chapter_reader(PAGE_VERSIONS_CHAPTER)?;
 
-            // Scan the metadata VecMap backwards, starting from the given entry.
+            // Scan the metadata BTreeMap backwards, starting from the given entry.
             let minkey = (blknum, Lsn(0));
             let maxkey = (blknum, lsn);
             let iter = inner
@@ -240,65 +199,58 @@ impl Layer for DeltaLayer {
                 .iter()
                 .rev();
             for ((_blknum, pv_lsn), blob_range) in iter {
-                match &cached_img_lsn {
-                    Some(cached_lsn) if pv_lsn <= cached_lsn => {
-                        return Ok(PageReconstructResult::Cached)
-                    }
-                    _ => {}
-                }
-
                 let pv = PageVersion::des(&read_blob(&page_version_reader, blob_range)?)?;
 
-                match pv {
-                    PageVersion::Page(img) => {
-                        // Found a page image, return it
-                        reconstruct_data.page_img = Some(img);
+                if let Some(img) = pv.page_image {
+                    // Found a page image, return it
+                    reconstruct_data.page_img = Some(img);
+                    need_image = false;
+                    break;
+                } else if let Some(rec) = pv.record {
+                    let will_init = rec.will_init;
+                    reconstruct_data.records.push((*pv_lsn, rec));
+                    if will_init {
+                        // This WAL record initializes the page, so no need to go further back
                         need_image = false;
                         break;
                     }
-                    PageVersion::Wal(rec) => {
-                        let will_init = rec.will_init();
-                        reconstruct_data.records.push((*pv_lsn, rec));
-                        if will_init {
-                            // This WAL record initializes the page, so no need to go further back
-                            need_image = false;
-                            break;
-                        }
-                    }
+                } else {
+                    // No base image, and no WAL record. Huh?
+                    bail!("no page image or WAL record for requested page");
                 }
             }
 
-            // If we didn't find any records for this, check if the request is beyond EOF
-            if need_image
-                && reconstruct_data.records.is_empty()
-                && self.seg.rel.is_blocky()
-                && blknum >= inner.get_seg_size(lsn)?
-            {
-                return Ok(PageReconstructResult::Missing(self.start_lsn));
-            }
-
             // release metadata lock and close the file
         }
 
         // If an older page image is needed to reconstruct the page, let the
         // caller know.
         if need_image {
-            Ok(PageReconstructResult::Continue(Lsn(self.start_lsn.0 - 1)))
+            Ok(PageReconstructResult::Continue(self.start_lsn))
         } else {
             Ok(PageReconstructResult::Complete)
         }
     }
 
     /// Get size of the relation at given LSN
-    fn get_seg_size(&self, lsn: Lsn) -> Result<SegmentBlk> {
+    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
         assert!(lsn >= self.start_lsn);
         ensure!(
             self.seg.rel.is_blocky(),
             "get_seg_size() called on a non-blocky rel"
         );
 
+        // Scan the BTreeMap backwards, starting from the given entry.
         let inner = self.load()?;
-        inner.get_seg_size(lsn)
+        let slice = inner
+            .relsizes
+            .slice_range((Included(&Lsn(0)), Included(&lsn)));
+
+        if let Some((_entry_lsn, entry)) = slice.last() {
+            Ok(*entry)
+        } else {
+            Err(anyhow::anyhow!("could not find seg size in delta layer"))
+        }
     }
 
     /// Does this segment exist at given LSN?
@@ -319,13 +271,8 @@ impl Layer for DeltaLayer {
     fn unload(&self) -> Result<()> {
         let mut inner = self.inner.lock().unwrap();
         inner.page_version_metas = VecMap::default();
-        inner.seg_sizes = VecMap::default();
+        inner.relsizes = VecMap::default();
         inner.loaded = false;
-
-        // Note: we keep the Book open. Is that a good idea? The virtual file
-        // machinery has its own rules for closing the file descriptor if it's not
-        // needed, but the Book struct uses up some memory, too.
-
         Ok(())
     }
 
@@ -339,10 +286,6 @@ impl Layer for DeltaLayer {
         true
     }
 
-    fn is_in_memory(&self) -> bool {
-        false
-    }
-
     /// debugging function to print out the contents of the layer
     fn dump(&self) -> Result<()> {
         println!(
@@ -350,17 +293,13 @@ impl Layer for DeltaLayer {
             self.tenantid, self.timelineid, self.seg, self.start_lsn, self.end_lsn
         );
 
-        println!("--- seg sizes ---");
+        println!("--- relsizes ---");
         let inner = self.load()?;
-        for (k, v) in inner.seg_sizes.as_slice() {
+        for (k, v) in inner.relsizes.as_slice() {
             println!("  {}: {}", k, v);
         }
         println!("--- page versions ---");
-
-        let path = self.path();
-        let file = std::fs::File::open(&path)?;
-        let book = Book::new(file)?;
-
+        let (_path, book) = self.open_book()?;
         let chapter = book.chapter_reader(PAGE_VERSIONS_CHAPTER)?;
         for ((blk, lsn), blob_range) in inner.page_version_metas.as_slice() {
             let mut desc = String::new();
@@ -368,22 +307,19 @@ impl Layer for DeltaLayer {
             let buf = read_blob(&chapter, blob_range)?;
             let pv = PageVersion::des(&buf)?;
 
-            match pv {
-                PageVersion::Page(img) => {
-                    write!(&mut desc, " img {} bytes", img.len())?;
-                }
-                PageVersion::Wal(rec) => {
-                    let wal_desc = walrecord::describe_wal_record(&rec);
-                    write!(
-                        &mut desc,
-                        " rec {} bytes will_init: {} {}",
-                        blob_range.size,
-                        rec.will_init(),
-                        wal_desc
-                    )?;
-                }
+            if let Some(img) = pv.page_image.as_ref() {
+                write!(&mut desc, " img {} bytes", img.len())?;
+            }
+            if let Some(rec) = pv.record.as_ref() {
+                let wal_desc = waldecoder::describe_wal_record(&rec.rec);
+                write!(
+                    &mut desc,
+                    " rec {} bytes will_init: {} {}",
+                    rec.rec.len(),
+                    rec.will_init,
+                    wal_desc
+                )?;
             }
-
             println!("  blk {} at {}: {}", blk, lsn, desc);
         }
 
@@ -406,6 +342,114 @@ impl DeltaLayer {
         }
     }
 
+    /// Create a new delta file, using the given page versions and relsizes.
+    /// The page versions are passed by an iterator; the iterator must return
+    /// page versions in blknum+lsn order.
+    ///
+    /// This is used to write the in-memory layer to disk. The in-memory layer uses the same
+    /// data structure with two btreemaps as we do, so passing the btreemaps is currently
+    /// expedient.
+    #[allow(clippy::too_many_arguments)]
+    pub fn create<'a>(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        seg: SegmentTag,
+        start_lsn: Lsn,
+        end_lsn: Lsn,
+        dropped: bool,
+        page_versions: impl Iterator<Item = (u32, Lsn, &'a PageVersion)>,
+        relsizes: VecMap<Lsn, u32>,
+    ) -> Result<DeltaLayer> {
+        if seg.rel.is_blocky() {
+            assert!(!relsizes.is_empty());
+        }
+
+        let delta_layer = DeltaLayer {
+            path_or_conf: PathOrConf::Conf(conf),
+            timelineid,
+            tenantid,
+            seg,
+            start_lsn,
+            end_lsn,
+            dropped,
+            inner: Mutex::new(DeltaLayerInner {
+                loaded: true,
+                page_version_metas: VecMap::default(),
+                relsizes,
+            }),
+        };
+        let mut inner = delta_layer.inner.lock().unwrap();
+
+        // Write the in-memory btreemaps into a file
+        let path = delta_layer.path();
+
+        // Note: This overwrites any existing file. There shouldn't be any.
+        // FIXME: throw an error instead?
+        let file = File::create(&path)?;
+        let buf_writer = BufWriter::new(file);
+        let book = BookWriter::new(buf_writer, DELTA_FILE_MAGIC)?;
+
+        let mut page_version_writer = BlobWriter::new(book, PAGE_VERSIONS_CHAPTER);
+
+        for (blknum, lsn, page_version) in page_versions {
+            let buf = PageVersion::ser(page_version)?;
+            let blob_range = page_version_writer.write_blob(&buf)?;
+
+            inner
+                .page_version_metas
+                .append((blknum, lsn), blob_range)
+                .unwrap();
+        }
+
+        let book = page_version_writer.close()?;
+
+        // Write out page versions
+        let mut chapter = book.new_chapter(PAGE_VERSION_METAS_CHAPTER);
+        let buf = VecMap::ser(&inner.page_version_metas)?;
+        chapter.write_all(&buf)?;
+        let book = chapter.close()?;
+
+        // and relsizes to separate chapter
+        let mut chapter = book.new_chapter(REL_SIZES_CHAPTER);
+        let buf = VecMap::ser(&inner.relsizes)?;
+        chapter.write_all(&buf)?;
+        let book = chapter.close()?;
+
+        let mut chapter = book.new_chapter(SUMMARY_CHAPTER);
+        let summary = Summary {
+            tenantid,
+            timelineid,
+            seg,
+
+            start_lsn,
+            end_lsn,
+
+            dropped,
+        };
+        Summary::ser_into(&summary, &mut chapter)?;
+        let book = chapter.close()?;
+
+        // This flushes the underlying 'buf_writer'.
+        let writer = book.close()?;
+        writer.get_ref().sync_all()?;
+
+        trace!("saved {}", &path.display());
+
+        drop(inner);
+
+        Ok(delta_layer)
+    }
+
+    fn open_book(&self) -> Result<(PathBuf, Book<File>)> {
+        let path = self.path();
+
+        let file = File::open(&path)?;
+        let book = Book::new(file)?;
+
+        Ok((path, book))
+    }
+
     ///
     /// Load the contents of the file into memory
     ///
@@ -417,14 +461,7 @@ impl DeltaLayer {
             return Ok(inner);
         }
 
-        let path = self.path();
-
-        // Open the file if it's not open already.
-        if inner.book.is_none() {
-            let file = VirtualFile::open(&path)?;
-            inner.book = Some(Book::new(file)?);
-        }
-        let book = inner.book.as_ref().unwrap();
+        let (path, book) = self.open_book()?;
 
         match &self.path_or_conf {
             PathOrConf::Conf(_) => {
@@ -454,14 +491,16 @@ impl DeltaLayer {
         let chapter = book.read_chapter(PAGE_VERSION_METAS_CHAPTER)?;
         let page_version_metas = VecMap::des(&chapter)?;
 
-        let chapter = book.read_chapter(SEG_SIZES_CHAPTER)?;
-        let seg_sizes = VecMap::des(&chapter)?;
+        let chapter = book.read_chapter(REL_SIZES_CHAPTER)?;
+        let relsizes = VecMap::des(&chapter)?;
 
         debug!("loaded from {}", &path.display());
 
-        inner.page_version_metas = page_version_metas;
-        inner.seg_sizes = seg_sizes;
-        inner.loaded = true;
+        *inner = DeltaLayerInner {
+            loaded: true,
+            page_version_metas,
+            relsizes,
+        };
 
         Ok(inner)
     }
@@ -483,9 +522,8 @@ impl DeltaLayer {
             dropped: filename.dropped,
             inner: Mutex::new(DeltaLayerInner {
                 loaded: false,
-                book: None,
                 page_version_metas: VecMap::default(),
-                seg_sizes: VecMap::default(),
+                relsizes: VecMap::default(),
             }),
         }
     }
@@ -493,10 +531,7 @@ impl DeltaLayer {
     /// Create a DeltaLayer struct representing an existing file on disk.
     ///
     /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary.
-    pub fn new_for_path<F>(path: &Path, book: &Book<F>) -> Result<Self>
-    where
-        F: std::os::unix::prelude::FileExt,
-    {
+    pub fn new_for_path(path: &Path, book: &Book<File>) -> Result<Self> {
         let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
         let summary = Summary::des(&chapter)?;
 
@@ -510,9 +545,8 @@ impl DeltaLayer {
             dropped: summary.dropped,
             inner: Mutex::new(DeltaLayerInner {
                 loaded: false,
-                book: None,
                 page_version_metas: VecMap::default(),
-                seg_sizes: VecMap::default(),
+                relsizes: VecMap::default(),
             }),
         })
     }
@@ -536,170 +570,3 @@ impl DeltaLayer {
         )
     }
 }
-
-/// A builder object for constructing a new delta layer.
-///
-/// Usage:
-///
-/// 1. Create the DeltaLayerWriter by calling DeltaLayerWriter::new(...)
-///
-/// 2. Write the contents by calling `put_page_version` for every page
-///    version to store in the layer.
-///
-/// 3. Call `finish`.
-///
-pub struct DeltaLayerWriter {
-    conf: &'static PageServerConf,
-    timelineid: ZTimelineId,
-    tenantid: ZTenantId,
-    seg: SegmentTag,
-    start_lsn: Lsn,
-    end_lsn: Lsn,
-    dropped: bool,
-
-    page_version_writer: ChapterWriter<BufWriter<VirtualFile>>,
-    pv_offset: u64,
-
-    page_version_metas: VecMap<(SegmentBlk, Lsn), BlobRange>,
-}
-
-impl DeltaLayerWriter {
-    ///
-    /// Start building a new delta layer.
-    ///
-    pub fn new(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        seg: SegmentTag,
-        start_lsn: Lsn,
-        end_lsn: Lsn,
-        dropped: bool,
-    ) -> Result<DeltaLayerWriter> {
-        // Create the file
-        //
-        // Note: This overwrites any existing file. There shouldn't be any.
-        // FIXME: throw an error instead?
-        let path = DeltaLayer::path_for(
-            &PathOrConf::Conf(conf),
-            timelineid,
-            tenantid,
-            &DeltaFileName {
-                seg,
-                start_lsn,
-                end_lsn,
-                dropped,
-            },
-        );
-        let file = VirtualFile::create(&path)?;
-        let buf_writer = BufWriter::new(file);
-        let book = BookWriter::new(buf_writer, DELTA_FILE_MAGIC)?;
-
-        // Open the page-versions chapter for writing. The calls to
-        // `put_page_version` will use this to write the contents.
-        let page_version_writer = book.new_chapter(PAGE_VERSIONS_CHAPTER);
-
-        Ok(DeltaLayerWriter {
-            conf,
-            timelineid,
-            tenantid,
-            seg,
-            start_lsn,
-            end_lsn,
-            dropped,
-            page_version_writer,
-            page_version_metas: VecMap::default(),
-            pv_offset: 0,
-        })
-    }
-
-    ///
-    /// Append a page version to the file.
-    ///
-    /// 'buf' is a serialized PageVersion.
-    /// The page versions must be appended in blknum, lsn order.
-    ///
-    pub fn put_page_version(&mut self, blknum: SegmentBlk, lsn: Lsn, buf: &[u8]) -> Result<()> {
-        // Remember the offset and size metadata. The metadata is written
-        // to a separate chapter, in `finish`.
-        let blob_range = BlobRange {
-            offset: self.pv_offset,
-            size: buf.len(),
-        };
-        self.page_version_metas
-            .append((blknum, lsn), blob_range)
-            .unwrap();
-
-        // write the page version
-        self.page_version_writer.write_all(buf)?;
-        self.pv_offset += buf.len() as u64;
-
-        Ok(())
-    }
-
-    ///
-    /// Finish writing the delta layer.
-    ///
-    /// 'seg_sizes' is a list of size changes to store with the actual data.
-    ///
-    pub fn finish(self, seg_sizes: VecMap<Lsn, SegmentBlk>) -> Result<DeltaLayer> {
-        // Close the page-versions chapter
-        let book = self.page_version_writer.close()?;
-
-        // Write out page versions metadata
-        let mut chapter = book.new_chapter(PAGE_VERSION_METAS_CHAPTER);
-        let buf = VecMap::ser(&self.page_version_metas)?;
-        chapter.write_all(&buf)?;
-        let book = chapter.close()?;
-
-        if self.seg.rel.is_blocky() {
-            assert!(!seg_sizes.is_empty());
-        }
-
-        // and seg_sizes to separate chapter
-        let mut chapter = book.new_chapter(SEG_SIZES_CHAPTER);
-        let buf = VecMap::ser(&seg_sizes)?;
-        chapter.write_all(&buf)?;
-        let book = chapter.close()?;
-
-        let mut chapter = book.new_chapter(SUMMARY_CHAPTER);
-        let summary = Summary {
-            tenantid: self.tenantid,
-            timelineid: self.timelineid,
-            seg: self.seg,
-
-            start_lsn: self.start_lsn,
-            end_lsn: self.end_lsn,
-
-            dropped: self.dropped,
-        };
-        Summary::ser_into(&summary, &mut chapter)?;
-        let book = chapter.close()?;
-
-        // This flushes the underlying 'buf_writer'.
-        book.close()?;
-
-        // Note: Because we opened the file in write-only mode, we cannot
-        // reuse the same VirtualFile for reading later. That's why we don't
-        // set inner.book here. The first read will have to re-open it.
-        let layer = DeltaLayer {
-            path_or_conf: PathOrConf::Conf(self.conf),
-            tenantid: self.tenantid,
-            timelineid: self.timelineid,
-            seg: self.seg,
-            start_lsn: self.start_lsn,
-            end_lsn: self.end_lsn,
-            dropped: self.dropped,
-            inner: Mutex::new(DeltaLayerInner {
-                loaded: false,
-                book: None,
-                page_version_metas: VecMap::default(),
-                seg_sizes: VecMap::default(),
-            }),
-        };
-
-        trace!("created delta layer {}", &layer.path().display());
-
-        Ok(layer)
-    }
-}

pageserver/src/layered_repository/ephemeral_file.rs

@@ -1,307 +0,0 @@
-//! Implementation of append-only file data structure
-//! used to keep in-memory layers spilled on disk.
-
-use crate::config::PageServerConf;
-use crate::page_cache;
-use crate::page_cache::PAGE_SZ;
-use crate::page_cache::{ReadBufResult, WriteBufResult};
-use crate::virtual_file::VirtualFile;
-use lazy_static::lazy_static;
-use std::cmp::min;
-use std::collections::HashMap;
-use std::fs::OpenOptions;
-use std::io::{Error, ErrorKind, Seek, SeekFrom, Write};
-use std::ops::DerefMut;
-use std::path::PathBuf;
-use std::sync::{Arc, RwLock};
-use zenith_utils::zid::ZTenantId;
-use zenith_utils::zid::ZTimelineId;
-
-use std::os::unix::fs::FileExt;
-
-lazy_static! {
-    ///
-    /// This is the global cache of file descriptors (File objects).
-    ///
-    static ref EPHEMERAL_FILES: RwLock<EphemeralFiles> = RwLock::new(EphemeralFiles {
-        next_file_id: 1,
-        files: HashMap::new(),
-    });
-}
-
-pub struct EphemeralFiles {
-    next_file_id: u64,
-
-    files: HashMap<u64, Arc<VirtualFile>>,
-}
-
-pub struct EphemeralFile {
-    file_id: u64,
-    _tenantid: ZTenantId,
-    _timelineid: ZTimelineId,
-    file: Arc<VirtualFile>,
-
-    pos: u64,
-}
-
-impl EphemeralFile {
-    pub fn create(
-        conf: &PageServerConf,
-        tenantid: ZTenantId,
-        timelineid: ZTimelineId,
-    ) -> Result<EphemeralFile, std::io::Error> {
-        let mut l = EPHEMERAL_FILES.write().unwrap();
-        let file_id = l.next_file_id;
-        l.next_file_id += 1;
-
-        let filename = conf
-            .timeline_path(&timelineid, &tenantid)
-            .join(PathBuf::from(format!("ephemeral-{}", file_id)));
-
-        let file = VirtualFile::open_with_options(
-            &filename,
-            OpenOptions::new().read(true).write(true).create(true),
-        )?;
-        let file_rc = Arc::new(file);
-        l.files.insert(file_id, file_rc.clone());
-
-        Ok(EphemeralFile {
-            file_id,
-            _tenantid: tenantid,
-            _timelineid: timelineid,
-            file: file_rc,
-            pos: 0,
-        })
-    }
-
-    pub fn fill_buffer(&self, buf: &mut [u8], blkno: u32) -> Result<(), Error> {
-        let mut off = 0;
-        while off < PAGE_SZ {
-            let n = self
-                .file
-                .read_at(&mut buf[off..], blkno as u64 * PAGE_SZ as u64 + off as u64)?;
-
-            if n == 0 {
-                // Reached EOF. Fill the rest of the buffer with zeros.
-                const ZERO_BUF: [u8; PAGE_SZ] = [0u8; PAGE_SZ];
-
-                buf[off..].copy_from_slice(&ZERO_BUF[off..]);
-                break;
-            }
-
-            off += n as usize;
-        }
-        Ok(())
-    }
-}
-
-/// Does the given filename look like an ephemeral file?
-pub fn is_ephemeral_file(filename: &str) -> bool {
-    if let Some(rest) = filename.strip_prefix("ephemeral-") {
-        rest.parse::<u32>().is_ok()
-    } else {
-        false
-    }
-}
-
-impl FileExt for EphemeralFile {
-    fn read_at(&self, dstbuf: &mut [u8], offset: u64) -> Result<usize, Error> {
-        // Look up the right page
-        let blkno = (offset / PAGE_SZ as u64) as u32;
-        let off = offset as usize % PAGE_SZ;
-        let len = min(PAGE_SZ - off, dstbuf.len());
-
-        let read_guard;
-        let mut write_guard;
-
-        let cache = page_cache::get();
-        let buf = match cache.read_ephemeral_buf(self.file_id, blkno) {
-            ReadBufResult::Found(guard) => {
-                read_guard = guard;
-                read_guard.as_ref()
-            }
-            ReadBufResult::NotFound(guard) => {
-                // Read the page from disk into the buffer
-                write_guard = guard;
-                self.fill_buffer(write_guard.deref_mut(), blkno)?;
-                write_guard.mark_valid();
-
-                // And then fall through to read the requested slice from the
-                // buffer.
-                write_guard.as_ref()
-            }
-        };
-
-        dstbuf[0..len].copy_from_slice(&buf[off..(off + len)]);
-        Ok(len)
-    }
-
-    fn write_at(&self, srcbuf: &[u8], offset: u64) -> Result<usize, Error> {
-        // Look up the right page
-        let blkno = (offset / PAGE_SZ as u64) as u32;
-        let off = offset as usize % PAGE_SZ;
-        let len = min(PAGE_SZ - off, srcbuf.len());
-
-        let mut write_guard;
-        let cache = page_cache::get();
-        let buf = match cache.write_ephemeral_buf(self.file_id, blkno) {
-            WriteBufResult::Found(guard) => {
-                write_guard = guard;
-                write_guard.deref_mut()
-            }
-            WriteBufResult::NotFound(guard) => {
-                // Read the page from disk into the buffer
-                // TODO: if we're overwriting the whole page, no need to read it in first
-                write_guard = guard;
-                self.fill_buffer(write_guard.deref_mut(), blkno)?;
-                write_guard.mark_valid();
-
-                // And then fall through to modify it.
-                write_guard.deref_mut()
-            }
-        };
-
-        buf[off..(off + len)].copy_from_slice(&srcbuf[0..len]);
-        write_guard.mark_dirty();
-        Ok(len)
-    }
-}
-
-impl Write for EphemeralFile {
-    fn write(&mut self, buf: &[u8]) -> Result<usize, Error> {
-        let n = self.write_at(buf, self.pos)?;
-        self.pos += n as u64;
-        Ok(n)
-    }
-
-    fn flush(&mut self) -> Result<(), std::io::Error> {
-        todo!()
-    }
-}
-
-impl Seek for EphemeralFile {
-    fn seek(&mut self, pos: SeekFrom) -> Result<u64, Error> {
-        match pos {
-            SeekFrom::Start(offset) => {
-                self.pos = offset;
-            }
-            SeekFrom::End(_offset) => {
-                return Err(Error::new(
-                    ErrorKind::Other,
-                    "SeekFrom::End not supported by EphemeralFile",
-                ));
-            }
-            SeekFrom::Current(offset) => {
-                let pos = self.pos as i128 + offset as i128;
-                if pos < 0 {
-                    return Err(Error::new(
-                        ErrorKind::InvalidInput,
-                        "offset would be negative",
-                    ));
-                }
-                if pos > u64::MAX as i128 {
-                    return Err(Error::new(ErrorKind::InvalidInput, "offset overflow"));
-                }
-                self.pos = pos as u64;
-            }
-        }
-        Ok(self.pos)
-    }
-}
-
-impl Drop for EphemeralFile {
-    fn drop(&mut self) {
-        // drop all pages from page cache
-        let cache = page_cache::get();
-        cache.drop_buffers_for_ephemeral(self.file_id);
-
-        // remove entry from the hash map
-        EPHEMERAL_FILES.write().unwrap().files.remove(&self.file_id);
-
-        // unlink file
-        // FIXME: print error
-        let _ = std::fs::remove_file(&self.file.path);
-    }
-}
-
-pub fn writeback(file_id: u64, blkno: u32, buf: &[u8]) -> Result<(), std::io::Error> {
-    if let Some(file) = EPHEMERAL_FILES.read().unwrap().files.get(&file_id) {
-        file.write_all_at(buf, blkno as u64 * PAGE_SZ as u64)?;
-        Ok(())
-    } else {
-        Err(std::io::Error::new(
-            ErrorKind::Other,
-            "could not write back page, not found in ephemeral files hash",
-        ))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use rand::seq::SliceRandom;
-    use rand::thread_rng;
-    use std::fs;
-    use std::str::FromStr;
-
-    fn repo_harness(
-        test_name: &str,
-    ) -> Result<(&'static PageServerConf, ZTenantId, ZTimelineId), Error> {
-        let repo_dir = PageServerConf::test_repo_dir(test_name);
-        let _ = fs::remove_dir_all(&repo_dir);
-        let conf = PageServerConf::dummy_conf(repo_dir);
-        // Make a static copy of the config. This can never be free'd, but that's
-        // OK in a test.
-        let conf: &'static PageServerConf = Box::leak(Box::new(conf));
-
-        let tenantid = ZTenantId::from_str("11000000000000000000000000000000").unwrap();
-        let timelineid = ZTimelineId::from_str("22000000000000000000000000000000").unwrap();
-        fs::create_dir_all(conf.timeline_path(&timelineid, &tenantid))?;
-
-        Ok((conf, tenantid, timelineid))
-    }
-
-    // Helper function to slurp contents of a file, starting at the current position,
-    // into a string
-    fn read_string(efile: &EphemeralFile, offset: u64, len: usize) -> Result<String, Error> {
-        let mut buf = Vec::new();
-        buf.resize(len, 0u8);
-
-        efile.read_exact_at(&mut buf, offset)?;
-
-        Ok(String::from_utf8_lossy(&buf)
-            .trim_end_matches('\0')
-            .to_string())
-    }
-
-    #[test]
-    fn test_ephemeral_files() -> Result<(), Error> {
-        let (conf, tenantid, timelineid) = repo_harness("ephemeral_files")?;
-
-        let mut file_a = EphemeralFile::create(conf, tenantid, timelineid)?;
-
-        file_a.write_all(b"foo")?;
-        assert_eq!("foo", read_string(&file_a, 0, 20)?);
-
-        file_a.write_all(b"bar")?;
-        assert_eq!("foobar", read_string(&file_a, 0, 20)?);
-
-        // Open a lot of files, enough to cause some page evictions.
-        let mut efiles = Vec::new();
-        for fileno in 0..100 {
-            let mut efile = EphemeralFile::create(conf, tenantid, timelineid)?;
-            efile.write_all(format!("file {}", fileno).as_bytes())?;
-            assert_eq!(format!("file {}", fileno), read_string(&efile, 0, 10)?);
-            efiles.push((fileno, efile));
-        }
-
-        // Check that all the files can still be read from. Use them in random order for
-        // good measure.
-        efiles.as_mut_slice().shuffle(&mut thread_rng());
-        for (fileno, efile) in efiles.iter_mut() {
-            assert_eq!(format!("file {}", fileno), read_string(efile, 0, 10)?);
-        }
-
-        Ok(())
-    }
-}

pageserver/src/layered_repository/filename.rs

@@ -1,14 +1,20 @@
 //!
 //! Helper functions for dealing with filenames of the image and delta layer files.
 //!
-use crate::config::PageServerConf;
 use crate::layered_repository::storage_layer::SegmentTag;
 use crate::relish::*;
+use crate::PageServerConf;
+use crate::{ZTenantId, ZTimelineId};
 use std::fmt;
+use std::fs;
 use std::path::PathBuf;
 
+use anyhow::Result;
+use log::*;
 use zenith_utils::lsn::Lsn;
 
+use super::metadata::METADATA_FILE_NAME;
+
 // Note: LayeredTimeline::load_layer_map() relies on this sort order
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
 pub struct DeltaFileName {
@@ -265,6 +271,36 @@ impl fmt::Display for ImageFileName {
     }
 }
 
+/// Scan timeline directory and create ImageFileName and DeltaFilename
+/// structs representing all files on disk
+///
+/// TODO: returning an Iterator would be more idiomatic
+pub fn list_files(
+    conf: &'static PageServerConf,
+    timelineid: ZTimelineId,
+    tenantid: ZTenantId,
+) -> Result<(Vec<ImageFileName>, Vec<DeltaFileName>)> {
+    let path = conf.timeline_path(&timelineid, &tenantid);
+
+    let mut deltafiles: Vec<DeltaFileName> = Vec::new();
+    let mut imgfiles: Vec<ImageFileName> = Vec::new();
+    for direntry in fs::read_dir(path)? {
+        let fname = direntry?.file_name();
+        let fname = fname.to_str().unwrap();
+
+        if let Some(deltafilename) = DeltaFileName::parse_str(fname) {
+            deltafiles.push(deltafilename);
+        } else if let Some(imgfilename) = ImageFileName::parse_str(fname) {
+            imgfiles.push(imgfilename);
+        } else if fname == METADATA_FILE_NAME || fname == "ancestor" || fname.ends_with(".old") {
+            // ignore these
+        } else {
+            warn!("unrecognized filename in timeline dir: {}", fname);
+        }
+    }
+    Ok((imgfiles, deltafiles))
+}
+
 /// Helper enum to hold a PageServerConf, or a path
 ///
 /// This is used by DeltaLayer and ImageLayer. Normally, this holds a reference to the

pageserver/src/layered_repository/global_layer_map.rs

@@ -1,142 +0,0 @@
-//!
-//! Global registry of open layers.
-//!
-//! Whenever a new in-memory layer is created to hold incoming WAL, it is registered
-//! in [`GLOBAL_LAYER_MAP`], so that we can keep track of the total number of
-//! in-memory layers in the system, and know when we need to evict some to release
-//! memory.
-//!
-//! Each layer is assigned a unique ID when it's registered in the global registry.
-//! The ID can be used to relocate the layer later, without having to hold locks.
-//!
-
-use std::sync::atomic::{AtomicU8, Ordering};
-use std::sync::{Arc, RwLock};
-
-use super::inmemory_layer::InMemoryLayer;
-
-use lazy_static::lazy_static;
-
-const MAX_USAGE_COUNT: u8 = 5;
-
-lazy_static! {
-    pub static ref GLOBAL_LAYER_MAP: RwLock<InMemoryLayers> =
-        RwLock::new(InMemoryLayers::default());
-}
-
-// TODO these types can probably be smaller
-#[derive(PartialEq, Eq, Clone, Copy)]
-pub struct LayerId {
-    index: usize,
-    tag: u64, // to avoid ABA problem
-}
-
-enum SlotData {
-    Occupied(Arc<InMemoryLayer>),
-    /// Vacant slots form a linked list, the value is the index
-    /// of the next vacant slot in the list.
-    Vacant(Option<usize>),
-}
-
-struct Slot {
-    tag: u64,
-    data: SlotData,
-    usage_count: AtomicU8, // for clock algorithm
-}
-
-#[derive(Default)]
-pub struct InMemoryLayers {
-    slots: Vec<Slot>,
-    num_occupied: usize,
-
-    // Head of free-slot list.
-    next_empty_slot_idx: Option<usize>,
-}
-
-impl InMemoryLayers {
-    pub fn insert(&mut self, layer: Arc<InMemoryLayer>) -> LayerId {
-        let slot_idx = match self.next_empty_slot_idx {
-            Some(slot_idx) => slot_idx,
-            None => {
-                let idx = self.slots.len();
-                self.slots.push(Slot {
-                    tag: 0,
-                    data: SlotData::Vacant(None),
-                    usage_count: AtomicU8::new(0),
-                });
-                idx
-            }
-        };
-        let slots_len = self.slots.len();
-
-        let slot = &mut self.slots[slot_idx];
-
-        match slot.data {
-            SlotData::Occupied(_) => {
-                panic!("an occupied slot was in the free list");
-            }
-            SlotData::Vacant(next_empty_slot_idx) => {
-                self.next_empty_slot_idx = next_empty_slot_idx;
-            }
-        }
-
-        slot.data = SlotData::Occupied(layer);
-        slot.usage_count.store(1, Ordering::Relaxed);
-
-        self.num_occupied += 1;
-        assert!(self.num_occupied <= slots_len);
-
-        LayerId {
-            index: slot_idx,
-            tag: slot.tag,
-        }
-    }
-
-    pub fn get(&self, layer_id: &LayerId) -> Option<Arc<InMemoryLayer>> {
-        let slot = self.slots.get(layer_id.index)?; // TODO should out of bounds indexes just panic?
-        if slot.tag != layer_id.tag {
-            return None;
-        }
-
-        if let SlotData::Occupied(layer) = &slot.data {
-            let _ = slot.usage_count.fetch_update(
-                Ordering::Relaxed,
-                Ordering::Relaxed,
-                |old_usage_count| {
-                    if old_usage_count < MAX_USAGE_COUNT {
-                        Some(old_usage_count + 1)
-                    } else {
-                        None
-                    }
-                },
-            );
-            Some(Arc::clone(layer))
-        } else {
-            None
-        }
-    }
-
-    // TODO this won't be a public API in the future
-    pub fn remove(&mut self, layer_id: &LayerId) {
-        let slot = &mut self.slots[layer_id.index];
-
-        if slot.tag != layer_id.tag {
-            return;
-        }
-
-        match &slot.data {
-            SlotData::Occupied(_layer) => {
-                // TODO evict the layer
-            }
-            SlotData::Vacant(_) => unimplemented!(),
-        }
-
-        slot.data = SlotData::Vacant(self.next_empty_slot_idx);
-        self.next_empty_slot_idx = Some(layer_id.index);
-
-        assert!(self.num_occupied > 0);
-        self.num_occupied -= 1;
-
-        slot.tag = slot.tag.wrapping_add(1);
-    }
-}

pageserver/src/layered_repository/image_layer.rs

@@ -21,25 +21,26 @@
 //!
 //! For non-blocky relishes, the image can be found in NONBLOCKY_IMAGE_CHAPTER.
 //!
-use crate::config::PageServerConf;
 use crate::layered_repository::filename::{ImageFileName, PathOrConf};
 use crate::layered_repository::storage_layer::{
-    Layer, PageReconstructData, PageReconstructResult, SegmentBlk, SegmentTag,
+    Layer, PageReconstructData, PageReconstructResult, SegmentTag,
 };
+use crate::layered_repository::LayeredTimeline;
 use crate::layered_repository::RELISH_SEG_SIZE;
-use crate::virtual_file::VirtualFile;
+use crate::PageServerConf;
 use crate::{ZTenantId, ZTimelineId};
-use anyhow::{anyhow, bail, ensure, Context, Result};
+use anyhow::{anyhow, bail, ensure, Result};
 use bytes::Bytes;
 use log::*;
 use serde::{Deserialize, Serialize};
 use std::convert::TryInto;
 use std::fs;
+use std::fs::File;
 use std::io::{BufWriter, Write};
 use std::path::{Path, PathBuf};
 use std::sync::{Mutex, MutexGuard};
 
-use bookfile::{Book, BookWriter, ChapterWriter};
+use bookfile::{Book, BookWriter};
 
 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;
@@ -98,13 +99,14 @@ pub struct ImageLayer {
 
 #[derive(Clone)]
 enum ImageType {
-    Blocky { num_blocks: SegmentBlk },
+    Blocky { num_blocks: u32 },
     NonBlocky,
 }
 
 pub struct ImageLayerInner {
-    /// If None, the 'image_type' has not been loaded into memory yet.
-    book: Option<Book<VirtualFile>>,
+    /// If false, the 'image_type' has not been
+    /// loaded into memory yet.
+    loaded: bool,
 
     /// Derived from filename and bookfile chapter metadata
     image_type: ImageType,
@@ -115,10 +117,6 @@ impl Layer for ImageLayer {
         PathBuf::from(self.layer_name().to_string())
     }
 
-    fn get_tenant_id(&self) -> ZTenantId {
-        self.tenantid
-    }
-
     fn get_timeline_id(&self) -> ZTimelineId {
         self.timelineid
     }
@@ -143,48 +141,35 @@ impl Layer for ImageLayer {
     /// Look up given page in the file
     fn get_page_reconstruct_data(
         &self,
-        blknum: SegmentBlk,
+        blknum: u32,
         lsn: Lsn,
-        cached_img_lsn: Option<Lsn>,
         reconstruct_data: &mut PageReconstructData,
     ) -> Result<PageReconstructResult> {
-        assert!((0..RELISH_SEG_SIZE).contains(&blknum));
         assert!(lsn >= self.lsn);
 
-        match cached_img_lsn {
-            Some(cached_lsn) if self.lsn <= cached_lsn => return Ok(PageReconstructResult::Cached),
-            _ => {}
-        }
-
         let inner = self.load()?;
 
+        let base_blknum = blknum % RELISH_SEG_SIZE;
+
+        let (_path, book) = self.open_book()?;
+
         let buf = match &inner.image_type {
             ImageType::Blocky { num_blocks } => {
-                // Check if the request is beyond EOF
-                if blknum >= *num_blocks {
+                if base_blknum >= *num_blocks {
                     return Ok(PageReconstructResult::Missing(lsn));
                 }
 
                 let mut buf = vec![0u8; BLOCK_SIZE];
-                let offset = BLOCK_SIZE as u64 * blknum as u64;
+                let offset = BLOCK_SIZE as u64 * base_blknum as u64;
 
-                let chapter = inner
-                    .book
-                    .as_ref()
-                    .unwrap()
-                    .chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
+                let chapter = book.chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
                 chapter.read_exact_at(&mut buf, offset)?;
 
                 buf
             }
             ImageType::NonBlocky => {
-                ensure!(blknum == 0);
-                inner
-                    .book
-                    .as_ref()
-                    .unwrap()
-                    .read_chapter(NONBLOCKY_IMAGE_CHAPTER)?
-                    .into_vec()
+                ensure!(base_blknum == 0);
+                book.read_chapter(NONBLOCKY_IMAGE_CHAPTER)?.into_vec()
             }
         };
 
@@ -193,7 +178,7 @@ impl Layer for ImageLayer {
     }
 
     /// Get size of the segment
-    fn get_seg_size(&self, _lsn: Lsn) -> Result<SegmentBlk> {
+    fn get_seg_size(&self, _lsn: Lsn) -> Result<u32> {
         let inner = self.load()?;
         match inner.image_type {
             ImageType::Blocky { num_blocks } => Ok(num_blocks),
@@ -206,7 +191,14 @@ impl Layer for ImageLayer {
         Ok(true)
     }
 
+    ///
+    /// Release most of the memory used by this layer. If it's accessed again later,
+    /// it will need to be loaded back.
+    ///
     fn unload(&self) -> Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.image_type = ImageType::Blocky { num_blocks: 0 };
+        inner.loaded = false;
         Ok(())
     }
 
@@ -220,10 +212,6 @@ impl Layer for ImageLayer {
         false
     }
 
-    fn is_in_memory(&self) -> bool {
-        false
-    }
-
     /// debugging function to print out the contents of the layer
     fn dump(&self) -> Result<()> {
         println!(
@@ -236,11 +224,8 @@ impl Layer for ImageLayer {
         match inner.image_type {
             ImageType::Blocky { num_blocks } => println!("({}) blocks ", num_blocks),
             ImageType::NonBlocky => {
-                let chapter = inner
-                    .book
-                    .as_ref()
-                    .unwrap()
-                    .read_chapter(NONBLOCKY_IMAGE_CHAPTER)?;
+                let (_path, book) = self.open_book()?;
+                let chapter = book.read_chapter(NONBLOCKY_IMAGE_CHAPTER)?;
                 println!("non-blocky ({} bytes)", chapter.len());
             }
         }
@@ -264,6 +249,120 @@ impl ImageLayer {
         }
     }
 
+    /// Create a new image file, using the given array of pages.
+    fn create(
+        conf: &'static PageServerConf,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        seg: SegmentTag,
+        lsn: Lsn,
+        base_images: Vec<Bytes>,
+    ) -> Result<ImageLayer> {
+        let image_type = if seg.rel.is_blocky() {
+            let num_blocks: u32 = base_images.len().try_into()?;
+            ImageType::Blocky { num_blocks }
+        } else {
+            assert_eq!(base_images.len(), 1);
+            ImageType::NonBlocky
+        };
+
+        let layer = ImageLayer {
+            path_or_conf: PathOrConf::Conf(conf),
+            timelineid,
+            tenantid,
+            seg,
+            lsn,
+            inner: Mutex::new(ImageLayerInner {
+                loaded: true,
+                image_type: image_type.clone(),
+            }),
+        };
+        let inner = layer.inner.lock().unwrap();
+
+        // Write the images into a file
+        let path = layer.path();
+        // Note: This overwrites any existing file. There shouldn't be any.
+        // FIXME: throw an error instead?
+        let file = File::create(&path)?;
+        let buf_writer = BufWriter::new(file);
+        let book = BookWriter::new(buf_writer, IMAGE_FILE_MAGIC)?;
+
+        let book = match &image_type {
+            ImageType::Blocky { .. } => {
+                let mut chapter = book.new_chapter(BLOCKY_IMAGES_CHAPTER);
+                for block_bytes in base_images {
+                    assert_eq!(block_bytes.len(), BLOCK_SIZE);
+                    chapter.write_all(&block_bytes)?;
+                }
+                chapter.close()?
+            }
+            ImageType::NonBlocky => {
+                let mut chapter = book.new_chapter(NONBLOCKY_IMAGE_CHAPTER);
+                chapter.write_all(&base_images[0])?;
+                chapter.close()?
+            }
+        };
+
+        let mut chapter = book.new_chapter(SUMMARY_CHAPTER);
+        let summary = Summary {
+            tenantid,
+            timelineid,
+            seg,
+
+            lsn,
+        };
+        Summary::ser_into(&summary, &mut chapter)?;
+        let book = chapter.close()?;
+
+        // This flushes the underlying 'buf_writer'.
+        let writer = book.close()?;
+        writer.get_ref().sync_all()?;
+
+        trace!("saved {}", path.display());
+
+        drop(inner);
+
+        Ok(layer)
+    }
+
+    // Create a new image file by materializing every page in a source layer
+    // at given LSN.
+    pub fn create_from_src(
+        conf: &'static PageServerConf,
+        timeline: &LayeredTimeline,
+        src: &dyn Layer,
+        lsn: Lsn,
+    ) -> Result<ImageLayer> {
+        let seg = src.get_seg_tag();
+        let timelineid = timeline.timelineid;
+
+        let startblk;
+        let size;
+        if seg.rel.is_blocky() {
+            size = src.get_seg_size(lsn)?;
+            startblk = seg.segno * RELISH_SEG_SIZE;
+        } else {
+            size = 1;
+            startblk = 0;
+        }
+
+        trace!(
+            "creating new ImageLayer for {} on timeline {} at {}",
+            seg,
+            timelineid,
+            lsn,
+        );
+
+        let mut base_images: Vec<Bytes> = Vec::new();
+        for blknum in startblk..(startblk + size) {
+            let img = timeline.materialize_page(seg, blknum, lsn, &*src)?;
+
+            base_images.push(img);
+        }
+
+        Self::create(conf, timelineid, timeline.tenantid, seg, lsn, base_images)
+    }
+
     ///
     /// Load the contents of the file into memory
     ///
@@ -271,19 +370,11 @@ impl ImageLayer {
         // quick exit if already loaded
         let mut inner = self.inner.lock().unwrap();
 
-        if inner.book.is_some() {
+        if inner.loaded {
             return Ok(inner);
         }
 
-        let path = self.path();
-        let file = VirtualFile::open(&path)
-            .with_context(|| format!("Failed to open virtual file '{}'", path.display()))?;
-        let book = Book::new(file).with_context(|| {
-            format!(
-                "Failed to open virtual file '{}' as a bookfile",
-                path.display()
-            )
-        })?;
+        let (path, book) = self.open_book()?;
 
         match &self.path_or_conf {
             PathOrConf::Conf(_) => {
@@ -314,7 +405,7 @@ impl ImageLayer {
             let chapter = book.chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
             let images_len = chapter.len();
             ensure!(images_len % BLOCK_SIZE as u64 == 0);
-            let num_blocks: SegmentBlk = (images_len / BLOCK_SIZE as u64).try_into()?;
+            let num_blocks: u32 = (images_len / BLOCK_SIZE as u64).try_into()?;
             ImageType::Blocky { num_blocks }
         } else {
             let _chapter = book.chapter_reader(NONBLOCKY_IMAGE_CHAPTER)?;
@@ -324,13 +415,22 @@ impl ImageLayer {
         debug!("loaded from {}", &path.display());
 
         *inner = ImageLayerInner {
-            book: Some(book),
+            loaded: true,
             image_type,
         };
 
         Ok(inner)
     }
 
+    fn open_book(&self) -> Result<(PathBuf, Book<File>)> {
+        let path = self.path();
+
+        let file = File::open(&path)?;
+        let book = Book::new(file)?;
+
+        Ok((path, book))
+    }
+
     /// Create an ImageLayer struct representing an existing file on disk
     pub fn new(
         conf: &'static PageServerConf,
@@ -345,7 +445,7 @@ impl ImageLayer {
             seg: filename.seg,
             lsn: filename.lsn,
             inner: Mutex::new(ImageLayerInner {
-                book: None,
+                loaded: false,
                 image_type: ImageType::Blocky { num_blocks: 0 },
             }),
         }
@@ -354,10 +454,7 @@ impl ImageLayer {
     /// Create an ImageLayer struct representing an existing file on disk.
     ///
     /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary.
-    pub fn new_for_path<F>(path: &Path, book: &Book<F>) -> Result<ImageLayer>
-    where
-        F: std::os::unix::prelude::FileExt,
-    {
+    pub fn new_for_path(path: &Path, book: &Book<File>) -> Result<ImageLayer> {
         let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
         let summary = Summary::des(&chapter)?;
 
@@ -368,7 +465,7 @@ impl ImageLayer {
             seg: summary.seg,
             lsn: summary.lsn,
             inner: Mutex::new(ImageLayerInner {
-                book: None,
+                loaded: false,
                 image_type: ImageType::Blocky { num_blocks: 0 },
             }),
         })
@@ -391,136 +488,3 @@ impl ImageLayer {
         )
     }
 }
-
-/// A builder object for constructing a new image layer.
-///
-/// Usage:
-///
-/// 1. Create the ImageLayerWriter by calling ImageLayerWriter::new(...)
-///
-/// 2. Write the contents by calling `put_page_image` for every page
-///    in the segment.
-///
-/// 3. Call `finish`.
-///
-pub struct ImageLayerWriter {
-    conf: &'static PageServerConf,
-    timelineid: ZTimelineId,
-    tenantid: ZTenantId,
-    seg: SegmentTag,
-    lsn: Lsn,
-
-    num_blocks: SegmentBlk,
-
-    page_image_writer: ChapterWriter<BufWriter<VirtualFile>>,
-    num_blocks_written: SegmentBlk,
-}
-
-impl ImageLayerWriter {
-    pub fn new(
-        conf: &'static PageServerConf,
-        timelineid: ZTimelineId,
-        tenantid: ZTenantId,
-        seg: SegmentTag,
-        lsn: Lsn,
-        num_blocks: SegmentBlk,
-    ) -> Result<ImageLayerWriter> {
-        // Create the file
-        //
-        // Note: This overwrites any existing file. There shouldn't be any.
-        // FIXME: throw an error instead?
-        let path = ImageLayer::path_for(
-            &PathOrConf::Conf(conf),
-            timelineid,
-            tenantid,
-            &ImageFileName { seg, lsn },
-        );
-        let file = VirtualFile::create(&path)?;
-        let buf_writer = BufWriter::new(file);
-        let book = BookWriter::new(buf_writer, IMAGE_FILE_MAGIC)?;
-
-        // Open the page-images chapter for writing. The calls to
-        // `put_page_image` will use this to write the contents.
-        let chapter = if seg.rel.is_blocky() {
-            book.new_chapter(BLOCKY_IMAGES_CHAPTER)
-        } else {
-            assert_eq!(num_blocks, 1);
-            book.new_chapter(NONBLOCKY_IMAGE_CHAPTER)
-        };
-
-        let writer = ImageLayerWriter {
-            conf,
-            timelineid,
-            tenantid,
-            seg,
-            lsn,
-            num_blocks,
-            page_image_writer: chapter,
-            num_blocks_written: 0,
-        };
-
-        Ok(writer)
-    }
-
-    ///
-    /// Write next page image to the file.
-    ///
-    /// The page versions must be appended in blknum order.
-    ///
-    pub fn put_page_image(&mut self, block_bytes: &[u8]) -> Result<()> {
-        assert!(self.num_blocks_written < self.num_blocks);
-        if self.seg.rel.is_blocky() {
-            assert_eq!(block_bytes.len(), BLOCK_SIZE);
-        }
-        self.page_image_writer.write_all(block_bytes)?;
-        self.num_blocks_written += 1;
-        Ok(())
-    }
-
-    pub fn finish(self) -> Result<ImageLayer> {
-        // Check that the `put_page_image' was called for every block.
-        assert!(self.num_blocks_written == self.num_blocks);
-
-        // Close the page-images chapter
-        let book = self.page_image_writer.close()?;
-
-        // Write out the summary chapter
-        let image_type = if self.seg.rel.is_blocky() {
-            ImageType::Blocky {
-                num_blocks: self.num_blocks,
-            }
-        } else {
-            ImageType::NonBlocky
-        };
-        let mut chapter = book.new_chapter(SUMMARY_CHAPTER);
-        let summary = Summary {
-            tenantid: self.tenantid,
-            timelineid: self.timelineid,
-            seg: self.seg,
-            lsn: self.lsn,
-        };
-        Summary::ser_into(&summary, &mut chapter)?;
-        let book = chapter.close()?;
-
-        // This flushes the underlying 'buf_writer'.
-        book.close()?;
-
-        // Note: Because we open the file in write-only mode, we cannot
-        // reuse the same VirtualFile for reading later. That's why we don't
-        // set inner.book here. The first read will have to re-open it.
-        let layer = ImageLayer {
-            path_or_conf: PathOrConf::Conf(self.conf),
-            timelineid: self.timelineid,
-            tenantid: self.tenantid,
-            seg: self.seg,
-            lsn: self.lsn,
-            inner: Mutex::new(ImageLayerInner {
-                book: None,
-                image_type,
-            }),
-        };
-        trace!("created image layer {}", layer.path().display());
-
-        Ok(layer)
-    }
-}

pageserver/src/layered_repository/inmemory_layer.rs

@@ -1,30 +1,26 @@
-//! An in-memory layer stores recently received PageVersions.
-//! The page versions are held in a BTreeMap. To avoid OOM errors, the map size is limited
-//! and layers can be spilled to disk into ephemeral files.
 //!
-//! And there's another BTreeMap to track the size of the relation.
+//! An in-memory layer stores recently received page versions in memory. The page versions
+//! are held in a BTreeMap, and there's another BTreeMap to track the size of the relation.
 //!
-use crate::config::PageServerConf;
-use crate::layered_repository::delta_layer::{DeltaLayer, DeltaLayerWriter};
-use crate::layered_repository::ephemeral_file::EphemeralFile;
 use crate::layered_repository::filename::DeltaFileName;
-use crate::layered_repository::image_layer::{ImageLayer, ImageLayerWriter};
 use crate::layered_repository::storage_layer::{
-    Layer, PageReconstructData, PageReconstructResult, PageVersion, SegmentBlk, SegmentTag,
-    RELISH_SEG_SIZE,
+    Layer, PageReconstructData, PageReconstructResult, PageVersion, SegmentTag, RELISH_SEG_SIZE,
 };
 use crate::layered_repository::LayeredTimeline;
 use crate::layered_repository::ZERO_PAGE;
-use crate::repository::ZenithWalRecord;
+use crate::layered_repository::{DeltaLayer, ImageLayer};
+use crate::repository::WALRecord;
+use crate::PageServerConf;
 use crate::{ZTenantId, ZTimelineId};
-use anyhow::{ensure, Result};
+use anyhow::{bail, ensure, Result};
 use bytes::Bytes;
 use log::*;
 use std::path::PathBuf;
 use std::sync::{Arc, RwLock};
-use zenith_utils::lsn::Lsn;
 use zenith_utils::vec_map::VecMap;
 
+use zenith_utils::lsn::Lsn;
+
 use super::page_versions::PageVersions;
 
 pub struct InMemoryLayer {
@@ -51,7 +47,7 @@ pub struct InMemoryLayer {
 }
 
 pub struct InMemoryLayerInner {
-    /// Frozen layers have an exclusive end LSN.
+    /// Frozen in-memory layers have an exclusive end LSN.
     /// Writes are only allowed when this is None
     end_lsn: Option<Lsn>,
 
@@ -66,13 +62,13 @@ pub struct InMemoryLayerInner {
     page_versions: PageVersions,
 
     ///
-    /// `seg_sizes` tracks the size of the segment at different points in time.
+    /// `segsizes` tracks the size of the segment at different points in time.
     ///
     /// For a blocky rel, there is always one entry, at the layer's start_lsn,
     /// so that determining the size never depends on the predecessor layer. For
-    /// a non-blocky rel, 'seg_sizes' is not used and is always empty.
+    /// a non-blocky rel, 'segsizes' is not used and is always empty.
     ///
-    seg_sizes: VecMap<Lsn, SegmentBlk>,
+    segsizes: VecMap<Lsn, u32>,
 }
 
 impl InMemoryLayerInner {
@@ -80,9 +76,9 @@ impl InMemoryLayerInner {
         assert!(self.end_lsn.is_none());
     }
 
-    fn get_seg_size(&self, lsn: Lsn) -> SegmentBlk {
+    fn get_seg_size(&self, lsn: Lsn) -> u32 {
         // Scan the BTreeMap backwards, starting from the given entry.
-        let slice = self.seg_sizes.slice_range(..=lsn);
+        let slice = self.segsizes.slice_range(..=lsn);
 
         // We make sure there is always at least one entry
         if let Some((_entry_lsn, entry)) = slice.last() {
@@ -94,9 +90,8 @@ impl InMemoryLayerInner {
 }
 
 impl Layer for InMemoryLayer {
-    // An in-memory layer can be spilled to disk into ephemeral file,
-    // This function is used only for debugging, so we don't need to be very precise.
-    // Construct a filename as if it was a delta layer.
+    // An in-memory layer doesn't really have a filename as it's not stored on disk,
+    // but we construct a filename as if it was a delta layer
     fn filename(&self) -> PathBuf {
         let inner = self.inner.read().unwrap();
 
@@ -118,10 +113,6 @@ impl Layer for InMemoryLayer {
         PathBuf::from(format!("inmem-{}", delta_filename))
     }
 
-    fn get_tenant_id(&self) -> ZTenantId {
-        self.tenantid
-    }
-
     fn get_timeline_id(&self) -> ZTimelineId {
         self.timelineid
     }
@@ -152,14 +143,13 @@ impl Layer for InMemoryLayer {
     /// Look up given page in the cache.
     fn get_page_reconstruct_data(
         &self,
-        blknum: SegmentBlk,
+        blknum: u32,
         lsn: Lsn,
-        cached_img_lsn: Option<Lsn>,
         reconstruct_data: &mut PageReconstructData,
     ) -> Result<PageReconstructResult> {
         let mut need_image = true;
 
-        assert!((0..RELISH_SEG_SIZE).contains(&blknum));
+        assert!(self.seg.blknum_in_seg(blknum));
 
         {
             let inner = self.inner.read().unwrap();
@@ -170,41 +160,23 @@ impl Layer for InMemoryLayer {
                 .get_block_lsn_range(blknum, ..=lsn)
                 .iter()
                 .rev();
-            for (entry_lsn, pos) in iter {
-                match &cached_img_lsn {
-                    Some(cached_lsn) if entry_lsn <= cached_lsn => {
-                        return Ok(PageReconstructResult::Cached)
-                    }
-                    _ => {}
-                }
-
-                let pv = inner.page_versions.read_pv(*pos)?;
-                match pv {
-                    PageVersion::Page(img) => {
-                        reconstruct_data.page_img = Some(img);
+            for (entry_lsn, entry) in iter {
+                if let Some(img) = &entry.page_image {
+                    reconstruct_data.page_img = Some(img.clone());
+                    need_image = false;
+                    break;
+                } else if let Some(rec) = &entry.record {
+                    reconstruct_data.records.push((*entry_lsn, rec.clone()));
+                    if rec.will_init {
+                        // This WAL record initializes the page, so no need to go further back
                         need_image = false;
                         break;
                     }
-                    PageVersion::Wal(rec) => {
-                        reconstruct_data.records.push((*entry_lsn, rec.clone()));
-                        if rec.will_init() {
-                            // This WAL record initializes the page, so no need to go further back
-                            need_image = false;
-                            break;
-                        }
-                    }
+                } else {
+                    // No base image, and no WAL record. Huh?
+                    bail!("no page image or WAL record for requested page");
                 }
             }
-
-            // If we didn't find any records for this, check if the request is beyond EOF
-            if need_image
-                && reconstruct_data.records.is_empty()
-                && self.seg.rel.is_blocky()
-                && blknum >= self.get_seg_size(lsn)?
-            {
-                return Ok(PageReconstructResult::Missing(self.start_lsn));
-            }
-
             // release lock on 'inner'
         }
 
@@ -222,7 +194,7 @@ impl Layer for InMemoryLayer {
     }
 
     /// Get size of the relation at given LSN
-    fn get_seg_size(&self, lsn: Lsn) -> Result<SegmentBlk> {
+    fn get_seg_size(&self, lsn: Lsn) -> Result<u32> {
         assert!(lsn >= self.start_lsn);
         ensure!(
             self.seg.rel.is_blocky(),
@@ -243,13 +215,9 @@ impl Layer for InMemoryLayer {
         assert!(lsn >= self.start_lsn);
 
         // Is the requested LSN after the segment was dropped?
-        if inner.dropped {
-            if let Some(end_lsn) = inner.end_lsn {
-                if lsn >= end_lsn {
-                    return Ok(false);
-                }
-            } else {
-                panic!("dropped in-memory layer with no end LSN");
+        if let Some(end_lsn) = inner.end_lsn {
+            if lsn >= end_lsn {
+                return Ok(false);
             }
         }
 
@@ -267,17 +235,13 @@ impl Layer for InMemoryLayer {
     /// Nothing to do here. When you drop the last reference to the layer, it will
     /// be deallocated.
     fn delete(&self) -> Result<()> {
-        panic!("can't delete an InMemoryLayer")
+        Ok(())
     }
 
     fn is_incremental(&self) -> bool {
         self.incremental
     }
 
-    fn is_in_memory(&self) -> bool {
-        true
-    }
-
     /// debugging function to print out the contents of the layer
     fn dump(&self) -> Result<()> {
         let inner = self.inner.read().unwrap();
@@ -293,18 +257,18 @@ impl Layer for InMemoryLayer {
             self.timelineid, self.seg, self.start_lsn, end_str, inner.dropped,
         );
 
-        for (k, v) in inner.seg_sizes.as_slice() {
-            println!("seg_sizes {}: {}", k, v);
+        for (k, v) in inner.segsizes.as_slice() {
+            println!("segsizes {}: {}", k, v);
         }
 
-        for (blknum, lsn, pos) in inner.page_versions.ordered_page_version_iter(None) {
-            let pv = inner.page_versions.read_pv(pos)?;
-            let pv_description = match pv {
-                PageVersion::Page(_img) => "page",
-                PageVersion::Wal(_rec) => "wal",
-            };
-
-            println!("blk {} at {}: {}\n", blknum, lsn, pv_description);
+        for (blknum, lsn, pv) in inner.page_versions.ordered_page_version_iter(None) {
+            println!(
+                "blk {} at {}: {}/{}\n",
+                blknum,
+                lsn,
+                pv.page_image.is_some(),
+                pv.record.is_some()
+            );
         }
 
         Ok(())
@@ -317,6 +281,12 @@ pub struct LayersOnDisk {
     pub image_layers: Vec<ImageLayer>,
 }
 
+impl LayersOnDisk {
+    pub fn is_empty(&self) -> bool {
+        self.delta_layers.is_empty() && self.image_layers.is_empty()
+    }
+}
+
 impl InMemoryLayer {
     /// Return the oldest page version that's stored in this layer
     pub fn get_oldest_pending_lsn(&self) -> Lsn {
@@ -341,14 +311,12 @@ impl InMemoryLayer {
             start_lsn
         );
 
-        // The segment is initially empty, so initialize 'seg_sizes' with 0.
-        let mut seg_sizes = VecMap::default();
+        // The segment is initially empty, so initialize 'segsizes' with 0.
+        let mut segsizes = VecMap::default();
         if seg.rel.is_blocky() {
-            seg_sizes.append(start_lsn, 0).unwrap();
+            segsizes.append(start_lsn, 0).unwrap();
         }
 
-        let file = EphemeralFile::create(conf, tenantid, timelineid)?;
-
         Ok(InMemoryLayer {
             conf,
             timelineid,
@@ -360,8 +328,8 @@ impl InMemoryLayer {
             inner: RwLock::new(InMemoryLayerInner {
                 end_lsn: None,
                 dropped: false,
-                page_versions: PageVersions::new(file),
-                seg_sizes,
+                page_versions: PageVersions::default(),
+                segsizes,
             }),
         })
     }
@@ -369,24 +337,33 @@ impl InMemoryLayer {
     // Write operations
 
     /// Remember new page version, as a WAL record over previous version
-    pub fn put_wal_record(
-        &self,
-        lsn: Lsn,
-        blknum: SegmentBlk,
-        rec: ZenithWalRecord,
-    ) -> Result<u32> {
-        self.put_page_version(blknum, lsn, PageVersion::Wal(rec))
+    pub fn put_wal_record(&self, lsn: Lsn, blknum: u32, rec: WALRecord) -> u32 {
+        self.put_page_version(
+            blknum,
+            lsn,
+            PageVersion {
+                page_image: None,
+                record: Some(rec),
+            },
+        )
     }
 
     /// Remember new page version, as a full page image
-    pub fn put_page_image(&self, blknum: SegmentBlk, lsn: Lsn, img: Bytes) -> Result<u32> {
-        self.put_page_version(blknum, lsn, PageVersion::Page(img))
+    pub fn put_page_image(&self, blknum: u32, lsn: Lsn, img: Bytes) -> u32 {
+        self.put_page_version(
+            blknum,
+            lsn,
+            PageVersion {
+                page_image: Some(img),
+                record: None,
+            },
+        )
     }
 
     /// Common subroutine of the public put_wal_record() and put_page_image() functions.
     /// Adds the page version to the in-memory tree
-    pub fn put_page_version(&self, blknum: SegmentBlk, lsn: Lsn, pv: PageVersion) -> Result<u32> {
-        assert!((0..RELISH_SEG_SIZE).contains(&blknum));
+    pub fn put_page_version(&self, blknum: u32, lsn: Lsn, pv: PageVersion) -> u32 {
+        assert!(self.seg.blknum_in_seg(blknum));
 
         trace!(
             "put_page_version blk {} of {} at {}/{}",
@@ -399,7 +376,7 @@ impl InMemoryLayer {
 
         inner.assert_writeable();
 
-        let old = inner.page_versions.append_or_update_last(blknum, lsn, pv)?;
+        let old = inner.page_versions.append_or_update_last(blknum, lsn, pv);
 
         if old.is_some() {
             // We already had an entry for this LSN. That's odd..
@@ -411,7 +388,7 @@ impl InMemoryLayer {
 
         // Also update the relation size, if this extended the relation.
         if self.seg.rel.is_blocky() {
-            let newsize = blknum + 1;
+            let newsize = blknum - self.seg.segno * RELISH_SEG_SIZE + 1;
 
             // use inner get_seg_size, since calling self.get_seg_size will try to acquire the lock,
             // which we've just acquired above
@@ -433,8 +410,12 @@ impl InMemoryLayer {
                 // PostgreSQL writes its WAL records and there's no guarantee of it. If it does
                 // happen, we would hit the "page version already exists" warning above on the
                 // subsequent call to initialize the gap page.
-                for gapblknum in oldsize..blknum {
-                    let zeropv = PageVersion::Page(ZERO_PAGE.clone());
+                let gapstart = self.seg.segno * RELISH_SEG_SIZE + oldsize;
+                for gapblknum in gapstart..blknum {
+                    let zeropv = PageVersion {
+                        page_image: Some(ZERO_PAGE.clone()),
+                        record: None,
+                    };
                     trace!(
                         "filling gap blk {} with zeros for write of {}",
                         gapblknum,
@@ -442,27 +423,27 @@ impl InMemoryLayer {
                     );
                     let old = inner
                         .page_versions
-                        .append_or_update_last(gapblknum, lsn, zeropv)?;
+                        .append_or_update_last(gapblknum, lsn, zeropv);
                     // We already had an entry for this LSN. That's odd..
 
                     if old.is_some() {
                         warn!(
-                            "Page version of seg {} blk {} at {} already exists",
-                            self.seg, blknum, lsn
+                            "Page version of rel {} blk {} at {} already exists",
+                            self.seg.rel, blknum, lsn
                         );
                     }
                 }
 
-                inner.seg_sizes.append_or_update_last(lsn, newsize).unwrap();
-                return Ok(newsize - oldsize);
+                inner.segsizes.append_or_update_last(lsn, newsize).unwrap();
+                return newsize - oldsize;
             }
         }
 
-        Ok(0)
+        0
     }
 
     /// Remember that the relation was truncated at given LSN
-    pub fn put_truncation(&self, lsn: Lsn, new_size: SegmentBlk) {
+    pub fn put_truncation(&self, lsn: Lsn, segsize: u32) {
         assert!(
             self.seg.rel.is_blocky(),
             "put_truncation() called on a non-blocky rel"
@@ -472,13 +453,10 @@ impl InMemoryLayer {
         inner.assert_writeable();
 
         // check that this we truncate to a smaller size than segment was before the truncation
-        let old_size = inner.get_seg_size(lsn);
-        assert!(new_size < old_size);
+        let oldsize = inner.get_seg_size(lsn);
+        assert!(segsize < oldsize);
 
-        let (old, _delta_size) = inner
-            .seg_sizes
-            .append_or_update_last(lsn, new_size)
-            .unwrap();
+        let old = inner.segsizes.append_or_update_last(lsn, segsize).unwrap();
 
         if old.is_some() {
             // We already had an entry for this LSN. That's odd..
@@ -524,14 +502,12 @@ impl InMemoryLayer {
         );
 
         // Copy the segment size at the start LSN from the predecessor layer.
-        let mut seg_sizes = VecMap::default();
+        let mut segsizes = VecMap::default();
         if seg.rel.is_blocky() {
             let size = src.get_seg_size(start_lsn)?;
-            seg_sizes.append(start_lsn, size).unwrap();
+            segsizes.append(start_lsn, size).unwrap();
         }
 
-        let file = EphemeralFile::create(conf, tenantid, timelineid)?;
-
         Ok(InMemoryLayer {
             conf,
             timelineid,
@@ -543,8 +519,8 @@ impl InMemoryLayer {
             inner: RwLock::new(InMemoryLayerInner {
                 end_lsn: None,
                 dropped: false,
-                page_versions: PageVersions::new(file),
-                seg_sizes,
+                page_versions: PageVersions::default(),
+                segsizes,
             }),
         })
     }
@@ -567,7 +543,7 @@ impl InMemoryLayer {
             assert!(self.start_lsn < end_lsn + 1);
             inner.end_lsn = Some(Lsn(end_lsn.0 + 1));
 
-            if let Some((lsn, _)) = inner.seg_sizes.as_slice().last() {
+            if let Some((lsn, _)) = inner.segsizes.as_slice().last() {
                 assert!(lsn <= &end_lsn, "{:?} {:?}", lsn, end_lsn);
             }
 
@@ -580,16 +556,12 @@ impl InMemoryLayer {
     /// Write the this frozen in-memory layer to disk.
     ///
     /// Returns new layers that replace this one.
-    /// If not dropped and reconstruct_pages is true, returns a new image layer containing the page versions
+    /// If not dropped, returns a new image layer containing the page versions
     /// at the `end_lsn`. Can also return a DeltaLayer that includes all the
     /// WAL records between start and end LSN. (The delta layer is not needed
     /// when a new relish is created with a single LSN, so that the start and
     /// end LSN are the same.)
-    pub fn write_to_disk(
-        &self,
-        timeline: &LayeredTimeline,
-        reconstruct_pages: bool,
-    ) -> Result<LayersOnDisk> {
+    pub fn write_to_disk(&self, timeline: &LayeredTimeline) -> Result<LayersOnDisk> {
         trace!(
             "write_to_disk {} get_end_lsn is {}",
             self.filename().display(),
@@ -606,102 +578,82 @@ impl InMemoryLayer {
         // would have to wait until we release it. That race condition is very
         // rare though, so we just accept the potential latency hit for now.
         let inner = self.inner.read().unwrap();
-
-        // Since `end_lsn` is exclusive, subtract 1 to calculate the last LSN
-        // that is included.
         let end_lsn_exclusive = inner.end_lsn.unwrap();
-        let end_lsn_inclusive = Lsn(end_lsn_exclusive.0 - 1);
 
-        // Figure out if we should create a delta layer, image layer, or both.
-        let image_lsn: Option<Lsn>;
-        let delta_end_lsn: Option<Lsn>;
-        if self.is_dropped() || !reconstruct_pages {
-            // The segment was dropped. Create just a delta layer containing all the
-            // changes up to and including the drop.
-            delta_end_lsn = Some(end_lsn_exclusive);
-            image_lsn = None;
-        } else if self.start_lsn == end_lsn_inclusive {
-            // The layer contains exactly one LSN. It's enough to write an image
-            // layer at that LSN.
-            delta_end_lsn = None;
-            image_lsn = Some(end_lsn_inclusive);
-        } else {
-            // Create a delta layer with all the changes up to the end LSN,
-            // and an image layer at the end LSN.
-            //
-            // Note that we the delta layer does *not* include the page versions
-            // at the end LSN. They are included in the image layer, and there's
-            // no need to store them twice.
-            delta_end_lsn = Some(end_lsn_inclusive);
-            image_lsn = Some(end_lsn_inclusive);
-        }
-
-        let mut delta_layers = Vec::new();
-        let mut image_layers = Vec::new();
-
-        if let Some(delta_end_lsn) = delta_end_lsn {
-            let mut delta_layer_writer = DeltaLayerWriter::new(
+        if inner.dropped {
+            let delta_layer = DeltaLayer::create(
                 self.conf,
                 self.timelineid,
                 self.tenantid,
                 self.seg,
                 self.start_lsn,
-                delta_end_lsn,
-                self.is_dropped(),
+                end_lsn_exclusive,
+                true,
+                inner.page_versions.ordered_page_version_iter(None),
+                inner.segsizes.clone(),
             )?;
+            trace!(
+                "freeze: created delta layer for dropped segment {} {}-{}",
+                self.seg,
+                self.start_lsn,
+                end_lsn_exclusive
+            );
+            return Ok(LayersOnDisk {
+                delta_layers: vec![delta_layer],
+                image_layers: Vec::new(),
+            });
+        }
 
-            // Write all page versions
-            let mut buf: Vec<u8> = Vec::new();
+        // Since `end_lsn` is inclusive, subtract 1.
+        // We want to make an ImageLayer for the last included LSN,
+        // so the DeltaLayer should exlcude that LSN.
+        let end_lsn_inclusive = Lsn(end_lsn_exclusive.0 - 1);
 
-            let page_versions_iter = inner
-                .page_versions
-                .ordered_page_version_iter(Some(delta_end_lsn));
-            for (blknum, lsn, pos) in page_versions_iter {
-                let len = inner.page_versions.read_pv_bytes(pos, &mut buf)?;
-                delta_layer_writer.put_page_version(blknum, lsn, &buf[..len])?;
-            }
+        let mut page_versions = inner
+            .page_versions
+            .ordered_page_version_iter(Some(end_lsn_inclusive));
 
-            // Create seg_sizes
-            let seg_sizes = if delta_end_lsn == end_lsn_exclusive {
-                inner.seg_sizes.clone()
-            } else {
-                inner.seg_sizes.split_at(&end_lsn_exclusive).0
-            };
+        let mut delta_layers = Vec::new();
 
-            let delta_layer = delta_layer_writer.finish(seg_sizes)?;
+        if self.start_lsn != end_lsn_inclusive {
+            let (segsizes, _) = inner.segsizes.split_at(&end_lsn_exclusive);
+            // Write the page versions before the cutoff to disk.
+            let delta_layer = DeltaLayer::create(
+                self.conf,
+                self.timelineid,
+                self.tenantid,
+                self.seg,
+                self.start_lsn,
+                end_lsn_inclusive,
+                false,
+                page_versions,
+                segsizes,
+            )?;
             delta_layers.push(delta_layer);
+            trace!(
+                "freeze: created delta layer {} {}-{}",
+                self.seg,
+                self.start_lsn,
+                end_lsn_inclusive
+            );
+        } else {
+            assert!(page_versions.next().is_none());
         }
 
         drop(inner);
 
         // Write a new base image layer at the cutoff point
-        if let Some(image_lsn) = image_lsn {
-            let size = if self.seg.rel.is_blocky() {
-                self.get_seg_size(image_lsn)?
-            } else {
-                1
-            };
-            let mut image_layer_writer = ImageLayerWriter::new(
-                self.conf,
-                self.timelineid,
-                self.tenantid,
-                self.seg,
-                image_lsn,
-                size,
-            )?;
-
-            for blknum in 0..size {
-                let img = timeline.materialize_page(self.seg, blknum, image_lsn, &*self)?;
-
-                image_layer_writer.put_page_image(&img)?;
-            }
-            let image_layer = image_layer_writer.finish()?;
-            image_layers.push(image_layer);
-        }
+        let image_layer =
+            ImageLayer::create_from_src(self.conf, timeline, self, end_lsn_inclusive)?;
+        trace!(
+            "freeze: created image layer {} at {}",
+            self.seg,
+            end_lsn_inclusive
+        );
 
         Ok(LayersOnDisk {
             delta_layers,
-            image_layers,
+            image_layers: vec![image_layer],
         })
     }
 }

pageserver/src/layered_repository/layer_map.rs

@@ -17,12 +17,11 @@ use anyhow::Result;
 use lazy_static::lazy_static;
 use std::cmp::Ordering;
 use std::collections::{BinaryHeap, HashMap};
+use std::sync::atomic::{self, AtomicU64};
 use std::sync::Arc;
 use zenith_metrics::{register_int_gauge, IntGauge};
 use zenith_utils::lsn::Lsn;
 
-use super::global_layer_map::{LayerId, GLOBAL_LAYER_MAP};
-
 lazy_static! {
     static ref NUM_INMEMORY_LAYERS: IntGauge =
         register_int_gauge!("pageserver_inmemory_layers", "Number of layers in memory")
@@ -32,6 +31,17 @@ lazy_static! {
             .expect("failed to define a metric");
 }
 
+static NEXT_LAYER_ID: AtomicU64 = AtomicU64::new(0);
+
+#[derive(PartialEq, Eq, Hash, Clone, Copy)]
+pub struct LayerId(u64);
+
+impl LayerId {
+    fn next() -> LayerId {
+        Self(NEXT_LAYER_ID.fetch_add(1, atomic::Ordering::Relaxed))
+    }
+}
+
 ///
 /// LayerMap tracks what layers exist on a timeline.
 ///
@@ -45,6 +55,8 @@ pub struct LayerMap {
     /// contains the oldest WAL record.
     open_layers: BinaryHeap<OpenLayerEntry>,
 
+    open_layers_by_id: HashMap<LayerId, Arc<InMemoryLayer>>,
+
     /// Generation number, used to distinguish newly inserted entries in the
     /// binary heap from older entries during checkpoint.
     current_generation: u64,
@@ -70,18 +82,21 @@ impl LayerMap {
     pub fn get_open(&self, tag: &SegmentTag) -> Option<Arc<InMemoryLayer>> {
         let segentry = self.segs.get(tag)?;
 
-        segentry
-            .open_layer_id
-            .and_then(|layer_id| GLOBAL_LAYER_MAP.read().unwrap().get(&layer_id))
+        segentry.open.as_ref().map(Arc::clone)
+    }
+
+    #[allow(dead_code)]
+    pub fn get_open_by_id(&self, layer_id: &LayerId) -> Option<Arc<InMemoryLayer>> {
+        self.open_layers_by_id.get(layer_id).cloned()
     }
 
     ///
     /// Insert an open in-memory layer
     ///
-    pub fn insert_open(&mut self, layer: Arc<InMemoryLayer>) {
+    pub fn insert_open(&mut self, layer: Arc<InMemoryLayer>) -> LayerId {
         let segentry = self.segs.entry(layer.get_seg_tag()).or_default();
 
-        let layer_id = segentry.update_open(Arc::clone(&layer));
+        segentry.update_open(Arc::clone(&layer));
 
         let oldest_pending_lsn = layer.get_oldest_pending_lsn();
 
@@ -90,46 +105,45 @@ impl LayerMap {
         // in the middle of a WAL record.
         assert!(oldest_pending_lsn.is_aligned());
 
+        let id = LayerId::next();
+
         // Also add it to the binary heap
         let open_layer_entry = OpenLayerEntry {
             oldest_pending_lsn: layer.get_oldest_pending_lsn(),
-            layer_id,
+            layer: Arc::clone(&layer),
             generation: self.current_generation,
+            id,
         };
         self.open_layers.push(open_layer_entry);
 
+        let old_layer = self.open_layers_by_id.insert(id, layer);
+        assert!(old_layer.is_none());
+
         NUM_INMEMORY_LAYERS.inc();
+
+        id
     }
 
-    /// Remove an open in-memory layer
-    pub fn remove_open(&mut self, layer_id: LayerId) {
-        // Note: we don't try to remove the entry from the binary heap.
-        // It will be removed lazily by peek_oldest_open() when it's made it to
-        // the top of the heap.
+    /// Remove the oldest in-memory layer
+    pub fn pop_oldest_open(&mut self) {
+        // Pop it from the binary heap
+        let oldest_entry = self.open_layers.pop().unwrap();
+        let segtag = oldest_entry.layer.get_seg_tag();
 
-        let layer_opt = {
-            let mut global_map = GLOBAL_LAYER_MAP.write().unwrap();
-            let layer_opt = global_map.get(&layer_id);
-            global_map.remove(&layer_id);
-            // TODO it's bad that a ref can still exist after being evicted from cache
-            layer_opt
-        };
-
-        if let Some(layer) = layer_opt {
-            let mut segentry = self.segs.get_mut(&layer.get_seg_tag()).unwrap();
-
-            if segentry.open_layer_id == Some(layer_id) {
-                // Also remove it from the SegEntry of this segment
-                segentry.open_layer_id = None;
-            } else {
-                // We could have already updated segentry.open for
-                // dropped (non-writeable) layer. This is fine.
-                assert!(!layer.is_writeable());
-                assert!(layer.is_dropped());
-            }
-
-            NUM_INMEMORY_LAYERS.dec();
+        // Also remove it from the SegEntry of this segment
+        let mut segentry = self.segs.get_mut(&segtag).unwrap();
+        if Arc::ptr_eq(segentry.open.as_ref().unwrap(), &oldest_entry.layer) {
+            segentry.open = None;
+        } else {
+            // We could have already updated segentry.open for
+            // dropped (non-writeable) layer. This is fine.
+            assert!(!oldest_entry.layer.is_writeable());
+            assert!(oldest_entry.layer.is_dropped());
         }
+
+        self.open_layers_by_id.remove(&oldest_entry.id).unwrap();
+
+        NUM_INMEMORY_LAYERS.dec();
     }
 
     ///
@@ -214,17 +228,10 @@ impl LayerMap {
     }
 
     /// Return the oldest in-memory layer, along with its generation number.
-    pub fn peek_oldest_open(&mut self) -> Option<(LayerId, Arc<InMemoryLayer>, u64)> {
-        let global_map = GLOBAL_LAYER_MAP.read().unwrap();
-
-        while let Some(oldest_entry) = self.open_layers.peek() {
-            if let Some(layer) = global_map.get(&oldest_entry.layer_id) {
-                return Some((oldest_entry.layer_id, layer, oldest_entry.generation));
-            } else {
-                self.open_layers.pop();
-            }
-        }
-        None
+    pub fn peek_oldest_open(&self) -> Option<(Arc<InMemoryLayer>, u64)> {
+        self.open_layers
+            .peek()
+            .map(|oldest_entry| (Arc::clone(&oldest_entry.layer), oldest_entry.generation))
     }
 
     /// Increment the generation number used to stamp open in-memory layers. Layers
@@ -247,12 +254,8 @@ impl LayerMap {
     pub fn dump(&self) -> Result<()> {
         println!("Begin dump LayerMap");
         for (seg, segentry) in self.segs.iter() {
-            if let Some(open) = &segentry.open_layer_id {
-                if let Some(layer) = GLOBAL_LAYER_MAP.read().unwrap().get(open) {
-                    layer.dump()?;
-                } else {
-                    println!("layer not found in global map");
-                }
+            if let Some(open) = &segentry.open {
+                open.dump()?;
             }
 
             for layer in segentry.historic.iter() {
@@ -285,7 +288,7 @@ impl IntervalItem for dyn Layer {
 /// IntervalTree.
 #[derive(Default)]
 struct SegEntry {
-    open_layer_id: Option<LayerId>,
+    open: Option<Arc<InMemoryLayer>>,
     historic: IntervalTree<dyn Layer>,
 }
 
@@ -301,10 +304,10 @@ impl SegEntry {
     }
 
     pub fn get(&self, lsn: Lsn) -> Option<Arc<dyn Layer>> {
-        if let Some(open_layer_id) = &self.open_layer_id {
-            let open_layer = GLOBAL_LAYER_MAP.read().unwrap().get(open_layer_id)?;
-            if open_layer.get_start_lsn() <= lsn {
-                return Some(open_layer);
+        if let Some(open) = &self.open {
+            if open.get_start_lsn() <= lsn {
+                let x: Arc<dyn Layer> = Arc::clone(open) as _;
+                return Some(x);
             }
         }
 
@@ -323,16 +326,11 @@ impl SegEntry {
     // Set new open layer for a SegEntry.
     // It's ok to rewrite previous open layer,
     // but only if it is not writeable anymore.
-    pub fn update_open(&mut self, layer: Arc<InMemoryLayer>) -> LayerId {
-        if let Some(prev_open_layer_id) = &self.open_layer_id {
-            if let Some(prev_open_layer) = GLOBAL_LAYER_MAP.read().unwrap().get(prev_open_layer_id)
-            {
-                assert!(!prev_open_layer.is_writeable());
-            }
+    pub fn update_open(&mut self, layer: Arc<InMemoryLayer>) {
+        if let Some(prev_open) = &self.open {
+            assert!(!prev_open.is_writeable());
         }
-        let open_layer_id = GLOBAL_LAYER_MAP.write().unwrap().insert(layer);
-        self.open_layer_id = Some(open_layer_id);
-        open_layer_id
+        self.open = Some(layer);
     }
 
     pub fn insert_historic(&mut self, layer: Arc<dyn Layer>) {
@@ -347,9 +345,10 @@ impl SegEntry {
 /// recently-added entries (i.e after last call to increment_generation()) from older
 /// entries with the same 'oldest_pending_lsn'.
 struct OpenLayerEntry {
-    oldest_pending_lsn: Lsn, // copy of layer.get_oldest_pending_lsn()
-    generation: u64,
-    layer_id: LayerId,
+    pub oldest_pending_lsn: Lsn, // copy of layer.get_oldest_pending_lsn()
+    pub generation: u64,
+    pub layer: Arc<InMemoryLayer>,
+    id: LayerId,
 }
 impl Ord for OpenLayerEntry {
     fn cmp(&self, other: &Self) -> Ordering {
@@ -402,7 +401,7 @@ impl<'a> Iterator for HistoricLayerIter<'a> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::config::PageServerConf;
+    use crate::PageServerConf;
     use std::str::FromStr;
     use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
@@ -414,13 +413,6 @@ mod tests {
         forknum: 0,
     });
 
-    lazy_static! {
-        static ref DUMMY_TIMELINEID: ZTimelineId =
-            ZTimelineId::from_str("00000000000000000000000000000000").unwrap();
-        static ref DUMMY_TENANTID: ZTenantId =
-            ZTenantId::from_str("00000000000000000000000000000000").unwrap();
-    }
-
     /// Construct a dummy InMemoryLayer for testing
     fn dummy_inmem_layer(
         conf: &'static PageServerConf,
@@ -431,8 +423,8 @@ mod tests {
         Arc::new(
             InMemoryLayer::create(
                 conf,
-                *DUMMY_TIMELINEID,
-                *DUMMY_TENANTID,
+                ZTimelineId::from_str("00000000000000000000000000000000").unwrap(),
+                ZTenantId::from_str("00000000000000000000000000000000").unwrap(),
                 SegmentTag {
                     rel: TESTREL_A,
                     segno,
@@ -448,7 +440,6 @@ mod tests {
     fn test_open_layers() -> Result<()> {
         let conf = PageServerConf::dummy_conf(PageServerConf::test_repo_dir("dummy_inmem_layer"));
         let conf = Box::leak(Box::new(conf));
-        std::fs::create_dir_all(conf.timeline_path(&DUMMY_TIMELINEID, &DUMMY_TENANTID))?;
 
         let mut layers = LayerMap::default();
 
@@ -465,10 +456,10 @@ mod tests {
         // A helper function (closure) to pop the next oldest open entry from the layer map,
         // and assert that it is what we'd expect
         let mut assert_pop_layer = |expected_segno: u32, expected_generation: u64| {
-            let (layer_id, l, generation) = layers.peek_oldest_open().unwrap();
+            let (l, generation) = layers.peek_oldest_open().unwrap();
             assert!(l.get_seg_tag().segno == expected_segno);
             assert!(generation == expected_generation);
-            layers.remove_open(layer_id);
+            layers.pop_oldest_open();
         };
 
         assert_pop_layer(0, gen1); // 0x100

pageserver/src/layered_repository/metadata.rs

@@ -1,7 +1,7 @@
 //! Every image of a certain timeline from [`crate::layered_repository::LayeredRepository`]
 //! has a metadata that needs to be stored persistently.
 //!
-//! Later, the file gets is used in [`crate::remote_storage::storage_sync`] as a part of
+//! Later, the file gets is used in [`crate::relish_storage::storage_sync`] as a part of
 //! external storage import and export operations.
 //!
 //! The module contains all structs and related helper methods related to timeline metadata.
@@ -15,12 +15,10 @@ use zenith_utils::{
     zid::{ZTenantId, ZTimelineId},
 };
 
-use crate::config::PageServerConf;
-
-// Taken from PG_CONTROL_MAX_SAFE_SIZE
-const METADATA_MAX_SAFE_SIZE: usize = 512;
-const METADATA_CHECKSUM_SIZE: usize = std::mem::size_of::<u32>();
-const METADATA_MAX_DATA_SIZE: usize = METADATA_MAX_SAFE_SIZE - METADATA_CHECKSUM_SIZE;
+use crate::{
+    layered_repository::{METADATA_CHECKSUM_SIZE, METADATA_MAX_DATA_SIZE, METADATA_MAX_SAFE_SIZE},
+    PageServerConf,
+};
 
 /// The name of the metadata file pageserver creates per timeline.
 pub const METADATA_FILE_NAME: &str = "metadata";
@@ -44,8 +42,6 @@ pub struct TimelineMetadata {
     prev_record_lsn: Option<Lsn>,
     ancestor_timeline: Option<ZTimelineId>,
     ancestor_lsn: Lsn,
-    latest_gc_cutoff_lsn: Lsn,
-    initdb_lsn: Lsn,
 }
 
 /// Points to a place in pageserver's local directory,
@@ -65,16 +61,12 @@ impl TimelineMetadata {
         prev_record_lsn: Option<Lsn>,
         ancestor_timeline: Option<ZTimelineId>,
         ancestor_lsn: Lsn,
-        latest_gc_cutoff_lsn: Lsn,
-        initdb_lsn: Lsn,
     ) -> Self {
         Self {
             disk_consistent_lsn,
             prev_record_lsn,
             ancestor_timeline,
             ancestor_lsn,
-            latest_gc_cutoff_lsn,
-            initdb_lsn,
         }
     }
 
@@ -129,14 +121,6 @@ impl TimelineMetadata {
     pub fn ancestor_lsn(&self) -> Lsn {
         self.ancestor_lsn
     }
-
-    pub fn latest_gc_cutoff_lsn(&self) -> Lsn {
-        self.latest_gc_cutoff_lsn
-    }
-
-    pub fn initdb_lsn(&self) -> Lsn {
-        self.initdb_lsn
-    }
 }
 
 /// This module is for direct conversion of metadata to bytes and back.
@@ -155,8 +139,6 @@ mod serialize {
         prev_record_lsn: &'a Option<Lsn>,
         ancestor_timeline: &'a Option<ZTimelineId>,
         ancestor_lsn: &'a Lsn,
-        latest_gc_cutoff_lsn: &'a Lsn,
-        initdb_lsn: &'a Lsn,
     }
 
     impl<'a> From<&'a TimelineMetadata> for SeTimelineMetadata<'a> {
@@ -166,8 +148,6 @@ mod serialize {
                 prev_record_lsn: &other.prev_record_lsn,
                 ancestor_timeline: &other.ancestor_timeline,
                 ancestor_lsn: &other.ancestor_lsn,
-                latest_gc_cutoff_lsn: &other.latest_gc_cutoff_lsn,
-                initdb_lsn: &other.initdb_lsn,
             }
         }
     }
@@ -178,8 +158,6 @@ mod serialize {
         prev_record_lsn: Option<Lsn>,
         ancestor_timeline: Option<ZTimelineId>,
         ancestor_lsn: Lsn,
-        latest_gc_cutoff_lsn: Lsn,
-        initdb_lsn: Lsn,
     }
 
     impl From<DeTimelineMetadata> for TimelineMetadata {
@@ -189,8 +167,6 @@ mod serialize {
                 prev_record_lsn: other.prev_record_lsn,
                 ancestor_timeline: other.ancestor_timeline,
                 ancestor_lsn: other.ancestor_lsn,
-                latest_gc_cutoff_lsn: other.latest_gc_cutoff_lsn,
-                initdb_lsn: other.initdb_lsn,
             }
         }
     }
@@ -209,8 +185,6 @@ mod tests {
             prev_record_lsn: Some(Lsn(0x100)),
             ancestor_timeline: Some(TIMELINE_ID),
             ancestor_lsn: Lsn(0),
-            latest_gc_cutoff_lsn: Lsn(0),
-            initdb_lsn: Lsn(0),
         };
 
         let metadata_bytes = original_metadata

pageserver/src/layered_repository/page_versions.rs

@@ -1,78 +1,40 @@
-//!
-//! Data structure to ingest incoming WAL into an append-only file.
-//!
-//! - The file is considered temporary, and will be discarded on crash
-//! - based on a B-tree
-//!
-
-use std::os::unix::fs::FileExt;
 use std::{collections::HashMap, ops::RangeBounds, slice};
 
-use anyhow::Result;
-
-use std::cmp::min;
-use std::io::Seek;
-
 use zenith_utils::{lsn::Lsn, vec_map::VecMap};
 
 use super::storage_layer::PageVersion;
-use crate::layered_repository::ephemeral_file::EphemeralFile;
 
-use zenith_utils::bin_ser::BeSer;
+const EMPTY_SLICE: &[(Lsn, PageVersion)] = &[];
 
-const EMPTY_SLICE: &[(Lsn, u64)] = &[];
-
-pub struct PageVersions {
-    map: HashMap<u32, VecMap<Lsn, u64>>,
-
-    /// The PageVersion structs are stored in a serialized format in this file.
-    /// Each serialized PageVersion is preceded by a 'u32' length field.
-    /// The 'map' stores offsets into this file.
-    file: EphemeralFile,
-}
+#[derive(Debug, Default)]
+pub struct PageVersions(HashMap<u32, VecMap<Lsn, PageVersion>>);
 
 impl PageVersions {
-    pub fn new(file: EphemeralFile) -> PageVersions {
-        PageVersions {
-            map: HashMap::new(),
-            file,
-        }
-    }
-
     pub fn append_or_update_last(
         &mut self,
         blknum: u32,
         lsn: Lsn,
         page_version: PageVersion,
-    ) -> Result<Option<u64>> {
-        // remember starting position
-        let pos = self.file.stream_position()?;
-
-        // make room for the 'length' field by writing zeros as a placeholder.
-        self.file.seek(std::io::SeekFrom::Start(pos + 4)).unwrap();
-
-        page_version.ser_into(&mut self.file).unwrap();
-
-        // write the 'length' field.
-        let len = self.file.stream_position()? - pos - 4;
-        let lenbuf = u32::to_ne_bytes(len as u32);
-        self.file.write_all_at(&lenbuf, pos)?;
-
-        let map = self.map.entry(blknum).or_insert_with(VecMap::default);
-        Ok(map.append_or_update_last(lsn, pos as u64).unwrap().0)
+    ) -> Option<PageVersion> {
+        let map = self.0.entry(blknum).or_insert_with(VecMap::default);
+        map.append_or_update_last(lsn, page_version).unwrap()
     }
 
     /// Get all [`PageVersion`]s in a block
-    fn get_block_slice(&self, blknum: u32) -> &[(Lsn, u64)] {
-        self.map
+    pub fn get_block_slice(&self, blknum: u32) -> &[(Lsn, PageVersion)] {
+        self.0
             .get(&blknum)
             .map(VecMap::as_slice)
             .unwrap_or(EMPTY_SLICE)
     }
 
     /// Get a range of [`PageVersions`] in a block
-    pub fn get_block_lsn_range<R: RangeBounds<Lsn>>(&self, blknum: u32, range: R) -> &[(Lsn, u64)] {
-        self.map
+    pub fn get_block_lsn_range<R: RangeBounds<Lsn>>(
+        &self,
+        blknum: u32,
+        range: R,
+    ) -> &[(Lsn, PageVersion)] {
+        self.0
             .get(&blknum)
             .map(|vec_map| vec_map.slice_range(range))
             .unwrap_or(EMPTY_SLICE)
@@ -81,7 +43,7 @@ impl PageVersions {
     /// Iterate through [`PageVersion`]s in (block, lsn) order.
     /// If a [`cutoff_lsn`] is set, only show versions with `lsn < cutoff_lsn`
     pub fn ordered_page_version_iter(&self, cutoff_lsn: Option<Lsn>) -> OrderedPageVersionIter<'_> {
-        let mut ordered_blocks: Vec<u32> = self.map.keys().cloned().collect();
+        let mut ordered_blocks: Vec<u32> = self.0.keys().cloned().collect();
         ordered_blocks.sort_unstable();
 
         let slice = ordered_blocks
@@ -97,56 +59,6 @@ impl PageVersions {
             cur_slice_iter: slice.iter(),
         }
     }
-
-    ///
-    /// Read a page version.
-    ///
-    pub fn read_pv(&self, off: u64) -> Result<PageVersion> {
-        let mut buf = Vec::new();
-        self.read_pv_bytes(off, &mut buf)?;
-        Ok(PageVersion::des(&buf)?)
-    }
-
-    ///
-    /// Read a page version, as raw bytes, at the given offset. The bytes
-    /// are read into 'buf', which is expanded if necessary. Returns the
-    /// size of the page version.
-    ///
-    pub fn read_pv_bytes(&self, off: u64, buf: &mut Vec<u8>) -> Result<usize> {
-        // read length
-        let mut lenbuf = [0u8; 4];
-        self.file.read_exact_at(&mut lenbuf, off)?;
-        let len = u32::from_ne_bytes(lenbuf) as usize;
-
-        // Resize the buffer to fit the data, if needed.
-        //
-        // We don't shrink the buffer if it's larger than necessary. That avoids
-        // repeatedly shrinking and expanding when you reuse the same buffer to
-        // read multiple page versions. Expanding a Vec requires initializing the
-        // new bytes, which is a waste of time because we're immediately overwriting
-        // it, but there's no way to avoid it without resorting to unsafe code.
-        if buf.len() < len {
-            buf.resize(len, 0);
-        }
-        self.file.read_exact_at(&mut buf[0..len], off + 4)?;
-
-        Ok(len)
-    }
-}
-
-pub struct PageVersionReader<'a> {
-    file: &'a EphemeralFile,
-    pos: u64,
-    end_pos: u64,
-}
-
-impl<'a> std::io::Read for PageVersionReader<'a> {
-    fn read(&mut self, buf: &mut [u8]) -> Result<usize, std::io::Error> {
-        let len = min(buf.len(), (self.end_pos - self.pos) as usize);
-        let n = self.file.read_at(&mut buf[..len], self.pos)?;
-        self.pos += n as u64;
-        Ok(n)
-    }
 }
 
 pub struct OrderedPageVersionIter<'a> {
@@ -157,7 +69,7 @@ pub struct OrderedPageVersionIter<'a> {
 
     cutoff_lsn: Option<Lsn>,
 
-    cur_slice_iter: slice::Iter<'a, (Lsn, u64)>,
+    cur_slice_iter: slice::Iter<'a, (Lsn, PageVersion)>,
 }
 
 impl OrderedPageVersionIter<'_> {
@@ -171,14 +83,14 @@ impl OrderedPageVersionIter<'_> {
 }
 
 impl<'a> Iterator for OrderedPageVersionIter<'a> {
-    type Item = (u32, Lsn, u64);
+    type Item = (u32, Lsn, &'a PageVersion);
 
     fn next(&mut self) -> Option<Self::Item> {
         loop {
-            if let Some((lsn, pos)) = self.cur_slice_iter.next() {
+            if let Some((lsn, page_version)) = self.cur_slice_iter.next() {
                 if self.is_lsn_before_cutoff(lsn) {
                     let blknum = self.ordered_blocks[self.cur_block_idx];
-                    return Some((blknum, *lsn, *pos));
+                    return Some((blknum, *lsn, page_version));
                 }
             }
 
@@ -192,50 +104,22 @@ impl<'a> Iterator for OrderedPageVersionIter<'a> {
 
 #[cfg(test)]
 mod tests {
-    use bytes::Bytes;
-
     use super::*;
-    use crate::config::PageServerConf;
-    use std::fs;
-    use std::str::FromStr;
-    use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
-    fn repo_harness(test_name: &str) -> Result<(&'static PageServerConf, ZTenantId, ZTimelineId)> {
-        let repo_dir = PageServerConf::test_repo_dir(test_name);
-        let _ = fs::remove_dir_all(&repo_dir);
-        let conf = PageServerConf::dummy_conf(repo_dir);
-        // Make a static copy of the config. This can never be free'd, but that's
-        // OK in a test.
-        let conf: &'static PageServerConf = Box::leak(Box::new(conf));
-
-        let tenantid = ZTenantId::from_str("11000000000000000000000000000000").unwrap();
-        let timelineid = ZTimelineId::from_str("22000000000000000000000000000000").unwrap();
-        fs::create_dir_all(conf.timeline_path(&timelineid, &tenantid))?;
-
-        Ok((conf, tenantid, timelineid))
-    }
+    const EMPTY_PAGE_VERSION: PageVersion = PageVersion {
+        page_image: None,
+        record: None,
+    };
 
     #[test]
-    fn test_ordered_iter() -> Result<()> {
-        let (conf, tenantid, timelineid) = repo_harness("test_ordered_iter")?;
-
-        let file = EphemeralFile::create(conf, tenantid, timelineid)?;
-
-        let mut page_versions = PageVersions::new(file);
-
+    fn test_ordered_iter() {
+        let mut page_versions = PageVersions::default();
         const BLOCKS: u32 = 1000;
         const LSNS: u64 = 50;
 
-        let empty_page = Bytes::from_static(&[0u8; 8192]);
-        let empty_page_version = PageVersion::Page(empty_page);
-
         for blknum in 0..BLOCKS {
             for lsn in 0..LSNS {
-                let old = page_versions.append_or_update_last(
-                    blknum,
-                    Lsn(lsn),
-                    empty_page_version.clone(),
-                )?;
+                let old = page_versions.append_or_update_last(blknum, Lsn(lsn), EMPTY_PAGE_VERSION);
                 assert!(old.is_none());
             }
         }
@@ -262,7 +146,5 @@ mod tests {
         }
         assert!(iter.next().is_none());
         assert!(iter.next().is_none()); // should be robust against excessive next() calls
-
-        Ok(())
     }
 }

pageserver/src/layered_repository/par_fsync.rs

@@ -1,55 +0,0 @@
-use std::{
-    io,
-    path::{Path, PathBuf},
-    sync::atomic::{AtomicUsize, Ordering},
-};
-
-use crate::virtual_file::VirtualFile;
-
-fn fsync_path(path: &Path) -> io::Result<()> {
-    let file = VirtualFile::open(path)?;
-    file.sync_all()
-}
-
-fn parallel_worker(paths: &[PathBuf], next_path_idx: &AtomicUsize) -> io::Result<()> {
-    while let Some(path) = paths.get(next_path_idx.fetch_add(1, Ordering::Relaxed)) {
-        fsync_path(path)?;
-    }
-
-    Ok(())
-}
-
-pub fn par_fsync(paths: &[PathBuf]) -> io::Result<()> {
-    const PARALLEL_PATH_THRESHOLD: usize = 1;
-    if paths.len() <= PARALLEL_PATH_THRESHOLD {
-        for path in paths {
-            fsync_path(path)?;
-        }
-        return Ok(());
-    }
-
-    /// Use at most this number of threads.
-    /// Increasing this limit will
-    /// - use more memory
-    /// - increase the cost of spawn/join latency
-    const MAX_NUM_THREADS: usize = 64;
-    let num_threads = paths.len().min(MAX_NUM_THREADS);
-    let next_path_idx = AtomicUsize::new(0);
-
-    crossbeam_utils::thread::scope(|s| -> io::Result<()> {
-        let mut handles = vec![];
-        // Spawn `num_threads - 1`, as the current thread is also a worker.
-        for _ in 1..num_threads {
-            handles.push(s.spawn(|_| parallel_worker(paths, &next_path_idx)));
-        }
-
-        parallel_worker(paths, &next_path_idx)?;
-
-        for handle in handles {
-            handle.join().unwrap()?;
-        }
-
-        Ok(())
-    })
-    .unwrap()
-}

pageserver/src/layered_repository/storage_layer.rs

@@ -3,8 +3,8 @@
 //!
 
 use crate::relish::RelishTag;
-use crate::repository::{BlockNumber, ZenithWalRecord};
-use crate::{ZTenantId, ZTimelineId};
+use crate::repository::WALRecord;
+use crate::ZTimelineId;
 use anyhow::Result;
 use bytes::Bytes;
 use serde::{Deserialize, Serialize};
@@ -26,18 +26,6 @@ pub struct SegmentTag {
     pub segno: u32,
 }
 
-/// SegmentBlk represents a block number within a segment, or the size of segment.
-///
-/// This is separate from BlockNumber, which is used for block number within the
-/// whole relish. Since this is just a type alias, the compiler will let you mix
-/// them freely, but we use the type alias as documentation to make it clear
-/// which one we're dealing with.
-///
-/// (We could turn this into "struct SegmentBlk(u32)" to forbid accidentally
-/// assigning a BlockNumber to SegmentBlk or vice versa, but that makes
-/// operations more verbose).
-pub type SegmentBlk = u32;
-
 impl fmt::Display for SegmentTag {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "{}.{}", self.rel, self.segno)
@@ -45,16 +33,15 @@ impl fmt::Display for SegmentTag {
 }
 
 impl SegmentTag {
-    /// Given a relish and block number, calculate the corresponding segment and
-    /// block number within the segment.
-    pub const fn from_blknum(rel: RelishTag, blknum: BlockNumber) -> (SegmentTag, SegmentBlk) {
-        (
-            SegmentTag {
-                rel,
-                segno: blknum / RELISH_SEG_SIZE,
-            },
-            blknum % RELISH_SEG_SIZE,
-        )
+    pub const fn from_blknum(rel: RelishTag, blknum: u32) -> SegmentTag {
+        SegmentTag {
+            rel,
+            segno: blknum / RELISH_SEG_SIZE,
+        }
+    }
+
+    pub fn blknum_in_seg(&self, blknum: u32) -> bool {
+        blknum / RELISH_SEG_SIZE == self.segno
     }
 }
 
@@ -64,10 +51,23 @@ impl SegmentTag {
 ///
 /// A page version can be stored as a full page image, or as WAL record that needs
 /// to be applied over the previous page version to reconstruct this version.
+///
+/// It's also possible to have both a WAL record and a page image in the same
+/// PageVersion. That happens if page version is originally stored as a WAL record
+/// but it is later reconstructed by a GetPage@LSN request by performing WAL
+/// redo. The get_page_at_lsn() code will store the reconstructed pag image next to
+/// the WAL record in that case. TODO: That's pretty accidental, not the result
+/// of any grand design. If we want to keep reconstructed page versions around, we
+/// probably should have a separate buffer cache so that we could control the
+/// replacement policy globally. Or if we keep a reconstructed page image, we
+/// could throw away the WAL record.
+///
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum PageVersion {
-    Page(Bytes),
-    Wal(ZenithWalRecord),
+pub struct PageVersion {
+    /// an 8kb page image
+    pub page_image: Option<Bytes>,
+    /// WAL record to get from previous page version to this one.
+    pub record: Option<WALRecord>,
 }
 
 ///
@@ -78,7 +78,7 @@ pub enum PageVersion {
 /// 'records' contains the records to apply over the base image.
 ///
 pub struct PageReconstructData {
-    pub records: Vec<(Lsn, ZenithWalRecord)>,
+    pub records: Vec<(Lsn, WALRecord)>,
     pub page_img: Option<Bytes>,
 }
 
@@ -93,8 +93,6 @@ pub enum PageReconstructResult {
     /// the returned LSN. This is usually considered an error, but might be OK
     /// in some circumstances.
     Missing(Lsn),
-    /// Use the cached image at `cached_img_lsn` as the base image
-    Cached,
 }
 
 ///
@@ -106,8 +104,6 @@ pub enum PageReconstructResult {
 /// in-memory and on-disk layers.
 ///
 pub trait Layer: Send + Sync {
-    fn get_tenant_id(&self) -> ZTenantId;
-
     /// Identify the timeline this relish belongs to
     fn get_timeline_id(&self) -> ZTimelineId;
 
@@ -138,8 +134,9 @@ pub trait Layer: Send + Sync {
     /// It is up to the caller to collect more data from previous layer and
     /// perform WAL redo, if necessary.
     ///
-    /// `cached_img_lsn` should be set to a cached page image's lsn < `lsn`.
-    /// This function will only return data after `cached_img_lsn`.
+    /// Note that the 'blknum' is the offset of the page from the beginning
+    /// of the *relish*, not the beginning of the segment. The requested
+    /// 'blknum' must be covered by this segment.
     ///
     /// See PageReconstructResult for possible return values. The collected data
     /// is appended to reconstruct_data; the caller should pass an empty struct
@@ -148,14 +145,13 @@ pub trait Layer: Send + Sync {
     /// to collect more data.
     fn get_page_reconstruct_data(
         &self,
-        blknum: SegmentBlk,
+        blknum: u32,
         lsn: Lsn,
-        cached_img_lsn: Option<Lsn>,
         reconstruct_data: &mut PageReconstructData,
     ) -> Result<PageReconstructResult>;
 
     /// Return size of the segment at given LSN. (Only for blocky relations.)
-    fn get_seg_size(&self, lsn: Lsn) -> Result<SegmentBlk>;
+    fn get_seg_size(&self, lsn: Lsn) -> Result<u32>;
 
     /// Does the segment exist at given LSN? Or was it dropped before it.
     fn get_seg_exists(&self, lsn: Lsn) -> Result<bool>;
@@ -166,9 +162,6 @@ pub trait Layer: Send + Sync {
     /// the previous non-incremental layer.
     fn is_incremental(&self) -> bool;
 
-    /// Returns true for layers that are represented in memory.
-    fn is_in_memory(&self) -> bool;
-
     /// Release memory used by this layer. There is no corresponding 'load'
     /// function, that's done implicitly when you call one of the get-functions.
     fn unload(&self) -> Result<()>;

pageserver/src/lib.rs

@@ -1,25 +1,49 @@
-pub mod basebackup;
-pub mod branches;
-pub mod config;
-pub mod http;
-pub mod import_datadir;
-pub mod layered_repository;
-pub mod page_cache;
-pub mod page_service;
-pub mod relish;
-pub mod remote_storage;
-pub mod repository;
-pub mod tenant_mgr;
-pub mod tenant_threads;
-pub mod virtual_file;
-pub mod walingest;
-pub mod walreceiver;
-pub mod walrecord;
-pub mod walredo;
+use layered_repository::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME};
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
+
+use std::path::PathBuf;
+use std::time::Duration;
 
 use lazy_static::lazy_static;
 use zenith_metrics::{register_int_gauge_vec, IntGaugeVec};
-use zenith_utils::zid::{ZTenantId, ZTimelineId};
+
+pub mod basebackup;
+pub mod branches;
+pub mod http;
+pub mod layered_repository;
+pub mod page_service;
+pub mod relish;
+pub mod relish_storage;
+pub mod repository;
+pub mod restore_local_repo;
+pub mod tenant_mgr;
+pub mod tenant_threads;
+pub mod waldecoder;
+pub mod walreceiver;
+pub mod walredo;
+
+pub mod defaults {
+    use const_format::formatcp;
+    use std::time::Duration;
+
+    pub const DEFAULT_PG_LISTEN_PORT: u16 = 64000;
+    pub const DEFAULT_PG_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_PG_LISTEN_PORT}");
+    pub const DEFAULT_HTTP_LISTEN_PORT: u16 = 9898;
+    pub const DEFAULT_HTTP_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_HTTP_LISTEN_PORT}");
+
+    // FIXME: This current value is very low. I would imagine something like 1 GB or 10 GB
+    // would be more appropriate. But a low value forces the code to be exercised more,
+    // which is good for now to trigger bugs.
+    pub const DEFAULT_CHECKPOINT_DISTANCE: u64 = 256 * 1024 * 1024;
+    pub const DEFAULT_CHECKPOINT_PERIOD: Duration = Duration::from_secs(1);
+
+    pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
+    pub const DEFAULT_GC_PERIOD: Duration = Duration::from_secs(100);
+
+    pub const DEFAULT_SUPERUSER: &str = "zenith_admin";
+    pub const DEFAULT_RELISH_STORAGE_MAX_CONCURRENT_SYNC_LIMITS: usize = 100;
+}
 
 lazy_static! {
     static ref LIVE_CONNECTIONS_COUNT: IntGaugeVec = register_int_gauge_vec!(
@@ -32,13 +56,165 @@ lazy_static! {
 
 pub const LOG_FILE_NAME: &str = "pageserver.log";
 
+#[derive(Debug, Clone)]
+pub struct PageServerConf {
+    pub daemonize: bool,
+    pub listen_pg_addr: String,
+    pub listen_http_addr: String,
+    // Flush out an inmemory layer, if it's holding WAL older than this
+    // This puts a backstop on how much WAL needs to be re-digested if the
+    // page server crashes.
+    pub checkpoint_distance: u64,
+    pub checkpoint_period: Duration,
+
+    pub gc_horizon: u64,
+    pub gc_period: Duration,
+    pub superuser: String,
+
+    // Repository directory, relative to current working directory.
+    // Normally, the page server changes the current working directory
+    // to the repository, and 'workdir' is always '.'. But we don't do
+    // that during unit testing, because the current directory is global
+    // to the process but different unit tests work on different
+    // repositories.
+    pub workdir: PathBuf,
+
+    pub pg_distrib_dir: PathBuf,
+
+    pub auth_type: AuthType,
+
+    pub auth_validation_public_key_path: Option<PathBuf>,
+    pub relish_storage_config: Option<RelishStorageConfig>,
+}
+
+impl PageServerConf {
+    //
+    // Repository paths, relative to workdir.
+    //
+
+    fn tenants_path(&self) -> PathBuf {
+        self.workdir.join(TENANTS_SEGMENT_NAME)
+    }
+
+    fn tenant_path(&self, tenantid: &ZTenantId) -> PathBuf {
+        self.tenants_path().join(tenantid.to_string())
+    }
+
+    fn tags_path(&self, tenantid: &ZTenantId) -> PathBuf {
+        self.tenant_path(tenantid).join("refs").join("tags")
+    }
+
+    fn tag_path(&self, tag_name: &str, tenantid: &ZTenantId) -> PathBuf {
+        self.tags_path(tenantid).join(tag_name)
+    }
+
+    fn branches_path(&self, tenantid: &ZTenantId) -> PathBuf {
+        self.tenant_path(tenantid).join("refs").join("branches")
+    }
+
+    fn branch_path(&self, branch_name: &str, tenantid: &ZTenantId) -> PathBuf {
+        self.branches_path(tenantid).join(branch_name)
+    }
+
+    fn timelines_path(&self, tenantid: &ZTenantId) -> PathBuf {
+        self.tenant_path(tenantid).join(TIMELINES_SEGMENT_NAME)
+    }
+
+    fn timeline_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
+        self.timelines_path(tenantid).join(timelineid.to_string())
+    }
+
+    fn ancestor_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf {
+        self.timeline_path(timelineid, tenantid).join("ancestor")
+    }
+
+    //
+    // Postgres distribution paths
+    //
+
+    pub fn pg_bin_dir(&self) -> PathBuf {
+        self.pg_distrib_dir.join("bin")
+    }
+
+    pub fn pg_lib_dir(&self) -> PathBuf {
+        self.pg_distrib_dir.join("lib")
+    }
+
+    #[cfg(test)]
+    fn test_repo_dir(test_name: &str) -> PathBuf {
+        PathBuf::from(format!("../tmp_check/test_{}", test_name))
+    }
+
+    #[cfg(test)]
+    fn dummy_conf(repo_dir: PathBuf) -> Self {
+        PageServerConf {
+            daemonize: false,
+            checkpoint_distance: defaults::DEFAULT_CHECKPOINT_DISTANCE,
+            checkpoint_period: Duration::from_secs(10),
+            gc_horizon: defaults::DEFAULT_GC_HORIZON,
+            gc_period: Duration::from_secs(10),
+            listen_pg_addr: defaults::DEFAULT_PG_LISTEN_ADDR.to_string(),
+            listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(),
+            superuser: "zenith_admin".to_string(),
+            workdir: repo_dir,
+            pg_distrib_dir: "".into(),
+            auth_type: AuthType::Trust,
+            auth_validation_public_key_path: None,
+            relish_storage_config: None,
+        }
+    }
+}
+
 /// Config for the Repository checkpointer
 #[derive(Debug, Clone, Copy)]
 pub enum CheckpointConfig {
     // Flush in-memory data that is older than this
     Distance(u64),
     // Flush all in-memory data
-    Flush,
-    // Flush all in-memory data and reconstruct all page images
     Forced,
 }
+
+/// External relish storage configuration, enough for creating a client for that storage.
+#[derive(Debug, Clone)]
+pub struct RelishStorageConfig {
+    /// Limits the number of concurrent sync operations between pageserver and relish storage.
+    pub max_concurrent_sync: usize,
+    /// The storage connection configuration.
+    pub storage: RelishStorageKind,
+}
+
+/// A kind of a relish storage to connect to, with its connection configuration.
+#[derive(Debug, Clone)]
+pub enum RelishStorageKind {
+    /// Storage based on local file system.
+    /// Specify a root folder to place all stored relish data into.
+    LocalFs(PathBuf),
+    /// AWS S3 based storage, storing all relishes into the root
+    /// of the S3 bucket from the config.
+    AwsS3(S3Config),
+}
+
+/// AWS S3 bucket coordinates and access credentials to manage the bucket contents (read and write).
+#[derive(Clone)]
+pub struct S3Config {
+    /// Name of the bucket to connect to.
+    pub bucket_name: String,
+    /// The region where the bucket is located at.
+    pub bucket_region: String,
+    /// "Login" to use when connecting to bucket.
+    /// Can be empty for cases like AWS k8s IAM
+    /// where we can allow certain pods to connect
+    /// to the bucket directly without any credentials.
+    pub access_key_id: Option<String>,
+    /// "Password" to use when connecting to bucket.
+    pub secret_access_key: Option<String>,
+}
+
+impl std::fmt::Debug for S3Config {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("S3Config")
+            .field("bucket_name", &self.bucket_name)
+            .field("bucket_region", &self.bucket_region)
+            .finish()
+    }
+}

pageserver/src/page_cache.rs

@@ -1,778 +0,0 @@
-//!
-//! Global page cache
-//!
-//! The page cache uses up most of the memory in the page server. It is shared
-//! by all tenants, and it is used to store different kinds of pages. Sharing
-//! the cache allows memory to be dynamically allocated where it's needed the
-//! most.
-//!
-//! The page cache consists of fixed-size buffers, 8 kB each to match the
-//! PostgreSQL buffer size, and a Slot struct for each buffer to contain
-//! information about what's stored in the buffer.
-//!
-//! # Locking
-//!
-//! There are two levels of locking involved: There's one lock for the "mapping"
-//! from page identifier (tenant ID, timeline ID, rel, block, LSN) to the buffer
-//! slot, and a separate lock on each slot. To read or write the contents of a
-//! slot, you must hold the lock on the slot in read or write mode,
-//! respectively. To change the mapping of a slot, i.e. to evict a page or to
-//! assign a buffer for a page, you must hold the mapping lock and the lock on
-//! the slot at the same time.
-//!
-//! Whenever you need to hold both locks simultenously, the slot lock must be
-//! acquired first. This consistent ordering avoids deadlocks. To look up a page
-//! in the cache, you would first look up the mapping, while holding the mapping
-//! lock, and then lock the slot. You must release the mapping lock in between,
-//! to obey the lock ordering and avoid deadlock.
-//!
-//! A slot can momentarily have invalid contents, even if it's already been
-//! inserted to the mapping, but you must hold the write-lock on the slot until
-//! the contents are valid. If you need to release the lock without initializing
-//! the contents, you must remove the mapping first. We make that easy for the
-//! callers with PageWriteGuard: when lock_for_write() returns an uninitialized
-//! page, the caller must explicitly call guard.mark_valid() after it has
-//! initialized it. If the guard is dropped without calling mark_valid(), the
-//! mapping is automatically removed and the slot is marked free.
-//!
-
-use std::{
-    collections::{hash_map::Entry, HashMap},
-    convert::TryInto,
-    sync::{
-        atomic::{AtomicU8, AtomicUsize, Ordering},
-        RwLock, RwLockReadGuard, RwLockWriteGuard,
-    },
-};
-
-use once_cell::sync::OnceCell;
-use tracing::error;
-use zenith_utils::{
-    lsn::Lsn,
-    zid::{ZTenantId, ZTimelineId},
-};
-
-use crate::layered_repository::writeback_ephemeral_file;
-use crate::{config::PageServerConf, relish::RelTag};
-
-static PAGE_CACHE: OnceCell<PageCache> = OnceCell::new();
-const TEST_PAGE_CACHE_SIZE: usize = 10;
-
-///
-/// Initialize the page cache. This must be called once at page server startup.
-///
-pub fn init(conf: &'static PageServerConf) {
-    if PAGE_CACHE
-        .set(PageCache::new(conf.page_cache_size))
-        .is_err()
-    {
-        panic!("page cache already initialized");
-    }
-}
-
-///
-/// Get a handle to the page cache.
-///
-pub fn get() -> &'static PageCache {
-    //
-    // In unit tests, page server startup doesn't happen and no one calls
-    // page_cache::init(). Initialize it here with a tiny cache, so that the
-    // page cache is usable in unit tests.
-    //
-    if cfg!(test) {
-        PAGE_CACHE.get_or_init(|| PageCache::new(TEST_PAGE_CACHE_SIZE))
-    } else {
-        PAGE_CACHE.get().expect("page cache not initialized")
-    }
-}
-
-pub const PAGE_SZ: usize = postgres_ffi::pg_constants::BLCKSZ as usize;
-const MAX_USAGE_COUNT: u8 = 5;
-
-///
-/// CacheKey uniquely identifies a "thing" to cache in the page cache.
-///
-#[derive(Debug, PartialEq, Eq, Clone)]
-enum CacheKey {
-    MaterializedPage {
-        hash_key: MaterializedPageHashKey,
-        lsn: Lsn,
-    },
-    EphemeralPage {
-        file_id: u64,
-        blkno: u32,
-    },
-}
-
-#[derive(Debug, PartialEq, Eq, Hash, Clone)]
-struct MaterializedPageHashKey {
-    tenant_id: ZTenantId,
-    timeline_id: ZTimelineId,
-    rel_tag: RelTag,
-    blknum: u32,
-}
-
-#[derive(Clone)]
-struct Version {
-    lsn: Lsn,
-    slot_idx: usize,
-}
-
-struct Slot {
-    inner: RwLock<SlotInner>,
-    usage_count: AtomicU8,
-}
-
-struct SlotInner {
-    key: Option<CacheKey>,
-    buf: &'static mut [u8; PAGE_SZ],
-    dirty: bool,
-}
-
-impl Slot {
-    /// Increment usage count on the buffer, with ceiling at MAX_USAGE_COUNT.
-    fn inc_usage_count(&self) {
-        let _ = self
-            .usage_count
-            .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |val| {
-                if val == MAX_USAGE_COUNT {
-                    None
-                } else {
-                    Some(val + 1)
-                }
-            });
-    }
-
-    /// Decrement usage count on the buffer, unless it's already zero.  Returns
-    /// the old usage count.
-    fn dec_usage_count(&self) -> u8 {
-        let count_res =
-            self.usage_count
-                .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |val| {
-                    if val == 0 {
-                        None
-                    } else {
-                        Some(val - 1)
-                    }
-                });
-
-        match count_res {
-            Ok(usage_count) => usage_count,
-            Err(usage_count) => usage_count,
-        }
-    }
-}
-
-pub struct PageCache {
-    /// This contains the mapping from the cache key to buffer slot that currently
-    /// contains the page, if any.
-    ///
-    /// TODO: This is protected by a single lock. If that becomes a bottleneck,
-    /// this HashMap can be replaced with a more concurrent version, there are
-    /// plenty of such crates around.
-    ///
-    /// If you add support for caching different kinds of objects, each object kind
-    /// can have a separate mapping map, next to this field.
-    materialized_page_map: RwLock<HashMap<MaterializedPageHashKey, Vec<Version>>>,
-
-    ephemeral_page_map: RwLock<HashMap<(u64, u32), usize>>,
-
-    /// The actual buffers with their metadata.
-    slots: Box<[Slot]>,
-
-    /// Index of the next candidate to evict, for the Clock replacement algorithm.
-    /// This is interpreted modulo the page cache size.
-    next_evict_slot: AtomicUsize,
-}
-
-///
-/// PageReadGuard is a "lease" on a buffer, for reading. The page is kept locked
-/// until the guard is dropped.
-///
-pub struct PageReadGuard<'i>(RwLockReadGuard<'i, SlotInner>);
-
-impl std::ops::Deref for PageReadGuard<'_> {
-    type Target = [u8; PAGE_SZ];
-
-    fn deref(&self) -> &Self::Target {
-        self.0.buf
-    }
-}
-
-///
-/// PageWriteGuard is a lease on a buffer for modifying it. The page is kept locked
-/// until the guard is dropped.
-///
-/// Counterintuitively, this is used even for a read, if the requested page is not
-/// currently found in the page cache. In that case, the caller of lock_for_read()
-/// is expected to fill in the page contents and call mark_valid(). Similarly
-/// lock_for_write() can return an invalid buffer that the caller is expected to
-/// to initialize.
-///
-pub struct PageWriteGuard<'i> {
-    inner: RwLockWriteGuard<'i, SlotInner>,
-
-    // Are the page contents currently valid?
-    valid: bool,
-}
-
-impl std::ops::DerefMut for PageWriteGuard<'_> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        self.inner.buf
-    }
-}
-
-impl std::ops::Deref for PageWriteGuard<'_> {
-    type Target = [u8; PAGE_SZ];
-
-    fn deref(&self) -> &Self::Target {
-        self.inner.buf
-    }
-}
-
-impl PageWriteGuard<'_> {
-    /// Mark that the buffer contents are now valid.
-    pub fn mark_valid(&mut self) {
-        assert!(self.inner.key.is_some());
-        assert!(
-            !self.valid,
-            "mark_valid called on a buffer that was already valid"
-        );
-        self.valid = true;
-    }
-    pub fn mark_dirty(&mut self) {
-        // only ephemeral pages can be dirty ATM.
-        assert!(matches!(
-            self.inner.key,
-            Some(CacheKey::EphemeralPage { .. })
-        ));
-        self.inner.dirty = true;
-    }
-}
-
-impl Drop for PageWriteGuard<'_> {
-    ///
-    /// If the buffer was allocated for a page that was not already in the
-    /// cache, but the lock_for_read/write() caller dropped the buffer without
-    /// initializing it, remove the mapping from the page cache.
-    ///
-    fn drop(&mut self) {
-        assert!(self.inner.key.is_some());
-        if !self.valid {
-            let self_key = self.inner.key.as_ref().unwrap();
-            PAGE_CACHE.get().unwrap().remove_mapping(self_key);
-            self.inner.key = None;
-            self.inner.dirty = false;
-        }
-    }
-}
-
-/// lock_for_read() return value
-pub enum ReadBufResult<'a> {
-    Found(PageReadGuard<'a>),
-    NotFound(PageWriteGuard<'a>),
-}
-
-/// lock_for_write() return value
-pub enum WriteBufResult<'a> {
-    Found(PageWriteGuard<'a>),
-    NotFound(PageWriteGuard<'a>),
-}
-
-impl PageCache {
-    //
-    // Section 1.1: Public interface functions for looking up and memorizing materialized page
-    // versions in the page cache
-    //
-
-    /// Look up a materialized page version.
-    ///
-    /// The 'lsn' is an upper bound, this will return the latest version of
-    /// the given block, but not newer than 'lsn'. Returns the actual LSN of the
-    /// returned page.
-    pub fn lookup_materialized_page(
-        &self,
-        tenant_id: ZTenantId,
-        timeline_id: ZTimelineId,
-        rel_tag: RelTag,
-        blknum: u32,
-        lsn: Lsn,
-    ) -> Option<(Lsn, PageReadGuard)> {
-        let mut cache_key = CacheKey::MaterializedPage {
-            hash_key: MaterializedPageHashKey {
-                tenant_id,
-                timeline_id,
-                rel_tag,
-                blknum,
-            },
-            lsn,
-        };
-
-        if let Some(guard) = self.try_lock_for_read(&mut cache_key) {
-            if let CacheKey::MaterializedPage { hash_key: _, lsn } = cache_key {
-                Some((lsn, guard))
-            } else {
-                panic!("unexpected key type in slot");
-            }
-        } else {
-            None
-        }
-    }
-
-    ///
-    /// Store an image of the given page in the cache.
-    ///
-    pub fn memorize_materialized_page(
-        &self,
-        tenant_id: ZTenantId,
-        timeline_id: ZTimelineId,
-        rel_tag: RelTag,
-        blknum: u32,
-        lsn: Lsn,
-        img: &[u8],
-    ) {
-        let cache_key = CacheKey::MaterializedPage {
-            hash_key: MaterializedPageHashKey {
-                tenant_id,
-                timeline_id,
-                rel_tag,
-                blknum,
-            },
-            lsn,
-        };
-
-        match self.lock_for_write(&cache_key) {
-            WriteBufResult::Found(write_guard) => {
-                // We already had it in cache. Another thread must've put it there
-                // concurrently. Check that it had the same contents that we
-                // replayed.
-                assert!(*write_guard == img);
-            }
-            WriteBufResult::NotFound(mut write_guard) => {
-                write_guard.copy_from_slice(img);
-                write_guard.mark_valid();
-            }
-        }
-    }
-
-    // Section 1.2: Public interface functions for working with Ephemeral pages.
-
-    pub fn read_ephemeral_buf(&self, file_id: u64, blkno: u32) -> ReadBufResult {
-        let mut cache_key = CacheKey::EphemeralPage { file_id, blkno };
-
-        self.lock_for_read(&mut cache_key)
-    }
-
-    pub fn write_ephemeral_buf(&self, file_id: u64, blkno: u32) -> WriteBufResult {
-        let cache_key = CacheKey::EphemeralPage { file_id, blkno };
-
-        self.lock_for_write(&cache_key)
-    }
-
-    /// Immediately drop all buffers belonging to given file, without writeback
-    pub fn drop_buffers_for_ephemeral(&self, drop_file_id: u64) {
-        for slot_idx in 0..self.slots.len() {
-            let slot = &self.slots[slot_idx];
-
-            let mut inner = slot.inner.write().unwrap();
-            if let Some(key) = &inner.key {
-                match key {
-                    CacheKey::EphemeralPage { file_id, blkno: _ } if *file_id == drop_file_id => {
-                        // remove mapping for old buffer
-                        self.remove_mapping(key);
-                        inner.key = None;
-                        inner.dirty = false;
-                    }
-                    _ => {}
-                }
-            }
-        }
-    }
-
-    //
-    // Section 2: Internal interface functions for lookup/update.
-    //
-    // To add support for a new kind of "thing" to cache, you will need
-    // to add public interface routines above, and code to deal with the
-    // "mappings" after this section. But the routines in this section should
-    // not require changes.
-
-    /// Look up a page in the cache.
-    ///
-    /// If the search criteria is not exact, *cache_key is updated with the key
-    /// for exact key of the returned page. (For materialized pages, that means
-    /// that the LSN in 'cache_key' is updated with the LSN of the returned page
-    /// version.)
-    ///
-    /// If no page is found, returns None and *cache_key is left unmodified.
-    ///
-    fn try_lock_for_read(&self, cache_key: &mut CacheKey) -> Option<PageReadGuard> {
-        let cache_key_orig = cache_key.clone();
-        if let Some(slot_idx) = self.search_mapping(cache_key) {
-            // The page was found in the mapping. Lock the slot, and re-check
-            // that it's still what we expected (because we released the mapping
-            // lock already, another thread could have evicted the page)
-            let slot = &self.slots[slot_idx];
-            let inner = slot.inner.read().unwrap();
-            if inner.key.as_ref() == Some(cache_key) {
-                slot.inc_usage_count();
-                return Some(PageReadGuard(inner));
-            } else {
-                // search_mapping might have modified the search key; restore it.
-                *cache_key = cache_key_orig;
-            }
-        }
-        None
-    }
-
-    /// Return a locked buffer for given block.
-    ///
-    /// Like try_lock_for_read(), if the search criteria is not exact and the
-    /// page is already found in the cache, *cache_key is updated.
-    ///
-    /// If the page is not found in the cache, this allocates a new buffer for
-    /// it. The caller may then initialize the buffer with the contents, and
-    /// call mark_valid().
-    ///
-    /// Example usage:
-    ///
-    /// ```ignore
-    /// let cache = page_cache::get();
-    ///
-    /// match cache.lock_for_read(&key) {
-    ///     ReadBufResult::Found(read_guard) => {
-    ///         // The page was found in cache. Use it
-    ///     },
-    ///     ReadBufResult::NotFound(write_guard) => {
-    ///         // The page was not found in cache. Read it from disk into the
-    ///         // buffer.
-    ///         //read_my_page_from_disk(write_guard);
-    ///
-    ///         // The buffer contents are now valid. Tell the page cache.
-    ///         write_guard.mark_valid();
-    ///     },
-    /// }
-    /// ```
-    ///
-    fn lock_for_read(&self, cache_key: &mut CacheKey) -> ReadBufResult {
-        loop {
-            // First check if the key already exists in the cache.
-            if let Some(read_guard) = self.try_lock_for_read(cache_key) {
-                return ReadBufResult::Found(read_guard);
-            }
-
-            // Not found. Find a victim buffer
-            let (slot_idx, mut inner) = self.find_victim();
-
-            // Insert mapping for this. At this point, we may find that another
-            // thread did the same thing concurrently. In that case, we evicted
-            // our victim buffer unnecessarily. Put it into the free list and
-            // continue with the slot that the other thread chose.
-            if let Some(_existing_slot_idx) = self.try_insert_mapping(cache_key, slot_idx) {
-                // TODO: put to free list
-
-                // We now just loop back to start from beginning. This is not
-                // optimal, we'll perform the lookup in the mapping again, which
-                // is not really necessary because we already got
-                // 'existing_slot_idx'.  But this shouldn't happen often enough
-                // to matter much.
-                continue;
-            }
-
-            // Make the slot ready
-            let slot = &self.slots[slot_idx];
-            inner.key = Some(cache_key.clone());
-            inner.dirty = false;
-            slot.usage_count.store(1, Ordering::Relaxed);
-
-            return ReadBufResult::NotFound(PageWriteGuard {
-                inner,
-                valid: false,
-            });
-        }
-    }
-
-    /// Look up a page in the cache and lock it in write mode. If it's not
-    /// found, returns None.
-    ///
-    /// When locking a page for writing, the search criteria is always "exact".
-    fn try_lock_for_write(&self, cache_key: &CacheKey) -> Option<PageWriteGuard> {
-        if let Some(slot_idx) = self.search_mapping_for_write(cache_key) {
-            // The page was found in the mapping. Lock the slot, and re-check
-            // that it's still what we expected (because we don't released the mapping
-            // lock already, another thread could have evicted the page)
-            let slot = &self.slots[slot_idx];
-            let inner = slot.inner.write().unwrap();
-            if inner.key.as_ref() == Some(cache_key) {
-                slot.inc_usage_count();
-                return Some(PageWriteGuard { inner, valid: true });
-            }
-        }
-        None
-    }
-
-    /// Return a write-locked buffer for given block.
-    ///
-    /// Similar to lock_for_read(), but the returned buffer is write-locked and
-    /// may be modified by the caller even if it's already found in the cache.
-    fn lock_for_write(&self, cache_key: &CacheKey) -> WriteBufResult {
-        loop {
-            // First check if the key already exists in the cache.
-            if let Some(write_guard) = self.try_lock_for_write(cache_key) {
-                return WriteBufResult::Found(write_guard);
-            }
-
-            // Not found. Find a victim buffer
-            let (slot_idx, mut inner) = self.find_victim();
-
-            // Insert mapping for this. At this point, we may find that another
-            // thread did the same thing concurrently. In that case, we evicted
-            // our victim buffer unnecessarily. Put it into the free list and
-            // continue with the slot that the other thread chose.
-            if let Some(_existing_slot_idx) = self.try_insert_mapping(cache_key, slot_idx) {
-                // TODO: put to free list
-
-                // We now just loop back to start from beginning. This is not
-                // optimal, we'll perform the lookup in the mapping again, which
-                // is not really necessary because we already got
-                // 'existing_slot_idx'.  But this shouldn't happen often enough
-                // to matter much.
-                continue;
-            }
-
-            // Make the slot ready
-            let slot = &self.slots[slot_idx];
-            inner.key = Some(cache_key.clone());
-            inner.dirty = false;
-            slot.usage_count.store(1, Ordering::Relaxed);
-
-            return WriteBufResult::NotFound(PageWriteGuard {
-                inner,
-                valid: false,
-            });
-        }
-    }
-
-    //
-    // Section 3: Mapping functions
-    //
-
-    /// Search for a page in the cache using the given search key.
-    ///
-    /// Returns the slot index, if any. If the search criteria is not exact,
-    /// *cache_key is updated with the actual key of the found page.
-    ///
-    /// NOTE: We don't hold any lock on the mapping on return, so the slot might
-    /// get recycled for an unrelated page immediately after this function
-    /// returns.  The caller is responsible for re-checking that the slot still
-    /// contains the page with the same key before using it.
-    ///
-    fn search_mapping(&self, cache_key: &mut CacheKey) -> Option<usize> {
-        match cache_key {
-            CacheKey::MaterializedPage { hash_key, lsn } => {
-                let map = self.materialized_page_map.read().unwrap();
-                let versions = map.get(hash_key)?;
-
-                let version_idx = match versions.binary_search_by_key(lsn, |v| v.lsn) {
-                    Ok(version_idx) => version_idx,
-                    Err(0) => return None,
-                    Err(version_idx) => version_idx - 1,
-                };
-                let version = &versions[version_idx];
-                *lsn = version.lsn;
-                Some(version.slot_idx)
-            }
-            CacheKey::EphemeralPage { file_id, blkno } => {
-                let map = self.ephemeral_page_map.read().unwrap();
-                Some(*map.get(&(*file_id, *blkno))?)
-            }
-        }
-    }
-
-    /// Search for a page in the cache using the given search key.
-    ///
-    /// Like 'search_mapping, but performs an "exact" search. Used for
-    /// allocating a new buffer.
-    fn search_mapping_for_write(&self, key: &CacheKey) -> Option<usize> {
-        match key {
-            CacheKey::MaterializedPage { hash_key, lsn } => {
-                let map = self.materialized_page_map.read().unwrap();
-                let versions = map.get(hash_key)?;
-
-                if let Ok(version_idx) = versions.binary_search_by_key(lsn, |v| v.lsn) {
-                    Some(versions[version_idx].slot_idx)
-                } else {
-                    None
-                }
-            }
-            CacheKey::EphemeralPage { file_id, blkno } => {
-                let map = self.ephemeral_page_map.read().unwrap();
-                Some(*map.get(&(*file_id, *blkno))?)
-            }
-        }
-    }
-
-    ///
-    /// Remove mapping for given key.
-    ///
-    fn remove_mapping(&self, old_key: &CacheKey) {
-        match old_key {
-            CacheKey::MaterializedPage {
-                hash_key: old_hash_key,
-                lsn: old_lsn,
-            } => {
-                let mut map = self.materialized_page_map.write().unwrap();
-                if let Entry::Occupied(mut old_entry) = map.entry(old_hash_key.clone()) {
-                    let versions = old_entry.get_mut();
-
-                    if let Ok(version_idx) = versions.binary_search_by_key(old_lsn, |v| v.lsn) {
-                        versions.remove(version_idx);
-                        if versions.is_empty() {
-                            old_entry.remove_entry();
-                        }
-                    }
-                } else {
-                    panic!("could not find old key in mapping")
-                }
-            }
-            CacheKey::EphemeralPage { file_id, blkno } => {
-                let mut map = self.ephemeral_page_map.write().unwrap();
-                map.remove(&(*file_id, *blkno))
-                    .expect("could not find old key in mapping");
-            }
-        }
-    }
-
-    ///
-    /// Insert mapping for given key.
-    ///
-    /// If a mapping already existed for the given key, returns the slot index
-    /// of the existing mapping and leaves it untouched.
-    fn try_insert_mapping(&self, new_key: &CacheKey, slot_idx: usize) -> Option<usize> {
-        match new_key {
-            CacheKey::MaterializedPage {
-                hash_key: new_key,
-                lsn: new_lsn,
-            } => {
-                let mut map = self.materialized_page_map.write().unwrap();
-                let versions = map.entry(new_key.clone()).or_default();
-                match versions.binary_search_by_key(new_lsn, |v| v.lsn) {
-                    Ok(version_idx) => Some(versions[version_idx].slot_idx),
-                    Err(version_idx) => {
-                        versions.insert(
-                            version_idx,
-                            Version {
-                                lsn: *new_lsn,
-                                slot_idx,
-                            },
-                        );
-                        None
-                    }
-                }
-            }
-            CacheKey::EphemeralPage { file_id, blkno } => {
-                let mut map = self.ephemeral_page_map.write().unwrap();
-                match map.entry((*file_id, *blkno)) {
-                    Entry::Occupied(entry) => Some(*entry.get()),
-                    Entry::Vacant(entry) => {
-                        entry.insert(slot_idx);
-                        None
-                    }
-                }
-            }
-        }
-    }
-
-    //
-    // Section 4: Misc internal helpers
-    //
-
-    /// Find a slot to evict.
-    ///
-    /// On return, the slot is empty and write-locked.
-    fn find_victim(&self) -> (usize, RwLockWriteGuard<SlotInner>) {
-        let iter_limit = self.slots.len() * 2;
-        let mut iters = 0;
-        loop {
-            let slot_idx = self.next_evict_slot.fetch_add(1, Ordering::Relaxed) % self.slots.len();
-
-            let slot = &self.slots[slot_idx];
-
-            if slot.dec_usage_count() == 0 || iters >= iter_limit {
-                let mut inner = slot.inner.write().unwrap();
-
-                if let Some(old_key) = &inner.key {
-                    if inner.dirty {
-                        if let Err(err) = Self::writeback(old_key, inner.buf) {
-                            // Writing the page to disk failed.
-                            //
-                            // FIXME: What to do here, when? We could propagate the error to the
-                            // caller, but victim buffer is generally unrelated to the original
-                            // call. It can even belong to a different tenant. Currently, we
-                            // report the error to the log and continue the clock sweep to find
-                            // a different victim. But if the problem persists, the page cache
-                            // could fill up with dirty pages that we cannot evict, and we will
-                            // loop retrying the writebacks indefinitely.
-                            error!("writeback of buffer {:?} failed: {}", old_key, err);
-                            continue;
-                        }
-                    }
-
-                    // remove mapping for old buffer
-                    self.remove_mapping(old_key);
-                    inner.dirty = false;
-                    inner.key = None;
-                }
-                return (slot_idx, inner);
-            }
-
-            iters += 1;
-        }
-    }
-
-    fn writeback(cache_key: &CacheKey, buf: &[u8]) -> Result<(), std::io::Error> {
-        match cache_key {
-            CacheKey::MaterializedPage {
-                hash_key: _,
-                lsn: _,
-            } => {
-                panic!("unexpected dirty materialized page");
-            }
-            CacheKey::EphemeralPage { file_id, blkno } => {
-                writeback_ephemeral_file(*file_id, *blkno, buf)
-            }
-        }
-    }
-
-    /// Initialize a new page cache
-    ///
-    /// This should be called only once at page server startup.
-    fn new(num_pages: usize) -> Self {
-        assert!(num_pages > 0, "page cache size must be > 0");
-
-        let page_buffer = Box::leak(vec![0u8; num_pages * PAGE_SZ].into_boxed_slice());
-
-        let slots = page_buffer
-            .chunks_exact_mut(PAGE_SZ)
-            .map(|chunk| {
-                let buf: &mut [u8; PAGE_SZ] = chunk.try_into().unwrap();
-
-                Slot {
-                    inner: RwLock::new(SlotInner {
-                        key: None,
-                        buf,
-                        dirty: false,
-                    }),
-                    usage_count: AtomicU8::new(0),
-                }
-            })
-            .collect();
-
-        Self {
-            materialized_page_map: Default::default(),
-            ephemeral_page_map: Default::default(),
-            slots,
-            next_evict_slot: AtomicUsize::new(0),
-        }
-    }
-}

pageserver/src/page_service.rs

@@ -10,7 +10,7 @@
 //     *callmemaybe <zenith timelineid> $url* -- ask pageserver to start walreceiver on $url
 //
 
-use anyhow::{anyhow, bail, ensure, Context, Result};
+use anyhow::{anyhow, bail, ensure, Result};
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use lazy_static::lazy_static;
 use regex::Regex;
@@ -35,12 +35,11 @@ use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
 use crate::basebackup;
 use crate::branches;
-use crate::config::PageServerConf;
 use crate::relish::*;
 use crate::repository::Timeline;
 use crate::tenant_mgr;
 use crate::walreceiver;
-use crate::CheckpointConfig;
+use crate::PageServerConf;
 
 // Wrapped in libpq CopyData
 enum PagestreamFeMessage {
@@ -280,8 +279,7 @@ impl PageServerHandler {
         let _enter = info_span!("pagestream", timeline = %timelineid, tenant = %tenantid).entered();
 
         // Check that the timeline exists
-        let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)
-            .context("Cannot handle pagerequests for a remote timeline")?;
+        let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)?;
 
         /* switch client to COPYBOTH */
         pgb.write_message(&BeMessage::CopyBothResponse)?;
@@ -303,17 +301,17 @@ impl PageServerHandler {
                             PagestreamFeMessage::Exists(req) => SMGR_QUERY_TIME
                                 .with_label_values(&["get_rel_exists"])
                                 .observe_closure_duration(|| {
-                                    self.handle_get_rel_exists_request(timeline.as_ref(), &req)
+                                    self.handle_get_rel_exists_request(&*timeline, &req)
                                 }),
                             PagestreamFeMessage::Nblocks(req) => SMGR_QUERY_TIME
                                 .with_label_values(&["get_rel_size"])
                                 .observe_closure_duration(|| {
-                                    self.handle_get_nblocks_request(timeline.as_ref(), &req)
+                                    self.handle_get_nblocks_request(&*timeline, &req)
                                 }),
                             PagestreamFeMessage::GetPage(req) => SMGR_QUERY_TIME
                                 .with_label_values(&["get_page_at_lsn"])
                                 .observe_closure_duration(|| {
-                                    self.handle_get_page_at_lsn_request(timeline.as_ref(), &req)
+                                    self.handle_get_page_at_lsn_request(&*timeline, &req)
                                 }),
                         };
 
@@ -457,13 +455,7 @@ impl PageServerHandler {
         let _enter = span.enter();
 
         // check that the timeline exists
-        let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)
-            .context("Cannot handle basebackup request for a remote timeline")?;
-        if let Some(lsn) = lsn {
-            timeline
-                .check_lsn_is_in_scope(lsn)
-                .context("invalid basebackup lsn")?;
-        }
+        let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)?;
 
         // switch client to COPYOUT
         pgb.write_message(&BeMessage::CopyOutResponse)?;
@@ -533,10 +525,17 @@ impl postgres_backend::Handler for PageServerHandler {
     fn process_query(
         &mut self,
         pgb: &mut PostgresBackend,
-        query_string: &str,
+        query_string: Bytes,
     ) -> anyhow::Result<()> {
         debug!("process query {:?}", query_string);
 
+        // remove null terminator, if any
+        let mut query_string = query_string;
+        if query_string.last() == Some(&0) {
+            query_string.truncate(query_string.len() - 1);
+        }
+        let query_string = std::str::from_utf8(&query_string)?;
+
         if query_string.starts_with("pagestream ") {
             let (_, params_raw) = query_string.split_at("pagestream ".len());
             let params = params_raw.split(' ').collect::<Vec<_>>();
@@ -591,8 +590,7 @@ impl postgres_backend::Handler for PageServerHandler {
                 info_span!("callmemaybe", timeline = %timelineid, tenant = %tenantid).entered();
 
             // Check that the timeline exists
-            tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)
-                .context("Failed to fetch local timeline for callmemaybe requests")?;
+            tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)?;
 
             walreceiver::launch_wal_receiver(self.conf, timelineid, &connstr, tenantid.to_owned());
 
@@ -693,7 +691,9 @@ impl postgres_backend::Handler for PageServerHandler {
                 .unwrap_or(Ok(self.conf.gc_horizon))?;
 
             let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
+
             let result = repo.gc_iteration(Some(timelineid), gc_horizon, true)?;
+
             pgb.write_message_noflush(&BeMessage::RowDescription(&[
                 RowDescriptor::int8_col(b"layer_relfiles_total"),
                 RowDescriptor::int8_col(b"layer_relfiles_needed_by_cutoff"),
@@ -759,25 +759,6 @@ impl postgres_backend::Handler for PageServerHandler {
                 Some(result.elapsed.as_millis().to_string().as_bytes()),
             ]))?
             .write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;
-        } else if query_string.starts_with("checkpoint ") {
-            // Run checkpoint immediately on given timeline.
-
-            // checkpoint <tenant_id> <timeline_id>
-            let re = Regex::new(r"^checkpoint ([[:xdigit:]]+)\s([[:xdigit:]]+)($|\s)?").unwrap();
-
-            let caps = re
-                .captures(query_string)
-                .ok_or_else(|| anyhow!("invalid checkpoint command: '{}'", query_string))?;
-
-            let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;
-            let timelineid = ZTimelineId::from_str(caps.get(2).unwrap().as_str())?;
-
-            let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)
-                .context("Failed to fetch local timeline for checkpoint request")?;
-
-            timeline.checkpoint(CheckpointConfig::Forced)?;
-            pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
-                .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
         } else {
             bail!("unknown command");
         }

pageserver/src/relish.rs

@@ -224,3 +224,8 @@ impl SlruKind {
         }
     }
 }
+
+pub const FIRST_NONREL_RELISH_TAG: RelishTag = RelishTag::Slru {
+    slru: SlruKind::Clog,
+    segno: 0,
+};

pageserver/src/relish_storage.rs

@@ -0,0 +1,323 @@
+//! A set of generic storage abstractions for the page server to use when backing up and restoring its state from the external storage.
+//! This particular module serves as a public API border between pageserver and the internal storage machinery.
+//! No other modules from this tree are supposed to be used directly by the external code.
+//!
+//! There are a few components the storage machinery consists of:
+//! * [`RelishStorage`] trait a CRUD-like generic abstraction to use for adapting external storages with a few implementations:
+//!     * [`local_fs`] allows to use local file system as an external storage
+//!     * [`rust_s3`] uses AWS S3 bucket entirely as an external storage
+//!
+//! * synchronization logic at [`storage_sync`] module that keeps pageserver state (both runtime one and the workdir files) and storage state in sync.
+//!
+//! * public API via to interact with the external world: [`run_storage_sync_thread`] and [`schedule_timeline_upload`]
+//!
+//! Here's a schematic overview of all interactions relish storage and the rest of the pageserver perform:
+//!
+//! +------------------------+                                    +--------->-------+
+//! |                        |  - - - (init async loop) - - - ->  |                 |
+//! |                        |                                    |                 |
+//! |                        |  ------------------------------->  |      async      |
+//! |       pageserver       |   (schedule frozen layer upload)   | upload/download |
+//! |                        |                                    |      loop       |
+//! |                        |  <-------------------------------  |                 |
+//! |                        |    (register downloaded layers)    |                 |
+//! +------------------------+                                    +---------<-------+
+//!                                                                         |
+//!                                                                         |
+//!                                          CRUD layer file operations     |
+//!                                     (upload/download/delete/list, etc.) |
+//!                                                                         V
+//!                                                            +------------------------+
+//!                                                            |                        |
+//!                                                            | [`RelishStorage`] impl |
+//!                                                            |                        |
+//!                                                            | pageserver assumes it  |
+//!                                                            | owns exclusive write   |
+//!                                                            | access to this storage |
+//!                                                            +------------------------+
+//!
+//! First, during startup, the pageserver inits the storage sync thread with the async loop, or leaves the loop unitialised, if configured so.
+//! Some time later, during pageserver checkpoints, in-memory data is flushed onto disk along with its metadata.
+//! If the storage sync loop was successfully started before, pageserver schedules the new image uploads after every checkpoint.
+//! See [`crate::layered_repository`] for the upload calls and the adjacent logic.
+//!
+//! The storage logic considers `image` as a set of local files, fully representing a certain timeline at given moment (identified with `disk_consistent_lsn`).
+//! Timeline can change its state, by adding more files on disk and advancing its `disk_consistent_lsn`: this happens after pageserver checkpointing and is followed
+//! by the storage upload, if enabled.
+//! When a certain image gets uploaded, the sync loop remembers the fact, preventing further reuploads of the same image state.
+//! No files are deleted from either local or remote storage, only the missing ones locally/remotely get downloaded/uploaded, local metadata file will be overwritten
+//! when the newer timeline is downloaded.
+//!
+//! Meanwhile, the loop inits the storage connection and checks the remote files stored.
+//! This is done once at startup only, relying on the fact that pageserver uses the storage alone (ergo, nobody else uploads the files to the storage but this server).
+//! Based on the remote image data, the storage sync logic queues image downloads, while accepting any potential upload tasks from pageserver and managing the tasks by their priority.
+//! On the image download, a [`crate::tenant_mgr::register_relish_download`] function is called to register the new image in pageserver, initializing all related threads and internal state.
+//!
+//! When the pageserver terminates, the upload loop finishes a current image sync task (if any) and exits.
+//!
+//! NOTES:
+//! * pageserver assumes it has exclusive write access to the relish storage. If supported, the way multiple pageservers can be separated in the same storage
+//! (i.e. using different directories in the local filesystem external storage), but totally up to the storage implementation and not covered with the trait API.
+//!
+//! * the uploads do not happen right after pageserver startup, they are registered when
+//!     1. pageserver does the checkpoint, which happens further in the future after the server start
+//!     2. pageserver loads the timeline from disk for the first time
+//!
+//! * the uploads do not happen right after the upload registration: the sync loop might be occupied with other tasks, or tasks with bigger priority could be waiting already
+//!
+//! * all synchronization tasks (including the public API to register uploads and downloads and the sync queue management) happens on an image scale: a big set of relish files,
+//! enough to represent (and recover, if needed) a certain timeline state. On the contrary, all internal storage CRUD calls are made per reilsh file from those images.
+//! This way, the synchronization is able to download the image partially, if some state was synced before, but exposes correctly synced images only.
+
+mod local_fs;
+mod rust_s3;
+mod storage_sync;
+
+use std::{
+    path::{Path, PathBuf},
+    thread,
+};
+
+use anyhow::{anyhow, ensure, Context};
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
+
+pub use self::storage_sync::schedule_timeline_upload;
+use self::{local_fs::LocalFs, rust_s3::S3};
+use crate::{
+    layered_repository::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME},
+    PageServerConf, RelishStorageKind,
+};
+
+/// Based on the config, initiates the remote storage connection and starts a separate thread
+/// that ensures that pageserver and the remote storage are in sync with each other.
+/// If no external configuraion connection given, no thread or storage initialization is done.
+pub fn run_storage_sync_thread(
+    config: &'static PageServerConf,
+) -> anyhow::Result<Option<thread::JoinHandle<anyhow::Result<()>>>> {
+    match &config.relish_storage_config {
+        Some(relish_storage_config) => {
+            let max_concurrent_sync = relish_storage_config.max_concurrent_sync;
+            let handle = match &relish_storage_config.storage {
+                RelishStorageKind::LocalFs(root) => storage_sync::spawn_storage_sync_thread(
+                    config,
+                    LocalFs::new(root.clone(), &config.workdir)?,
+                    max_concurrent_sync,
+                ),
+                RelishStorageKind::AwsS3(s3_config) => storage_sync::spawn_storage_sync_thread(
+                    config,
+                    S3::new(s3_config, &config.workdir)?,
+                    max_concurrent_sync,
+                ),
+            };
+            handle.map(Some)
+        }
+        None => Ok(None),
+    }
+}
+
+/// Storage (potentially remote) API to manage its state.
+/// This storage tries to be unaware of any layered repository context,
+/// providing basic CRUD operations with storage files.
+#[async_trait::async_trait]
+trait RelishStorage: Send + Sync {
+    /// A way to uniquely reference relish in the remote storage.
+    type RelishStoragePath;
+
+    /// Attempts to derive the storage path out of the local path, if the latter is correct.
+    fn storage_path(&self, local_path: &Path) -> anyhow::Result<Self::RelishStoragePath>;
+
+    /// Gets the layered storage information about the given entry.
+    fn info(&self, storage_path: &Self::RelishStoragePath) -> anyhow::Result<RemoteRelishInfo>;
+
+    /// Lists all items the storage has right now.
+    async fn list_relishes(&self) -> anyhow::Result<Vec<Self::RelishStoragePath>>;
+
+    /// Streams the remote storage entry contents into the buffered writer given, returns the filled writer.
+    async fn download_relish<W: 'static + std::io::Write + Send>(
+        &self,
+        from: &Self::RelishStoragePath,
+        // rust_s3 `get_object_stream` method requires `std::io::BufWriter` for some reason, not the async counterpart
+        // that forces us to consume and return the writer to satisfy the blocking operation async wrapper requirements
+        to: std::io::BufWriter<W>,
+    ) -> anyhow::Result<std::io::BufWriter<W>>;
+
+    async fn delete_relish(&self, path: &Self::RelishStoragePath) -> anyhow::Result<()>;
+
+    /// Streams the local file contents into remote into the remote storage entry.
+    async fn upload_relish<R: tokio::io::AsyncRead + std::marker::Unpin + Send>(
+        &self,
+        from: &mut tokio::io::BufReader<R>,
+        to: &Self::RelishStoragePath,
+    ) -> anyhow::Result<()>;
+}
+
+/// Information about a certain remote storage entry.
+#[derive(Debug, PartialEq, Eq)]
+struct RemoteRelishInfo {
+    tenant_id: ZTenantId,
+    timeline_id: ZTimelineId,
+    /// Path in the pageserver workdir where the file should go to.
+    download_destination: PathBuf,
+    is_metadata: bool,
+}
+
+fn strip_path_prefix<'a>(prefix: &'a Path, path: &'a Path) -> anyhow::Result<&'a Path> {
+    if prefix == path {
+        anyhow::bail!(
+            "Prefix and the path are equal, cannot strip: '{}'",
+            prefix.display()
+        )
+    } else {
+        path.strip_prefix(prefix).with_context(|| {
+            format!(
+                "Path '{}' is not prefixed with '{}'",
+                path.display(),
+                prefix.display(),
+            )
+        })
+    }
+}
+
+fn parse_ids_from_path<'a, R: std::fmt::Display>(
+    path_segments: impl Iterator<Item = &'a str>,
+    path_log_representation: &R,
+) -> anyhow::Result<(ZTenantId, ZTimelineId)> {
+    let mut segments = path_segments.skip_while(|&segment| segment != TENANTS_SEGMENT_NAME);
+    let tenants_segment = segments.next().ok_or_else(|| {
+        anyhow!(
+            "Found no '{}' segment in the storage path '{}'",
+            TENANTS_SEGMENT_NAME,
+            path_log_representation
+        )
+    })?;
+    ensure!(
+        tenants_segment == TENANTS_SEGMENT_NAME,
+        "Failed to extract '{}' segment from storage path '{}'",
+        TENANTS_SEGMENT_NAME,
+        path_log_representation
+    );
+    let tenant_id = segments
+        .next()
+        .ok_or_else(|| {
+            anyhow!(
+                "Found no tenant id in the storage path '{}'",
+                path_log_representation
+            )
+        })?
+        .parse::<ZTenantId>()
+        .with_context(|| {
+            format!(
+                "Failed to parse tenant id from storage path '{}'",
+                path_log_representation
+            )
+        })?;
+
+    let timelines_segment = segments.next().ok_or_else(|| {
+        anyhow!(
+            "Found no '{}' segment in the storage path '{}'",
+            TIMELINES_SEGMENT_NAME,
+            path_log_representation
+        )
+    })?;
+    ensure!(
+        timelines_segment == TIMELINES_SEGMENT_NAME,
+        "Failed to extract '{}' segment from storage path '{}'",
+        TIMELINES_SEGMENT_NAME,
+        path_log_representation
+    );
+    let timeline_id = segments
+        .next()
+        .ok_or_else(|| {
+            anyhow!(
+                "Found no timeline id in the storage path '{}'",
+                path_log_representation
+            )
+        })?
+        .parse::<ZTimelineId>()
+        .with_context(|| {
+            format!(
+                "Failed to parse timeline id from storage path '{}'",
+                path_log_representation
+            )
+        })?;
+
+    Ok((tenant_id, timeline_id))
+}
+
+/// A set of common test utils to share in unit tests inside the module tree.
+#[cfg(test)]
+mod test_utils {
+    use std::path::{Path, PathBuf};
+
+    use anyhow::ensure;
+
+    use crate::{
+        layered_repository::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME},
+        repository::repo_harness::{RepoHarness, TIMELINE_ID},
+    };
+
+    /// Gives a timeline path with pageserver workdir stripped off.
+    pub fn relative_timeline_path(harness: &RepoHarness) -> anyhow::Result<PathBuf> {
+        let timeline_path = harness.timeline_path(&TIMELINE_ID);
+        Ok(timeline_path
+            .strip_prefix(&harness.conf.workdir)?
+            .to_path_buf())
+    }
+
+    /// Creates a path with custom tenant id in one of its segments.
+    /// Useful for emulating paths with wrong ids.
+    pub fn custom_tenant_id_path(
+        path_with_tenant_id: &Path,
+        new_tenant_id: &str,
+    ) -> anyhow::Result<PathBuf> {
+        let mut new_path = PathBuf::new();
+        let mut is_tenant_id = false;
+        let mut tenant_id_replaced = false;
+        for segment in path_with_tenant_id {
+            match segment.to_str() {
+                Some(TENANTS_SEGMENT_NAME) => is_tenant_id = true,
+                Some(_tenant_id_str) if is_tenant_id => {
+                    is_tenant_id = false;
+                    new_path.push(new_tenant_id);
+                    tenant_id_replaced = true;
+                    continue;
+                }
+                _ => {}
+            }
+            new_path.push(segment)
+        }
+
+        ensure!(tenant_id_replaced, "Found no tenant id segment to replace");
+        Ok(new_path)
+    }
+
+    /// Creates a path with custom timeline id in one of its segments.
+    /// Useful for emulating paths with wrong ids.
+    pub fn custom_timeline_id_path(
+        path_with_timeline_id: &Path,
+        new_timeline_id: &str,
+    ) -> anyhow::Result<PathBuf> {
+        let mut new_path = PathBuf::new();
+        let mut is_timeline_id = false;
+        let mut timeline_id_replaced = false;
+        for segment in path_with_timeline_id {
+            match segment.to_str() {
+                Some(TIMELINES_SEGMENT_NAME) => is_timeline_id = true,
+                Some(_timeline_id_str) if is_timeline_id => {
+                    is_timeline_id = false;
+                    new_path.push(new_timeline_id);
+                    timeline_id_replaced = true;
+                    continue;
+                }
+                _ => {}
+            }
+            new_path.push(segment)
+        }
+
+        ensure!(
+            timeline_id_replaced,
+            "Found no timeline id segment to replace"
+        );
+        Ok(new_path)
+    }
+}

pageserver/src/relish_storage/README.md

@@ -0,0 +1,82 @@
+# Non-implementation details
+
+This document describes the current state of the backup system in pageserver, existing limitations and concerns, why some things are done the way they are the future development plans.
+Detailed description on how the synchronization works and how it fits into the rest of the pageserver can be found in the [storage module](./../relish_storage.rs) and its submodules.
+Ideally, this document should disappear after current implementation concerns are mitigated, with the remaining useful knowledge bits moved into rustdocs.
+
+## Approach
+
+Backup functionality is a new component, appeared way after the core DB functionality was implemented.
+Pageserver layer functionality is also quite volatile at the moment, there's a risk its local file management changes over time.
+
+To avoid adding more chaos into that, backup functionality is currently designed as a relatively standalone component, with the majority of its logic placed in a standalone async loop.
+This way, the backups are managed in background, not affecting directly other pageserver parts: this way the backup and restoration process may lag behind, but eventually keep up with the reality. To track that, a set of prometheus metrics is exposed from pageserver.
+
+## What's done
+
+Current implementation
+* provides remote storage wrappers for AWS S3 and local FS
+* uploads layers, frozen by pageserver checkpoint thread
+* downloads and registers layers, found on the remote storage, but missing locally
+
+No good optimisations or performance testing is done, the feature is disabled by default and gets polished over time.
+It's planned to deal with all questions that are currently on and prepare the feature to be enabled by default in cloud environments.
+
+### Peculiarities
+
+As mentioned, the backup component is rather new and under development currently, so not all things are done properly from the start.
+Here's the list of known compromises with comments:
+
+* Remote storage model is the same as the `tenants/` directory contents of the pageserver's local workdir storage.
+This is relatively simple to implement, but may be costly to use in AWS S3: an initial data image contains ~782 relish file and a metadata file, ~31 MB combined.
+AWS charges per API call and for traffic either, layers are expected to be updated frequently, so this model most probably is ineffective.
+Additionally, pageservers might need to migrate images between tenants, which does not improve the situation.
+
+Storage sync API operates images when backing up or restoring a backup, so we're fluent to repack the layer contents the way we want to, which most probably will be done later.
+
+* no proper file comparison
+
+Currently, every layer contains `Lsn` in their name, to map the data it holds against a certain DB state.
+Then the images with same ids and different `Lsn`'s are compared, files are considered equal if their local file paths are equal (for remote files, "local file path" is their download destination).
+No file contents assertion is done currently, but should be.
+AWS S3 returns file checksums during the `list` operation, so that can be used to ensure the backup consistency, but that needs further research and, since current pageserver impl also needs to deal with layer file checksums.
+
+For now, due to this, we consider local workdir files as source of truth, not removing them ever and adjusting remote files instead, if image files mismatch.
+
+* no proper retry management
+
+Now, the storage sync attempts to redo the upload/download operation for the image files that failed.
+No proper task eviction or backpressure is implemented currently: the tasks will stay in the queue forever, reattempting the downloads.
+
+This will be fixed when more details on the file consistency model will be agreed on.
+
+* sad rust-s3 api
+
+rust-s3 is not very pleasant to use:
+1. it returns `anyhow::Result` and it's hard to distinguish "missing file" cases from "no connection" one, for instance
+2. at least one function it its API that we need (`get_object_stream`) has `async` keyword and blocks (!), see details [here](https://github.com/zenithdb/zenith/pull/752#discussion_r728373091)
+3. it's a prerelease library with unclear maintenance status
+4. noisy on debug level
+
+But it's already used in the project, so for now it's reused to avoid bloating the dependency tree.
+Based on previous evaluation, even `rusoto-s3` could be a better choice over this library, but needs further benchmarking.
+
+
+* gc and branches are ignored
+
+So far, we don't consider non-main images and don't adjust the remote storage based on GC thread loop results.
+Only checkpointer loop affects the remote storage.
+
+* more layers should be downloaded on demand
+
+Since we download and load remote layers into pageserver, there's a possibility a need for those layers' ancestors arise.
+Most probably, every downloaded image's ancestor is not present in locally too, but currently there's no logic for downloading such ancestors and their metadata,
+so the pageserver is unable to respond property on requests to such ancestors.
+
+To implement the downloading, more `tenant_mgr` refactoring is needed to properly handle web requests for layers and handle the state changes.
+[Here](https://github.com/zenithdb/zenith/pull/689#issuecomment-931216193) are the details about initial state management updates needed.
+
+* no IT tests
+
+Automated S3 testing is lacking currently, due to no convenient way to enable backups during the tests.
+After it's fixed, benchmark runs should also be carried out to find bottlenecks.

pageserver/src/relish_storage/local_fs.rs

@@ -1,23 +1,24 @@
-//! Local filesystem acting as a remote storage.
+//! Local filesystem relish storage.
 //! Multiple pageservers can use the same "storage" of this kind by using different storage roots.
 //!
 //! This storage used in pageserver tests, but can also be used in cases when a certain persistent
 //! volume is mounted to the local FS.
 
 use std::{
+    ffi::OsStr,
     future::Future,
+    io::Write,
     path::{Path, PathBuf},
     pin::Pin,
 };
 
-use anyhow::{bail, ensure, Context};
-use tokio::{
-    fs,
-    io::{self, AsyncReadExt, AsyncSeekExt, AsyncWriteExt},
-};
+use anyhow::{bail, Context};
+use tokio::{fs, io};
 use tracing::*;
 
-use super::{strip_path_prefix, RemoteStorage};
+use crate::layered_repository::metadata::METADATA_FILE_NAME;
+
+use super::{parse_ids_from_path, strip_path_prefix, RelishStorage, RemoteRelishInfo};
 
 pub struct LocalFs {
     pageserver_workdir: &'static Path,
@@ -25,7 +26,7 @@ pub struct LocalFs {
 }
 
 impl LocalFs {
-    /// Attempts to create local FS storage, along with its root directory.
+    /// Attempts to create local FS relish storage, along with the storage root directory.
     pub fn new(root: PathBuf, pageserver_workdir: &'static Path) -> anyhow::Result<Self> {
         if !root.exists() {
             std::fs::create_dir_all(&root).with_context(|| {
@@ -56,30 +57,90 @@ impl LocalFs {
 }
 
 #[async_trait::async_trait]
-impl RemoteStorage for LocalFs {
-    type StoragePath = PathBuf;
+impl RelishStorage for LocalFs {
+    type RelishStoragePath = PathBuf;
 
-    fn storage_path(&self, local_path: &Path) -> anyhow::Result<Self::StoragePath> {
+    fn storage_path(&self, local_path: &Path) -> anyhow::Result<Self::RelishStoragePath> {
         Ok(self.root.join(
             strip_path_prefix(self.pageserver_workdir, local_path)
                 .context("local path does not belong to this storage")?,
         ))
     }
 
-    fn local_path(&self, storage_path: &Self::StoragePath) -> anyhow::Result<PathBuf> {
+    fn info(&self, storage_path: &Self::RelishStoragePath) -> anyhow::Result<RemoteRelishInfo> {
+        let is_metadata =
+            storage_path.file_name().and_then(OsStr::to_str) == Some(METADATA_FILE_NAME);
         let relative_path = strip_path_prefix(&self.root, storage_path)
             .context("local path does not belong to this storage")?;
-        Ok(self.pageserver_workdir.join(relative_path))
+        let download_destination = self.pageserver_workdir.join(relative_path);
+        let (tenant_id, timeline_id) = parse_ids_from_path(
+            relative_path.iter().filter_map(|segment| segment.to_str()),
+            &relative_path.display(),
+        )?;
+        Ok(RemoteRelishInfo {
+            tenant_id,
+            timeline_id,
+            download_destination,
+            is_metadata,
+        })
     }
 
-    async fn list(&self) -> anyhow::Result<Vec<Self::StoragePath>> {
+    async fn list_relishes(&self) -> anyhow::Result<Vec<Self::RelishStoragePath>> {
         Ok(get_all_files(&self.root).await?.into_iter().collect())
     }
 
-    async fn upload(
+    async fn download_relish<W: 'static + std::io::Write + Send>(
         &self,
-        mut from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
-        to: &Self::StoragePath,
+        from: &Self::RelishStoragePath,
+        mut to: std::io::BufWriter<W>,
+    ) -> anyhow::Result<std::io::BufWriter<W>> {
+        let file_path = self.resolve_in_storage(from)?;
+
+        if file_path.exists() && file_path.is_file() {
+            let updated_buffer = tokio::task::spawn_blocking(move || {
+                let mut source = std::io::BufReader::new(
+                    std::fs::OpenOptions::new()
+                        .read(true)
+                        .open(&file_path)
+                        .with_context(|| {
+                            format!(
+                                "Failed to open source file '{}' to use in the download",
+                                file_path.display()
+                            )
+                        })?,
+                );
+                std::io::copy(&mut source, &mut to)
+                    .context("Failed to download the relish file")?;
+                to.flush().context("Failed to flush the download buffer")?;
+                Ok::<_, anyhow::Error>(to)
+            })
+            .await
+            .context("Failed to spawn a blocking task")??;
+            Ok(updated_buffer)
+        } else {
+            bail!(
+                "File '{}' either does not exist or is not a file",
+                file_path.display()
+            )
+        }
+    }
+
+    async fn delete_relish(&self, path: &Self::RelishStoragePath) -> anyhow::Result<()> {
+        let file_path = self.resolve_in_storage(path)?;
+        if file_path.exists() && file_path.is_file() {
+            Ok(fs::remove_file(file_path).await?)
+        } else {
+            bail!(
+                "File '{}' either does not exist or is not a file",
+                file_path.display()
+            )
+        }
+    }
+
+    async fn upload_relish<R: io::AsyncRead + std::marker::Unpin + Send>(
+        &self,
+        from: &mut io::BufReader<R>,
+        to: &Self::RelishStoragePath,
     ) -> anyhow::Result<()> {
         let target_file_path = self.resolve_in_storage(to)?;
         create_target_directory(&target_file_path).await?;
@@ -97,128 +158,11 @@ impl RemoteStorage for LocalFs {
                 })?,
         );
 
-        io::copy(&mut from, &mut destination)
+        io::copy_buf(from, &mut destination)
             .await
-            .with_context(|| {
-                format!(
-                    "Failed to upload file to the local storage at '{}'",
-                    target_file_path.display()
-                )
-            })?;
-        destination.flush().await.with_context(|| {
-            format!(
-                "Failed to upload file to the local storage at '{}'",
-                target_file_path.display()
-            )
-        })?;
+            .context("Failed to upload relish to local storage")?;
         Ok(())
     }
-
-    async fn download(
-        &self,
-        from: &Self::StoragePath,
-        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<()> {
-        let file_path = self.resolve_in_storage(from)?;
-
-        if file_path.exists() && file_path.is_file() {
-            let mut source = io::BufReader::new(
-                fs::OpenOptions::new()
-                    .read(true)
-                    .open(&file_path)
-                    .await
-                    .with_context(|| {
-                        format!(
-                            "Failed to open source file '{}' to use in the download",
-                            file_path.display()
-                        )
-                    })?,
-            );
-            io::copy(&mut source, to).await.with_context(|| {
-                format!(
-                    "Failed to download file '{}' from the local storage",
-                    file_path.display()
-                )
-            })?;
-            source.flush().await?;
-            Ok(())
-        } else {
-            bail!(
-                "File '{}' either does not exist or is not a file",
-                file_path.display()
-            )
-        }
-    }
-
-    async fn download_range(
-        &self,
-        from: &Self::StoragePath,
-        start_inclusive: u64,
-        end_exclusive: Option<u64>,
-        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<()> {
-        if let Some(end_exclusive) = end_exclusive {
-            ensure!(
-                end_exclusive > start_inclusive,
-                "Invalid range, start ({}) is bigger then end ({:?})",
-                start_inclusive,
-                end_exclusive
-            );
-            if start_inclusive == end_exclusive.saturating_sub(1) {
-                return Ok(());
-            }
-        }
-        let file_path = self.resolve_in_storage(from)?;
-
-        if file_path.exists() && file_path.is_file() {
-            let mut source = io::BufReader::new(
-                fs::OpenOptions::new()
-                    .read(true)
-                    .open(&file_path)
-                    .await
-                    .with_context(|| {
-                        format!(
-                            "Failed to open source file '{}' to use in the download",
-                            file_path.display()
-                        )
-                    })?,
-            );
-            source
-                .seek(io::SeekFrom::Start(start_inclusive))
-                .await
-                .context("Failed to seek to the range start in a local storage file")?;
-            match end_exclusive {
-                Some(end_exclusive) => {
-                    io::copy(&mut source.take(end_exclusive - start_inclusive), to).await
-                }
-                None => io::copy(&mut source, to).await,
-            }
-            .with_context(|| {
-                format!(
-                    "Failed to download file '{}' range from the local storage",
-                    file_path.display()
-                )
-            })?;
-            Ok(())
-        } else {
-            bail!(
-                "File '{}' either does not exist or is not a file",
-                file_path.display()
-            )
-        }
-    }
-
-    async fn delete(&self, path: &Self::StoragePath) -> anyhow::Result<()> {
-        let file_path = self.resolve_in_storage(path)?;
-        if file_path.exists() && file_path.is_file() {
-            Ok(fs::remove_file(file_path).await?)
-        } else {
-            bail!(
-                "File '{}' either does not exist or is not a file",
-                file_path.display()
-            )
-        }
-    }
 }
 
 fn get_all_files<'a, P>(
@@ -258,7 +202,7 @@ async fn create_target_directory(target_file_path: &Path) -> anyhow::Result<()>
     let target_dir = match target_file_path.parent() {
         Some(parent_dir) => parent_dir,
         None => bail!(
-            "File path '{}' has no parent directory",
+            "Relish path '{}' has no parent directory",
             target_file_path.display()
         ),
     };
@@ -272,6 +216,9 @@ async fn create_target_directory(target_file_path: &Path) -> anyhow::Result<()>
 mod pure_tests {
     use crate::{
         layered_repository::metadata::METADATA_FILE_NAME,
+        relish_storage::test_utils::{
+            custom_tenant_id_path, custom_timeline_id_path, relative_timeline_path,
+        },
         repository::repo_harness::{RepoHarness, TIMELINE_ID},
     };
 
@@ -286,13 +233,13 @@ mod pure_tests {
             root: storage_root.clone(),
         };
 
-        let local_path = repo_harness.timeline_path(&TIMELINE_ID).join("file_name");
+        let local_path = repo_harness.timeline_path(&TIMELINE_ID).join("relish_name");
         let expected_path = storage_root.join(local_path.strip_prefix(&repo_harness.conf.workdir)?);
 
         assert_eq!(
             expected_path,
             storage.storage_path(&local_path).expect("Matching path should map to storage path normally"),
-            "File paths from pageserver workdir should be stored in local fs storage with the same path they have relative to the workdir"
+            "Relish paths from pageserver workdir should be stored in local fs storage with the same path they have relative to the workdir"
         );
 
         Ok(())
@@ -339,8 +286,8 @@ mod pure_tests {
     }
 
     #[test]
-    fn local_path_positive() -> anyhow::Result<()> {
-        let repo_harness = RepoHarness::create("local_path_positive")?;
+    fn info_positive() -> anyhow::Result<()> {
+        let repo_harness = RepoHarness::create("info_positive")?;
         let storage_root = PathBuf::from("somewhere").join("else");
         let storage = LocalFs {
             pageserver_workdir: &repo_harness.conf.workdir,
@@ -350,13 +297,16 @@ mod pure_tests {
         let name = "not a metadata";
         let local_path = repo_harness.timeline_path(&TIMELINE_ID).join(name);
         assert_eq!(
-            local_path,
+            RemoteRelishInfo {
+                tenant_id: repo_harness.tenant_id,
+                timeline_id: TIMELINE_ID,
+                download_destination: local_path.clone(),
+                is_metadata: false,
+            },
             storage
-                .local_path(
-                    &storage_root.join(local_path.strip_prefix(&repo_harness.conf.workdir)?)
-                )
-                .expect("For a valid input, valid local path should be parsed"),
-            "Should be able to parse metadata out of the correctly named remote delta file"
+                .info(&storage_root.join(local_path.strip_prefix(&repo_harness.conf.workdir)?))
+                .expect("For a valid input, valid S3 info should be parsed"),
+            "Should be able to parse metadata out of the correctly named remote delta relish"
         );
 
         let local_metadata_path = repo_harness
@@ -364,10 +314,15 @@ mod pure_tests {
             .join(METADATA_FILE_NAME);
         let remote_metadata_path = storage.storage_path(&local_metadata_path)?;
         assert_eq!(
-            local_metadata_path,
+            RemoteRelishInfo {
+                tenant_id: repo_harness.tenant_id,
+                timeline_id: TIMELINE_ID,
+                download_destination: local_metadata_path,
+                is_metadata: true,
+            },
             storage
-                .local_path(&remote_metadata_path)
-                .expect("For a valid input, valid local path should be parsed"),
+                .info(&remote_metadata_path)
+                .expect("For a valid input, valid S3 info should be parsed"),
             "Should be able to parse metadata out of the correctly named remote metadata file"
         );
 
@@ -375,30 +330,54 @@ mod pure_tests {
     }
 
     #[test]
-    fn local_path_negatives() -> anyhow::Result<()> {
+    fn info_negatives() -> anyhow::Result<()> {
         #[track_caller]
-        #[allow(clippy::ptr_arg)] // have to use &PathBuf due to `storage.local_path` parameter requirements
-        fn local_path_error(storage: &LocalFs, storage_path: &PathBuf) -> String {
-            match storage.local_path(storage_path) {
-                Ok(wrong_path) => panic!(
-                    "Expected local path input {:?} to cause an error, but got file path: {:?}",
-                    storage_path, wrong_path,
+        #[allow(clippy::ptr_arg)] // have to use &PathBuf due to `storage.info` parameter requirements
+        fn storage_info_error(storage: &LocalFs, storage_path: &PathBuf) -> String {
+            match storage.info(storage_path) {
+                Ok(wrong_info) => panic!(
+                    "Expected storage path input {:?} to cause an error, but got relish info: {:?}",
+                    storage_path, wrong_info,
                 ),
                 Err(e) => format!("{:?}", e),
             }
         }
 
-        let repo_harness = RepoHarness::create("local_path_negatives")?;
+        let repo_harness = RepoHarness::create("info_negatives")?;
         let storage_root = PathBuf::from("somewhere").join("else");
         let storage = LocalFs {
             pageserver_workdir: &repo_harness.conf.workdir,
-            root: storage_root,
+            root: storage_root.clone(),
         };
 
         let totally_wrong_path = "wrong_wrong_wrong";
-        let error_message = local_path_error(&storage, &PathBuf::from(totally_wrong_path));
+        let error_message = storage_info_error(&storage, &PathBuf::from(totally_wrong_path));
         assert!(error_message.contains(totally_wrong_path));
 
+        let relative_timeline_path = relative_timeline_path(&repo_harness)?;
+
+        let relative_relish_path =
+            custom_tenant_id_path(&relative_timeline_path, "wrong_tenant_id")?
+                .join("wrong_tenant_id_name");
+        let wrong_tenant_id_path = storage_root.join(&relative_relish_path);
+        let error_message = storage_info_error(&storage, &wrong_tenant_id_path);
+        assert!(
+            error_message.contains(relative_relish_path.to_str().unwrap()),
+            "Error message '{}' does not contain the expected substring",
+            error_message
+        );
+
+        let relative_relish_path =
+            custom_timeline_id_path(&relative_timeline_path, "wrong_timeline_id")?
+                .join("wrong_timeline_id_name");
+        let wrong_timeline_id_path = storage_root.join(&relative_relish_path);
+        let error_message = storage_info_error(&storage, &wrong_timeline_id_path);
+        assert!(
+            error_message.contains(relative_relish_path.to_str().unwrap()),
+            "Error message '{}' does not contain the expected substring",
+            error_message
+        );
+
         Ok(())
     }
 
@@ -414,7 +393,7 @@ mod pure_tests {
         };
 
         let storage_path = dummy_storage.storage_path(&original_path)?;
-        let download_destination = dummy_storage.local_path(&storage_path)?;
+        let download_destination = dummy_storage.info(&storage_path)?.download_destination;
 
         assert_eq!(
             original_path, download_destination,
@@ -427,24 +406,26 @@ mod pure_tests {
 
 #[cfg(test)]
 mod fs_tests {
-    use super::*;
-    use crate::repository::repo_harness::{RepoHarness, TIMELINE_ID};
+    use crate::{
+        relish_storage::test_utils::relative_timeline_path, repository::repo_harness::RepoHarness,
+    };
+
+    use super::*;
 
-    use std::io::Write;
     use tempfile::tempdir;
 
     #[tokio::test]
-    async fn upload_file() -> anyhow::Result<()> {
-        let repo_harness = RepoHarness::create("upload_file")?;
+    async fn upload_relish() -> anyhow::Result<()> {
+        let repo_harness = RepoHarness::create("upload_relish")?;
         let storage = create_storage()?;
 
-        let source = create_file_for_upload(
+        let mut source = create_file_for_upload(
             &storage.pageserver_workdir.join("whatever"),
             "whatever_contents",
         )
         .await?;
         let target_path = PathBuf::from("/").join("somewhere").join("else");
-        match storage.upload(source, &target_path).await {
+        match storage.upload_relish(&mut source, &target_path).await {
             Ok(()) => panic!("Should not allow storing files with wrong target path"),
             Err(e) => {
                 let message = format!("{:?}", e);
@@ -452,22 +433,36 @@ mod fs_tests {
                 assert!(message.contains("does not belong to the current storage"));
             }
         }
-        assert!(storage.list().await?.is_empty());
+        assert!(storage.list_relishes().await?.is_empty());
 
         let target_path_1 = upload_dummy_file(&repo_harness, &storage, "upload_1").await?;
         assert_eq!(
-            storage.list().await?,
+            storage.list_relishes().await?,
             vec![target_path_1.clone()],
             "Should list a single file after first upload"
         );
 
         let target_path_2 = upload_dummy_file(&repo_harness, &storage, "upload_2").await?;
         assert_eq!(
-            list_files_sorted(&storage).await?,
+            list_relishes_sorted(&storage).await?,
             vec![target_path_1.clone(), target_path_2.clone()],
             "Should list a two different files after second upload"
         );
 
+        // match storage.upload_relish(&mut source, &target_path_1).await {
+        //     Ok(()) => panic!("Should not allow reuploading storage files"),
+        //     Err(e) => {
+        //         let message = format!("{:?}", e);
+        //         assert!(message.contains(&target_path_1.display().to_string()));
+        //         assert!(message.contains("File exists"));
+        //     }
+        // }
+        assert_eq!(
+            list_relishes_sorted(&storage).await?,
+            vec![target_path_1, target_path_2],
+            "Should list a two different files after all upload attempts"
+        );
+
         Ok(())
     }
 
@@ -478,17 +473,17 @@ mod fs_tests {
     }
 
     #[tokio::test]
-    async fn download_file() -> anyhow::Result<()> {
-        let repo_harness = RepoHarness::create("download_file")?;
+    async fn download_relish() -> anyhow::Result<()> {
+        let repo_harness = RepoHarness::create("download_relish")?;
         let storage = create_storage()?;
         let upload_name = "upload_1";
         let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;
 
-        let mut content_bytes = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        storage.download(&upload_target, &mut content_bytes).await?;
-        content_bytes.flush().await?;
-
-        let contents = String::from_utf8(content_bytes.into_inner().into_inner())?;
+        let contents_bytes = storage
+            .download_relish(&upload_target, std::io::BufWriter::new(Vec::new()))
+            .await?
+            .into_inner()?;
+        let contents = String::from_utf8(contents_bytes)?;
         assert_eq!(
             dummy_contents(upload_name),
             contents,
@@ -496,7 +491,10 @@ mod fs_tests {
         );
 
         let non_existing_path = PathBuf::from("somewhere").join("else");
-        match storage.download(&non_existing_path, &mut io::sink()).await {
+        match storage
+            .download_relish(&non_existing_path, std::io::BufWriter::new(Vec::new()))
+            .await
+        {
             Ok(_) => panic!("Should not allow downloading non-existing storage files"),
             Err(e) => {
                 let error_string = e.to_string();
@@ -508,128 +506,16 @@ mod fs_tests {
     }
 
     #[tokio::test]
-    async fn download_file_range_positive() -> anyhow::Result<()> {
-        let repo_harness = RepoHarness::create("download_file_range_positive")?;
+    async fn delete_relish() -> anyhow::Result<()> {
+        let repo_harness = RepoHarness::create("delete_relish")?;
         let storage = create_storage()?;
         let upload_name = "upload_1";
         let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;
 
-        let mut full_range_bytes = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        storage
-            .download_range(&upload_target, 0, None, &mut full_range_bytes)
-            .await?;
-        full_range_bytes.flush().await?;
-        assert_eq!(
-            dummy_contents(upload_name),
-            String::from_utf8(full_range_bytes.into_inner().into_inner())?,
-            "Download full range should return the whole upload"
-        );
+        storage.delete_relish(&upload_target).await?;
+        assert!(storage.list_relishes().await?.is_empty());
 
-        let mut zero_range_bytes = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let same_byte = 1_000_000_000;
-        storage
-            .download_range(
-                &upload_target,
-                same_byte,
-                Some(same_byte + 1), // exclusive end
-                &mut zero_range_bytes,
-            )
-            .await?;
-        zero_range_bytes.flush().await?;
-        assert!(
-            zero_range_bytes.into_inner().into_inner().is_empty(),
-            "Zero byte range should not download any part of the file"
-        );
-
-        let uploaded_bytes = dummy_contents(upload_name).into_bytes();
-        let (first_part_local, second_part_local) = uploaded_bytes.split_at(3);
-
-        let mut first_part_remote = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        storage
-            .download_range(
-                &upload_target,
-                0,
-                Some(first_part_local.len() as u64),
-                &mut first_part_remote,
-            )
-            .await?;
-        first_part_remote.flush().await?;
-        let first_part_remote = first_part_remote.into_inner().into_inner();
-        assert_eq!(
-            first_part_local,
-            first_part_remote.as_slice(),
-            "First part bytes should be returned when requested"
-        );
-
-        let mut second_part_remote = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        storage
-            .download_range(
-                &upload_target,
-                first_part_local.len() as u64,
-                Some((first_part_local.len() + second_part_local.len()) as u64),
-                &mut second_part_remote,
-            )
-            .await?;
-        second_part_remote.flush().await?;
-        let second_part_remote = second_part_remote.into_inner().into_inner();
-        assert_eq!(
-            second_part_local,
-            second_part_remote.as_slice(),
-            "Second part bytes should be returned when requested"
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn download_file_range_negative() -> anyhow::Result<()> {
-        let repo_harness = RepoHarness::create("download_file_range_negative")?;
-        let storage = create_storage()?;
-        let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;
-
-        let start = 10000;
-        let end = 234;
-        assert!(start > end, "Should test an incorrect range");
-        match storage
-            .download_range(&upload_target, start, Some(end), &mut io::sink())
-            .await
-        {
-            Ok(_) => panic!("Should not allow downloading wrong ranges"),
-            Err(e) => {
-                let error_string = e.to_string();
-                assert!(error_string.contains("Invalid range"));
-                assert!(error_string.contains(&start.to_string()));
-                assert!(error_string.contains(&end.to_string()));
-            }
-        }
-
-        let non_existing_path = PathBuf::from("somewhere").join("else");
-        match storage
-            .download_range(&non_existing_path, 1, Some(3), &mut io::sink())
-            .await
-        {
-            Ok(_) => panic!("Should not allow downloading non-existing storage file ranges"),
-            Err(e) => {
-                let error_string = e.to_string();
-                assert!(error_string.contains("does not exist"));
-                assert!(error_string.contains(&non_existing_path.display().to_string()));
-            }
-        }
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn delete_file() -> anyhow::Result<()> {
-        let repo_harness = RepoHarness::create("delete_file")?;
-        let storage = create_storage()?;
-        let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;
-
-        storage.delete(&upload_target).await?;
-        assert!(storage.list().await?.is_empty());
-
-        match storage.delete(&upload_target).await {
+        match storage.delete_relish(&upload_target).await {
             Ok(()) => panic!("Should not allow deleting non-existing storage files"),
             Err(e) => {
                 let error_string = e.to_string();
@@ -645,12 +531,13 @@ mod fs_tests {
         storage: &LocalFs,
         name: &str,
     ) -> anyhow::Result<PathBuf> {
-        let timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let relative_timeline_path = timeline_path.strip_prefix(&harness.conf.workdir)?;
-        let storage_path = storage.root.join(relative_timeline_path).join(name);
+        let storage_path = storage
+            .root
+            .join(relative_timeline_path(harness)?)
+            .join(name);
         storage
-            .upload(
-                create_file_for_upload(
+            .upload_relish(
+                &mut create_file_for_upload(
                     &storage.pageserver_workdir.join(name),
                     &dummy_contents(name),
                 )
@@ -681,9 +568,9 @@ mod fs_tests {
         format!("contents for {}", name)
     }
 
-    async fn list_files_sorted(storage: &LocalFs) -> anyhow::Result<Vec<PathBuf>> {
-        let mut files = storage.list().await?;
-        files.sort();
-        Ok(files)
+    async fn list_relishes_sorted(storage: &LocalFs) -> anyhow::Result<Vec<PathBuf>> {
+        let mut relishes = storage.list_relishes().await?;
+        relishes.sort();
+        Ok(relishes)
     }
 }

pageserver/src/relish_storage/rust_s3.rs

@@ -1,16 +1,19 @@
-//! AWS S3 storage wrapper around `rust_s3` library.
-//! Currently does not allow multiple pageservers to use the same bucket concurrently: objects are
+//! AWS S3 relish storage wrapper around `rust_s3` library.
+//! Currently does not allow multiple pageservers to use the same bucket concurrently: relishes are
 //! placed in the root of the bucket.
 
-use std::path::{Path, PathBuf};
+use std::{
+    io::Write,
+    path::{Path, PathBuf},
+};
 
 use anyhow::Context;
 use s3::{bucket::Bucket, creds::Credentials, region::Region};
-use tokio::io::{self, AsyncWriteExt};
 
 use crate::{
-    config::S3Config,
-    remote_storage::{strip_path_prefix, RemoteStorage},
+    layered_repository::metadata::METADATA_FILE_NAME,
+    relish_storage::{parse_ids_from_path, strip_path_prefix, RelishStorage, RemoteRelishInfo},
+    S3Config,
 };
 
 const S3_FILE_SEPARATOR: char = '/';
@@ -28,14 +31,14 @@ impl S3ObjectKey {
     }
 }
 
-/// AWS S3 storage.
+/// AWS S3 relish storage.
 pub struct S3 {
     pageserver_workdir: &'static Path,
     bucket: Bucket,
 }
 
 impl S3 {
-    /// Creates the storage, errors if incorrect AWS S3 configuration provided.
+    /// Creates the relish storage, errors if incorrect AWS S3 configuration provided.
     pub fn new(aws_config: &S3Config, pageserver_workdir: &'static Path) -> anyhow::Result<Self> {
         let region = aws_config
             .bucket_region
@@ -62,10 +65,10 @@ impl S3 {
 }
 
 #[async_trait::async_trait]
-impl RemoteStorage for S3 {
-    type StoragePath = S3ObjectKey;
+impl RelishStorage for S3 {
+    type RelishStoragePath = S3ObjectKey;
 
-    fn storage_path(&self, local_path: &Path) -> anyhow::Result<Self::StoragePath> {
+    fn storage_path(&self, local_path: &Path) -> anyhow::Result<Self::RelishStoragePath> {
         let relative_path = strip_path_prefix(self.pageserver_workdir, local_path)?;
         let mut key = String::new();
         for segment in relative_path {
@@ -75,11 +78,22 @@ impl RemoteStorage for S3 {
         Ok(S3ObjectKey(key))
     }
 
-    fn local_path(&self, storage_path: &Self::StoragePath) -> anyhow::Result<PathBuf> {
-        Ok(storage_path.download_destination(self.pageserver_workdir))
+    fn info(&self, storage_path: &Self::RelishStoragePath) -> anyhow::Result<RemoteRelishInfo> {
+        let storage_path_key = &storage_path.0;
+        let is_metadata =
+            storage_path_key.ends_with(&format!("{}{}", S3_FILE_SEPARATOR, METADATA_FILE_NAME));
+        let download_destination = storage_path.download_destination(self.pageserver_workdir);
+        let (tenant_id, timeline_id) =
+            parse_ids_from_path(storage_path_key.split(S3_FILE_SEPARATOR), storage_path_key)?;
+        Ok(RemoteRelishInfo {
+            tenant_id,
+            timeline_id,
+            download_destination,
+            is_metadata,
+        })
     }
 
-    async fn list(&self) -> anyhow::Result<Vec<Self::StoragePath>> {
+    async fn list_relishes(&self) -> anyhow::Result<Vec<Self::RelishStoragePath>> {
         let list_response = self
             .bucket
             .list(String::new(), None)
@@ -93,93 +107,32 @@ impl RemoteStorage for S3 {
             .collect())
     }
 
-    async fn upload(
+    async fn download_relish<W: 'static + std::io::Write + Send>(
         &self,
-        mut from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
-        to: &Self::StoragePath,
-    ) -> anyhow::Result<()> {
-        let mut upload_contents = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        io::copy(&mut from, &mut upload_contents)
-            .await
-            .context("Failed to read the upload contents")?;
-        upload_contents
-            .flush()
-            .await
-            .context("Failed to read the upload contents")?;
-        let upload_contents = upload_contents.into_inner().into_inner();
-
-        let (_, code) = self
+        from: &Self::RelishStoragePath,
+        mut to: std::io::BufWriter<W>,
+    ) -> anyhow::Result<std::io::BufWriter<W>> {
+        let code = self
             .bucket
-            .put_object(to.key(), &upload_contents)
-            .await
-            .with_context(|| format!("Failed to create s3 object with key {}", to.key()))?;
-        if code != 200 {
-            Err(anyhow::format_err!(
-                "Received non-200 exit code during creating object with key '{}', code: {}",
-                to.key(),
-                code
-            ))
-        } else {
-            Ok(())
-        }
-    }
-
-    async fn download(
-        &self,
-        from: &Self::StoragePath,
-        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<()> {
-        let (data, code) = self
-            .bucket
-            .get_object(from.key())
+            .get_object_stream(from.key(), &mut to)
             .await
             .with_context(|| format!("Failed to download s3 object with key {}", from.key()))?;
         if code != 200 {
             Err(anyhow::format_err!(
-                "Received non-200 exit code during downloading object, code: {}",
+                "Received non-200 exit code during downloading object from directory, code: {}",
                 code
             ))
         } else {
-            // we don't have to write vector into the destination this way, `to_write_all` would be enough.
-            // but we want to prepare for migration on `rusoto`, that has a streaming HTTP body instead here, with
-            // which it makes more sense to use `io::copy`.
-            io::copy(&mut data.as_slice(), to)
-                .await
-                .context("Failed to write downloaded data into the destination buffer")?;
-            Ok(())
-        }
-    }
-
-    async fn download_range(
-        &self,
-        from: &Self::StoragePath,
-        start_inclusive: u64,
-        end_exclusive: Option<u64>,
-        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<()> {
-        // S3 accepts ranges as https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35
-        // and needs both ends to be exclusive
-        let end_inclusive = end_exclusive.map(|end| end.saturating_sub(1));
-        let (data, code) = self
-            .bucket
-            .get_object_range(from.key(), start_inclusive, end_inclusive)
+            tokio::task::spawn_blocking(move || {
+                to.flush().context("Failed to flush the download buffer")?;
+                Ok::<_, anyhow::Error>(to)
+            })
             .await
-            .with_context(|| format!("Failed to download s3 object with key {}", from.key()))?;
-        if code != 206 {
-            Err(anyhow::format_err!(
-                "Received non-206 exit code during downloading object range, code: {}",
-                code
-            ))
-        } else {
-            // see `download` function above for the comment on why `Vec<u8>` buffer is copied this way
-            io::copy(&mut data.as_slice(), to)
-                .await
-                .context("Failed to write downloaded range into the destination buffer")?;
-            Ok(())
+            .context("Failed to join the download buffer flush task")?
         }
     }
 
-    async fn delete(&self, path: &Self::StoragePath) -> anyhow::Result<()> {
+    async fn delete_relish(&self, path: &Self::RelishStoragePath) -> anyhow::Result<()> {
         let (_, code) = self
             .bucket
             .delete_object(path.key())
@@ -195,12 +148,35 @@ impl RemoteStorage for S3 {
             Ok(())
         }
     }
+
+    async fn upload_relish<R: tokio::io::AsyncRead + std::marker::Unpin + Send>(
+        &self,
+        from: &mut tokio::io::BufReader<R>,
+        to: &Self::RelishStoragePath,
+    ) -> anyhow::Result<()> {
+        let code = self
+            .bucket
+            .put_object_stream(from, to.key())
+            .await
+            .with_context(|| format!("Failed to create s3 object with key {}", to.key()))?;
+        if code != 200 {
+            Err(anyhow::format_err!(
+                "Received non-200 exit code during creating object with key '{}', code: {}",
+                to.key(),
+                code
+            ))
+        } else {
+            Ok(())
+        }
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use crate::{
-        layered_repository::metadata::METADATA_FILE_NAME,
+        relish_storage::test_utils::{
+            custom_tenant_id_path, custom_timeline_id_path, relative_timeline_path,
+        },
         repository::repo_harness::{RepoHarness, TIMELINE_ID},
     };
 
@@ -237,7 +213,7 @@ mod tests {
         let repo_harness = RepoHarness::create("storage_path_positive")?;
 
         let segment_1 = "matching";
-        let segment_2 = "file";
+        let segment_2 = "relish";
         let local_path = &repo_harness.conf.workdir.join(segment_1).join(segment_2);
         let expected_key = S3ObjectKey(format!(
             "{SEPARATOR}{}{SEPARATOR}{}",
@@ -302,26 +278,35 @@ mod tests {
     }
 
     #[test]
-    fn local_path_positive() -> anyhow::Result<()> {
-        let repo_harness = RepoHarness::create("local_path_positive")?;
+    fn info_positive() -> anyhow::Result<()> {
+        let repo_harness = RepoHarness::create("info_positive")?;
         let storage = dummy_storage(&repo_harness.conf.workdir);
-        let timeline_dir = repo_harness.timeline_path(&TIMELINE_ID);
-        let relative_timeline_path = timeline_dir.strip_prefix(&repo_harness.conf.workdir)?;
+        let relative_timeline_path = relative_timeline_path(&repo_harness)?;
 
         let s3_key = create_s3_key(&relative_timeline_path.join("not a metadata"));
         assert_eq!(
-            s3_key.download_destination(&repo_harness.conf.workdir),
+            RemoteRelishInfo {
+                tenant_id: repo_harness.tenant_id,
+                timeline_id: TIMELINE_ID,
+                download_destination: s3_key.download_destination(&repo_harness.conf.workdir),
+                is_metadata: false,
+            },
             storage
-                .local_path(&s3_key)
+                .info(&s3_key)
                 .expect("For a valid input, valid S3 info should be parsed"),
-            "Should be able to parse metadata out of the correctly named remote delta file"
+            "Should be able to parse metadata out of the correctly named remote delta relish"
         );
 
         let s3_key = create_s3_key(&relative_timeline_path.join(METADATA_FILE_NAME));
         assert_eq!(
-            s3_key.download_destination(&repo_harness.conf.workdir),
+            RemoteRelishInfo {
+                tenant_id: repo_harness.tenant_id,
+                timeline_id: TIMELINE_ID,
+                download_destination: s3_key.download_destination(&repo_harness.conf.workdir),
+                is_metadata: true,
+            },
             storage
-                .local_path(&s3_key)
+                .info(&s3_key)
                 .expect("For a valid input, valid S3 info should be parsed"),
             "Should be able to parse metadata out of the correctly named remote metadata file"
         );
@@ -329,6 +314,43 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn info_negatives() -> anyhow::Result<()> {
+        #[track_caller]
+        fn storage_info_error(storage: &S3, s3_key: &S3ObjectKey) -> String {
+            match storage.info(s3_key) {
+                Ok(wrong_info) => panic!(
+                    "Expected key {:?} to error, but got relish info: {:?}",
+                    s3_key, wrong_info,
+                ),
+                Err(e) => e.to_string(),
+            }
+        }
+
+        let repo_harness = RepoHarness::create("info_negatives")?;
+        let storage = dummy_storage(&repo_harness.conf.workdir);
+        let relative_timeline_path = relative_timeline_path(&repo_harness)?;
+
+        let totally_wrong_path = "wrong_wrong_wrong";
+        let error_message =
+            storage_info_error(&storage, &S3ObjectKey(totally_wrong_path.to_string()));
+        assert!(error_message.contains(totally_wrong_path));
+
+        let wrong_tenant_id = create_s3_key(
+            &custom_tenant_id_path(&relative_timeline_path, "wrong_tenant_id")?.join("name"),
+        );
+        let error_message = storage_info_error(&storage, &wrong_tenant_id);
+        assert!(error_message.contains(&wrong_tenant_id.0));
+
+        let wrong_timeline_id = create_s3_key(
+            &custom_timeline_id_path(&relative_timeline_path, "wrong_timeline_id")?.join("name"),
+        );
+        let error_message = storage_info_error(&storage, &wrong_timeline_id);
+        assert!(error_message.contains(&wrong_timeline_id.0));
+
+        Ok(())
+    }
+
     #[test]
     fn download_destination_matches_original_path() -> anyhow::Result<()> {
         let repo_harness = RepoHarness::create("download_destination_matches_original_path")?;
@@ -337,7 +359,7 @@ mod tests {
         let dummy_storage = dummy_storage(&repo_harness.conf.workdir);
 
         let key = dummy_storage.storage_path(&original_path)?;
-        let download_destination = dummy_storage.local_path(&key)?;
+        let download_destination = dummy_storage.info(&key)?.download_destination;
 
         assert_eq!(
             original_path, download_destination,
@@ -359,9 +381,9 @@ mod tests {
         }
     }
 
-    fn create_s3_key(relative_file_path: &Path) -> S3ObjectKey {
+    fn create_s3_key(relative_relish_path: &Path) -> S3ObjectKey {
         S3ObjectKey(
-            relative_file_path
+            relative_relish_path
                 .iter()
                 .fold(String::new(), |mut path_string, segment| {
                     path_string.push(S3_FILE_SEPARATOR);

pageserver/src/remote_storage.rs

@@ -1,360 +0,0 @@
-//! A set of generic storage abstractions for the page server to use when backing up and restoring its state from the external storage.
-//! This particular module serves as a public API border between pageserver and the internal storage machinery.
-//! No other modules from this tree are supposed to be used directly by the external code.
-//!
-//! There are a few components the storage machinery consists of:
-//! * [`RemoteStorage`] trait a CRUD-like generic abstraction to use for adapting external storages with a few implementations:
-//!     * [`local_fs`] allows to use local file system as an external storage
-//!     * [`rust_s3`] uses AWS S3 bucket entirely as an external storage
-//!
-//! * synchronization logic at [`storage_sync`] module that keeps pageserver state (both runtime one and the workdir files) and storage state in sync.
-//! Synchronization internals are split into submodules
-//!     * [`storage_sync::compression`] for a custom remote storage format used to store timeline files in archives
-//!     * [`storage_sync::index`] to keep track of remote tenant files, the metadata and their mappings to local files
-//!     * [`storage_sync::upload`] and [`storage_sync::download`] to manage archive creation and upload; download and extraction, respectively
-//!
-//! * public API via to interact with the external world:
-//!     * [`start_local_timeline_sync`] to launch a background async loop to handle the synchronization
-//!     * [`schedule_timeline_checkpoint_upload`] and [`schedule_timeline_download`] to enqueue a new upload and download tasks,
-//!       to be processed by the async loop
-//!
-//! Here's a schematic overview of all interactions backup and the rest of the pageserver perform:
-//!
-//! +------------------------+                                    +--------->-------+
-//! |                        |  - - - (init async loop) - - - ->  |                 |
-//! |                        |                                    |                 |
-//! |                        |  ------------------------------->  |      async      |
-//! |       pageserver       |    (enqueue timeline sync task)    | upload/download |
-//! |                        |                                    |      loop       |
-//! |                        |  <-------------------------------  |                 |
-//! |                        |  (apply new timeline sync states)  |                 |
-//! +------------------------+                                    +---------<-------+
-//!                                                                         |
-//!                                                                         |
-//!                                          CRUD layer file operations     |
-//!                                     (upload/download/delete/list, etc.) |
-//!                                                                         V
-//!                                                            +------------------------+
-//!                                                            |                        |
-//!                                                            | [`RemoteStorage`] impl |
-//!                                                            |                        |
-//!                                                            | pageserver assumes it  |
-//!                                                            | owns exclusive write   |
-//!                                                            | access to this storage |
-//!                                                            +------------------------+
-//!
-//! First, during startup, the pageserver inits the storage sync thread with the async loop, or leaves the loop uninitialised, if configured so.
-//! The loop inits the storage connection and checks the remote files stored.
-//! This is done once at startup only, relying on the fact that pageserver uses the storage alone (ergo, nobody else uploads the files to the storage but this server).
-//! Based on the remote storage data, the sync logic immediately schedules sync tasks for local timelines and reports about remote only timelines to pageserver, so it can
-//! query their downloads later if they are accessed.
-//!
-//! Some time later, during pageserver checkpoints, in-memory data is flushed onto disk along with its metadata.
-//! If the storage sync loop was successfully started before, pageserver schedules the new checkpoint file uploads after every checkpoint.
-//! The checkpoint uploads are disabled, if no remote storage configuration is provided (no sync loop is started this way either).
-//! See [`crate::layered_repository`] for the upload calls and the adjacent logic.
-//!
-//! Synchronization logic is able to communicate back with updated timeline sync states, [`TimelineSyncState`],
-//! submitted via [`crate::tenant_mgr::set_timeline_states`] function. Tenant manager applies corresponding timeline updates in pageserver's in-memory state.
-//! Such submissions happen in two cases:
-//! * once after the sync loop startup, to signal pageserver which timelines will be synchronized in the near future
-//! * after every loop step, in case a timeline needs to be reloaded or evicted from pageserver's memory
-//!
-//! When the pageserver terminates, the upload loop finishes a current sync task (if any) and exits.
-//!
-//! The storage logic considers `image` as a set of local files, fully representing a certain timeline at given moment (identified with `disk_consistent_lsn`).
-//! Timeline can change its state, by adding more files on disk and advancing its `disk_consistent_lsn`: this happens after pageserver checkpointing and is followed
-//! by the storage upload, if enabled.
-//! Yet timeline cannot alter already existing files, and normally cannot remote those too: only a GC process is capable of removing unused files.
-//! This way, remote storage synchronization relies on the fact that every checkpoint is incremental and local files are "immutable":
-//! * when a certain checkpoint gets uploaded, the sync loop remembers the fact, preventing further reuploads of the same state
-//! * no files are deleted from either local or remote storage, only the missing ones locally/remotely get downloaded/uploaded, local metadata file will be overwritten
-//! when the newer image is downloaded
-//!
-//! To optimize S3 storage (and access), the sync loop compresses the checkpoint files before placing them to S3, and uncompresses them back, keeping track of timeline files and metadata.
-//! Also, the remote file list is queried once only, at startup, to avoid possible extra costs and latency issues.
-//!
-//! NOTES:
-//! * pageserver assumes it has exclusive write access to the remote storage. If supported, the way multiple pageservers can be separated in the same storage
-//! (i.e. using different directories in the local filesystem external storage), but totally up to the storage implementation and not covered with the trait API.
-//!
-//! * the sync tasks may not processed immediately after the submission: if they error and get re-enqueued, their execution might be backed off to ensure error cap is not exceeded too fast.
-//! The sync queue processing also happens in batches, so the sync tasks can wait in the queue for some time.
-
-mod local_fs;
-mod rust_s3;
-mod storage_sync;
-
-use std::{
-    collections::HashMap,
-    ffi, fs,
-    path::{Path, PathBuf},
-    thread,
-};
-
-use anyhow::{bail, Context};
-use tokio::io;
-use tracing::{error, info};
-use zenith_utils::zid::{ZTenantId, ZTimelineId};
-
-pub use self::storage_sync::{schedule_timeline_checkpoint_upload, schedule_timeline_download};
-use self::{local_fs::LocalFs, rust_s3::S3};
-use crate::{
-    config::{PageServerConf, RemoteStorageKind},
-    layered_repository::metadata::{TimelineMetadata, METADATA_FILE_NAME},
-    repository::TimelineSyncState,
-};
-
-/// Any timeline has its own id and its own tenant it belongs to,
-/// the sync processes group timelines by both for simplicity.
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)]
-pub struct TimelineSyncId(ZTenantId, ZTimelineId);
-
-impl std::fmt::Display for TimelineSyncId {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "(tenant: {}, timeline: {})", self.0, self.1)
-    }
-}
-
-/// A structure to combine all synchronization data to share with pageserver after a successful sync loop initialization.
-/// Successful initialization includes a case when sync loop is not started, in which case the startup data is returned still,
-/// to simplify the received code.
-pub struct SyncStartupData {
-    /// A sync state, derived from initial comparison of local timeline files and the remote archives,
-    /// before any sync tasks are executed.
-    /// To reuse the local file scan logic, the timeline states are returned even if no sync loop get started during init:
-    /// in this case, no remote files exist and all local timelines with correct metadata files are considered ready.
-    pub initial_timeline_states: HashMap<ZTenantId, HashMap<ZTimelineId, TimelineSyncState>>,
-    /// A handle to the sync loop, if it was started from the configuration provided.
-    pub sync_loop_handle: Option<thread::JoinHandle<anyhow::Result<()>>>,
-}
-
-/// Based on the config, initiates the remote storage connection and starts a separate thread
-/// that ensures that pageserver and the remote storage are in sync with each other.
-/// If no external configuration connection given, no thread or storage initialization is done.
-/// Along with that, scans tenant files local and remote (if the sync gets enabled) to check the initial timeline states.
-pub fn start_local_timeline_sync(
-    config: &'static PageServerConf,
-) -> anyhow::Result<SyncStartupData> {
-    let local_timeline_files = local_tenant_timeline_files(config)
-        .context("Failed to collect local tenant timeline files")?;
-
-    match &config.remote_storage_config {
-        Some(storage_config) => match &storage_config.storage {
-            RemoteStorageKind::LocalFs(root) => storage_sync::spawn_storage_sync_thread(
-                config,
-                local_timeline_files,
-                LocalFs::new(root.clone(), &config.workdir)?,
-                storage_config.max_concurrent_sync,
-                storage_config.max_sync_errors,
-            ),
-            RemoteStorageKind::AwsS3(s3_config) => storage_sync::spawn_storage_sync_thread(
-                config,
-                local_timeline_files,
-                S3::new(s3_config, &config.workdir)?,
-                storage_config.max_concurrent_sync,
-                storage_config.max_sync_errors,
-            ),
-        }
-        .context("Failed to spawn the storage sync thread"),
-        None => {
-            info!("No remote storage configured, skipping storage sync, considering all local timelines with correct metadata files enabled");
-            let mut initial_timeline_states: HashMap<
-                ZTenantId,
-                HashMap<ZTimelineId, TimelineSyncState>,
-            > = HashMap::new();
-            for (TimelineSyncId(tenant_id, timeline_id), (timeline_metadata, _)) in
-                local_timeline_files
-            {
-                initial_timeline_states
-                    .entry(tenant_id)
-                    .or_default()
-                    .insert(
-                        timeline_id,
-                        TimelineSyncState::Ready(timeline_metadata.disk_consistent_lsn()),
-                    );
-            }
-            Ok(SyncStartupData {
-                initial_timeline_states,
-                sync_loop_handle: None,
-            })
-        }
-    }
-}
-
-fn local_tenant_timeline_files(
-    config: &'static PageServerConf,
-) -> anyhow::Result<HashMap<TimelineSyncId, (TimelineMetadata, Vec<PathBuf>)>> {
-    let mut local_tenant_timeline_files = HashMap::new();
-    let tenants_dir = config.tenants_path();
-    for tenants_dir_entry in fs::read_dir(&tenants_dir)
-        .with_context(|| format!("Failed to list tenants dir {}", tenants_dir.display()))?
-    {
-        match &tenants_dir_entry {
-            Ok(tenants_dir_entry) => {
-                match collect_timelines_for_tenant(config, &tenants_dir_entry.path()) {
-                    Ok(collected_files) => {
-                        local_tenant_timeline_files.extend(collected_files.into_iter())
-                    }
-                    Err(e) => error!(
-                        "Failed to collect tenant files from dir '{}' for entry {:?}, reason: {:#}",
-                        tenants_dir.display(),
-                        tenants_dir_entry,
-                        e
-                    ),
-                }
-            }
-            Err(e) => error!(
-                "Failed to list tenants dir entry {:?} in directory {}, reason: {:#}",
-                tenants_dir_entry,
-                tenants_dir.display(),
-                e
-            ),
-        }
-    }
-
-    Ok(local_tenant_timeline_files)
-}
-
-fn collect_timelines_for_tenant(
-    config: &'static PageServerConf,
-    tenant_path: &Path,
-) -> anyhow::Result<HashMap<TimelineSyncId, (TimelineMetadata, Vec<PathBuf>)>> {
-    let mut timelines: HashMap<TimelineSyncId, (TimelineMetadata, Vec<PathBuf>)> = HashMap::new();
-    let tenant_id = tenant_path
-        .file_name()
-        .and_then(ffi::OsStr::to_str)
-        .unwrap_or_default()
-        .parse::<ZTenantId>()
-        .context("Could not parse tenant id out of the tenant dir name")?;
-    let timelines_dir = config.timelines_path(&tenant_id);
-
-    for timelines_dir_entry in fs::read_dir(&timelines_dir).with_context(|| {
-        format!(
-            "Failed to list timelines dir entry for tenant {}",
-            tenant_id
-        )
-    })? {
-        match timelines_dir_entry {
-            Ok(timelines_dir_entry) => {
-                let timeline_path = timelines_dir_entry.path();
-                match collect_timeline_files(&timeline_path) {
-                    Ok((timeline_id, metadata, timeline_files)) => {
-                        timelines.insert(
-                            TimelineSyncId(tenant_id, timeline_id),
-                            (metadata, timeline_files),
-                        );
-                    }
-                    Err(e) => error!(
-                        "Failed to process timeline dir contents at '{}', reason: {:#}",
-                        timeline_path.display(),
-                        e
-                    ),
-                }
-            }
-            Err(e) => error!(
-                "Failed to list timelines for entry tenant {}, reason: {:#}",
-                tenant_id, e
-            ),
-        }
-    }
-
-    Ok(timelines)
-}
-
-fn collect_timeline_files(
-    timeline_dir: &Path,
-) -> anyhow::Result<(ZTimelineId, TimelineMetadata, Vec<PathBuf>)> {
-    let mut timeline_files = Vec::new();
-    let mut timeline_metadata_path = None;
-
-    let timeline_id = timeline_dir
-        .file_name()
-        .and_then(ffi::OsStr::to_str)
-        .unwrap_or_default()
-        .parse::<ZTimelineId>()
-        .context("Could not parse timeline id out of the timeline dir name")?;
-    let timeline_dir_entries =
-        fs::read_dir(&timeline_dir).context("Failed to list timeline dir contents")?;
-    for entry in timeline_dir_entries {
-        let entry_path = entry.context("Failed to list timeline dir entry")?.path();
-        if entry_path.is_file() {
-            if entry_path.file_name().and_then(ffi::OsStr::to_str) == Some(METADATA_FILE_NAME) {
-                timeline_metadata_path = Some(entry_path);
-            } else {
-                timeline_files.push(entry_path);
-            }
-        }
-    }
-
-    let timeline_metadata_path = match timeline_metadata_path {
-        Some(path) => path,
-        None => bail!("No metadata file found in the timeline directory"),
-    };
-    let metadata = TimelineMetadata::from_bytes(
-        &fs::read(&timeline_metadata_path).context("Failed to read timeline metadata file")?,
-    )
-    .context("Failed to parse timeline metadata file bytes")?;
-
-    Ok((timeline_id, metadata, timeline_files))
-}
-
-/// Storage (potentially remote) API to manage its state.
-/// This storage tries to be unaware of any layered repository context,
-/// providing basic CRUD operations for storage files.
-#[async_trait::async_trait]
-trait RemoteStorage: Send + Sync {
-    /// A way to uniquely reference a file in the remote storage.
-    type StoragePath;
-
-    /// Attempts to derive the storage path out of the local path, if the latter is correct.
-    fn storage_path(&self, local_path: &Path) -> anyhow::Result<Self::StoragePath>;
-
-    /// Gets the download path of the given storage file.
-    fn local_path(&self, storage_path: &Self::StoragePath) -> anyhow::Result<PathBuf>;
-
-    /// Lists all items the storage has right now.
-    async fn list(&self) -> anyhow::Result<Vec<Self::StoragePath>>;
-
-    /// Streams the local file contents into remote into the remote storage entry.
-    async fn upload(
-        &self,
-        from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
-        to: &Self::StoragePath,
-    ) -> anyhow::Result<()>;
-
-    /// Streams the remote storage entry contents into the buffered writer given, returns the filled writer.
-    async fn download(
-        &self,
-        from: &Self::StoragePath,
-        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<()>;
-
-    /// Streams a given byte range of the remote storage entry contents into the buffered writer given, returns the filled writer.
-    async fn download_range(
-        &self,
-        from: &Self::StoragePath,
-        start_inclusive: u64,
-        end_exclusive: Option<u64>,
-        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<()>;
-
-    async fn delete(&self, path: &Self::StoragePath) -> anyhow::Result<()>;
-}
-
-fn strip_path_prefix<'a>(prefix: &'a Path, path: &'a Path) -> anyhow::Result<&'a Path> {
-    if prefix == path {
-        anyhow::bail!(
-            "Prefix and the path are equal, cannot strip: '{}'",
-            prefix.display()
-        )
-    } else {
-        path.strip_prefix(prefix).with_context(|| {
-            format!(
-                "Path '{}' is not prefixed with '{}'",
-                path.display(),
-                prefix.display(),
-            )
-        })
-    }
-}

pageserver/src/remote_storage/README.md

@@ -1,77 +0,0 @@
-# Non-implementation details
-
-This document describes the current state of the backup system in pageserver, existing limitations and concerns, why some things are done the way they are the future development plans.
-Detailed description on how the synchronization works and how it fits into the rest of the pageserver can be found in the [storage module](./../remote_storage.rs) and its submodules.
-Ideally, this document should disappear after current implementation concerns are mitigated, with the remaining useful knowledge bits moved into rustdocs.
-
-## Approach
-
-Backup functionality is a new component, appeared way after the core DB functionality was implemented.
-Pageserver layer functionality is also quite volatile at the moment, there's a risk its local file management changes over time.
-
-To avoid adding more chaos into that, backup functionality is currently designed as a relatively standalone component, with the majority of its logic placed in a standalone async loop.
-This way, the backups are managed in background, not affecting directly other pageserver parts: this way the backup and restoration process may lag behind, but eventually keep up with the reality. To track that, a set of prometheus metrics is exposed from pageserver.
-
-## What's done
-
-Current implementation
-* provides remote storage wrappers for AWS S3 and local FS
-* synchronizes the differences with local timelines and remote states as fast as possible
-* uploads new relishes, frozen by pageserver checkpoint thread
-* downloads and registers timelines, found on the remote storage, but missing locally, if those are requested somehow via pageserver (e.g. http api, gc)
-* uses compression when deals with files, for better S3 usage
-* maintains an index of what's stored remotely
-* evicts failing tasks and stops the corresponding timelines
-
-The tasks are delayed with every retry and the retries are capped, to avoid poisonous tasks.
-After any task eviction, or any error at startup checks (e.g. obviously different and wrong local and remote states fot the same timeline),
-the timeline has to be stopped from submitting further checkpoint upload tasks, which is done along the corresponding timeline status change.
-
-No good optimisations or performance testing is done, the feature is disabled by default and gets polished over time.
-It's planned to deal with all questions that are currently on and prepare the feature to be enabled by default in cloud environments.
-
-### Peculiarities
-
-As mentioned, the backup component is rather new and under development currently, so not all things are done properly from the start.
-Here's the list of known compromises with comments:
-
-* Remote storage file model is currently a custom archive format, that's not possible to deserialize without a particular Rust code of ours (including `serde`).
-We also don't optimize the archivation and pack every timeline checkpoint separately, so the resulting blob's size that gets on S3 could be arbitrary.
-But, it's a single blob, which is way better than storing ~780 small files separately.
-
-* Archive index restoration requires reading every blob's head.
-This could be avoided by a background thread/future storing the serialized index in the remote storage.
-
-* no proper file comparison
-
-No file checksum assertion is done currently, but should be (AWS S3 returns file checksums during the `list` operation)
-
-* sad rust-s3 api
-
-rust-s3 is not very pleasant to use:
-1. it returns `anyhow::Result` and it's hard to distinguish "missing file" cases from "no connection" one, for instance
-2. at least one function it its API that we need (`get_object_stream`) has `async` keyword and blocks (!), see details [here](https://github.com/zenithdb/zenith/pull/752#discussion_r728373091)
-3. it's a prerelease library with unclear maintenance status
-4. noisy on debug level
-
-But it's already used in the project, so for now it's reused to avoid bloating the dependency tree.
-Based on previous evaluation, even `rusoto-s3` could be a better choice over this library, but needs further benchmarking.
-
-
-* gc is ignored
-
-So far, we don't adjust the remote storage based on GC thread loop results, only checkpointer loop affects the remote storage.
-Index module could be used as a base to implement a deferred GC mechanism, a "defragmentation" that repacks archives into new ones after GC is done removing the files from the archives.
-
-* bracnhes implementaion could be improved
-
-Currently, there's a code to sync the branches along with the timeline files: on upload, every local branch files that are missing remotely are uploaded,
-on the timeline download, missing remote branch files are downlaoded.
-
-A branch is a per-tenant entity, yet a current implementaion requires synchronizing a timeline first to get the branch files locally.
-Currently, there's no other way to know about the remote branch files, neither the file contents is verified and updated.
-
-* no IT tests
-
-Automated S3 testing is lacking currently, due to no convenient way to enable backups during the tests.
-After it's fixed, benchmark runs should also be carried out to find bottlenecks.

pageserver/src/remote_storage/storage_sync/compression.rs

@@ -1,611 +0,0 @@
-//! A set of structs to represent a compressed part of the timeline, and methods to asynchronously compress and uncompress a stream of data,
-//! without holding the entire data in memory.
-//! For the latter, both compress and uncompress functions operate buffered streams (currently hardcoded size of [`ARCHIVE_STREAM_BUFFER_SIZE_BYTES`]),
-//! not attempting to hold the entire archive in memory.
-//!
-//! The compression is done with <a href="https://datatracker.ietf.org/doc/html/rfc8878">zstd</a> streaming algorithm via the `async-compression` crate.
-//! The crate does not contain any knobs to tweak the compression, but otherwise is one of the only ones that's both async and has an API to manage the part of an archive.
-//! Zstd was picked as the best algorithm among the ones available in the crate, after testing the initial timeline file compression.
-//!
-//! Archiving is almost agnostic to timeline file types, with an exception of the metadata file, that's currently distinguished in the [un]compression code.
-//! The metadata file is treated separately when [de]compression is involved, to reduce the risk of corrupting the metadata file.
-//! When compressed, the metadata file is always required and stored as the last file in the archive stream.
-//! When uncompressed, the metadata file gets naturally uncompressed last, to ensure that all other relishes are decompressed successfully first.
-//!
-//! Archive structure:
-//! +----------------------------------------+
-//! | header | file_1, ..., file_k, metadata |
-//! +----------------------------------------+
-//!
-//! The archive consists of two separate zstd archives:
-//! * header archive, that contains all files names and their sizes and relative paths in the timeline directory
-//! Header is a Rust structure, serialized into bytes and compressed with zstd.
-//! * files archive, that has metadata file as the last one, all compressed with zstd into a single binary blob
-//!
-//! Header offset is stored in the file name, along with the `disk_consistent_lsn` from the metadata file.
-//! See [`parse_archive_name`] and [`ARCHIVE_EXTENSION`] for the name details, example: `00000000016B9150-.zst_9732`.
-//! This way, the header could be retrieved without reading an entire archive file.
-
-use std::{
-    collections::BTreeSet,
-    future::Future,
-    io::Cursor,
-    path::{Path, PathBuf},
-    sync::Arc,
-};
-
-use anyhow::{anyhow, bail, ensure, Context};
-use async_compression::tokio::bufread::{ZstdDecoder, ZstdEncoder};
-use serde::{Deserialize, Serialize};
-use tokio::{
-    fs,
-    io::{self, AsyncReadExt, AsyncWriteExt},
-};
-use tracing::*;
-use zenith_utils::{bin_ser::BeSer, lsn::Lsn};
-
-use crate::layered_repository::metadata::{TimelineMetadata, METADATA_FILE_NAME};
-
-use super::index::RelativePath;
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct ArchiveHeader {
-    /// All regular timeline files, excluding the metadata file.
-    pub files: Vec<FileEntry>,
-    // Metadata file name is known to the system, as its location relative to the timeline dir,
-    // so no need to store anything but its size in bytes.
-    pub metadata_file_size: u64,
-}
-
-#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
-pub struct FileEntry {
-    /// Uncompressed file size, bytes.
-    pub size: u64,
-    /// A path, relative to the directory root, used when compressing the directory contents.
-    pub subpath: RelativePath,
-}
-
-const ARCHIVE_EXTENSION: &str = "-.zst_";
-const ARCHIVE_STREAM_BUFFER_SIZE_BYTES: usize = 4 * 1024 * 1024;
-
-/// Streams an archive of files given into a stream target, defined by the closure.
-///
-/// The closure approach is picked for cases like S3, where we would need a name of the file before we can get a stream to write the bytes into.
-/// Current idea is to place the header size in the name of the file, to enable the fast partial remote file index restoration without actually reading remote storage file contents.
-///
-/// Performs the compression in multiple steps:
-/// * prepares an archive header, stripping the `source_dir` prefix from the `files`
-/// * generates the name of the archive
-/// * prepares archive producer future, knowing the header and the file list
-/// An `impl AsyncRead` and `impl AsyncWrite` pair of connected streams is created to implement the partial contents streaming.
-/// The writer end gets into the archive producer future, to put the header and a stream of compressed files.
-/// * prepares archive consumer future, by executing the provided closure
-/// The closure gets the reader end stream and the name of the file to create a future that would stream the file contents elsewhere.
-/// * runs and waits for both futures to complete
-/// * on a successful completion of both futures, header, its size and the user-defined consumer future return data is returned
-/// Due to the design above, the archive name and related data is visible inside the consumer future only, so it's possible to return the data,
-/// needed for future processing.
-pub async fn archive_files_as_stream<Cons, ConsRet, Fut>(
-    source_dir: &Path,
-    files: impl Iterator<Item = &PathBuf>,
-    metadata: &TimelineMetadata,
-    create_archive_consumer: Cons,
-) -> anyhow::Result<(ArchiveHeader, u64, ConsRet)>
-where
-    Cons: FnOnce(Box<dyn io::AsyncRead + Unpin + Send + Sync + 'static>, String) -> Fut
-        + Send
-        + 'static,
-    Fut: Future<Output = anyhow::Result<ConsRet>> + Send + 'static,
-    ConsRet: Send + Sync + 'static,
-{
-    let metadata_bytes = metadata
-        .to_bytes()
-        .context("Failed to create metadata bytes")?;
-    let (archive_header, compressed_header_bytes) =
-        prepare_header(source_dir, files, &metadata_bytes)
-            .await
-            .context("Failed to prepare file for archivation")?;
-
-    let header_size = compressed_header_bytes.len() as u64;
-    let (write, read) = io::duplex(ARCHIVE_STREAM_BUFFER_SIZE_BYTES);
-    let archive_filler = write_archive_contents(
-        source_dir.to_path_buf(),
-        archive_header.clone(),
-        metadata_bytes,
-        write,
-    );
-    let archive_name = archive_name(metadata.disk_consistent_lsn(), header_size);
-    let archive_stream =
-        Cursor::new(compressed_header_bytes).chain(ZstdEncoder::new(io::BufReader::new(read)));
-
-    let (archive_creation_result, archive_upload_result) = tokio::join!(
-        tokio::spawn(archive_filler),
-        tokio::spawn(async move {
-            create_archive_consumer(Box::new(archive_stream), archive_name).await
-        })
-    );
-    archive_creation_result
-        .context("Failed to spawn archive creation future")?
-        .context("Failed to create an archive")?;
-    let upload_return_value = archive_upload_result
-        .context("Failed to spawn archive upload future")?
-        .context("Failed to upload the archive")?;
-
-    Ok((archive_header, header_size, upload_return_value))
-}
-
-/// Similar to [`archive_files_as_stream`], creates a pair of streams to uncompress the 2nd part of the archive,
-/// that contains files and is located after the header.
-/// S3 allows downloading partial file contents for a given file key (i.e. name), to accommodate this retrieval,
-/// a closure is used.
-/// Same concepts with two concurrent futures, user-defined closure, future and return value apply here, but the
-/// consumer and the receiver ends are swapped, since the uncompression happens.
-pub async fn uncompress_file_stream_with_index<Prod, ProdRet, Fut>(
-    destination_dir: PathBuf,
-    files_to_skip: Arc<BTreeSet<PathBuf>>,
-    disk_consistent_lsn: Lsn,
-    header: ArchiveHeader,
-    header_size: u64,
-    create_archive_file_part: Prod,
-) -> anyhow::Result<ProdRet>
-where
-    Prod: FnOnce(Box<dyn io::AsyncWrite + Unpin + Send + Sync + 'static>, String) -> Fut
-        + Send
-        + 'static,
-    Fut: Future<Output = anyhow::Result<ProdRet>> + Send + 'static,
-    ProdRet: Send + Sync + 'static,
-{
-    let (write, mut read) = io::duplex(ARCHIVE_STREAM_BUFFER_SIZE_BYTES);
-    let archive_name = archive_name(disk_consistent_lsn, header_size);
-
-    let (archive_download_result, archive_uncompress_result) = tokio::join!(
-        tokio::spawn(async move { create_archive_file_part(Box::new(write), archive_name).await }),
-        tokio::spawn(async move {
-            uncompress_with_header(&files_to_skip, &destination_dir, header, &mut read).await
-        })
-    );
-
-    let download_value = archive_download_result
-        .context("Failed to spawn archive download future")?
-        .context("Failed to download an archive")?;
-    archive_uncompress_result
-        .context("Failed to spawn archive uncompress future")?
-        .context("Failed to uncompress the archive")?;
-
-    Ok(download_value)
-}
-
-/// Reads archive header from the stream given:
-/// * parses the file name to get the header size
-/// * reads the exact amount of bytes
-/// * uncompresses and deserializes those
-pub async fn read_archive_header<A: io::AsyncRead + Send + Sync + Unpin>(
-    archive_name: &str,
-    from: &mut A,
-) -> anyhow::Result<ArchiveHeader> {
-    let (_, header_size) = parse_archive_name(Path::new(archive_name))?;
-
-    let mut compressed_header_bytes = vec![0; header_size as usize];
-    from.read_exact(&mut compressed_header_bytes)
-        .await
-        .with_context(|| {
-            format!(
-                "Failed to read header header from the archive {}",
-                archive_name
-            )
-        })?;
-
-    let mut header_bytes = Vec::new();
-    ZstdDecoder::new(io::BufReader::new(compressed_header_bytes.as_slice()))
-        .read_to_end(&mut header_bytes)
-        .await
-        .context("Failed to decompress a header from the archive")?;
-
-    Ok(ArchiveHeader::des(&header_bytes)
-        .context("Failed to deserialize a header from the archive")?)
-}
-
-/// Reads the archive metadata out of the archive name:
-/// * `disk_consistent_lsn` of the checkpoint that was archived
-/// * size of the archive header
-pub fn parse_archive_name(archive_path: &Path) -> anyhow::Result<(Lsn, u64)> {
-    let archive_name = archive_path
-        .file_name()
-        .ok_or_else(|| anyhow!("Archive '{}' has no file name", archive_path.display()))?
-        .to_string_lossy();
-    let (lsn_str, header_size_str) =
-        archive_name.rsplit_once(ARCHIVE_EXTENSION).ok_or_else(|| {
-            anyhow!(
-                "Archive '{}' has incorrect extension, expected to contain '{}'",
-                archive_path.display(),
-                ARCHIVE_EXTENSION
-            )
-        })?;
-    let disk_consistent_lsn = Lsn::from_hex(lsn_str).with_context(|| {
-        format!(
-            "Archive '{}' has an invalid disk consistent lsn in its extension",
-            archive_path.display(),
-        )
-    })?;
-    let header_size = header_size_str.parse::<u64>().with_context(|| {
-        format!(
-            "Archive '{}' has an invalid a header offset number in its extension",
-            archive_path.display(),
-        )
-    })?;
-    Ok((disk_consistent_lsn, header_size))
-}
-
-fn archive_name(disk_consistent_lsn: Lsn, header_size: u64) -> String {
-    let archive_name = format!(
-        "{:016X}{ARCHIVE_EXTENSION}{}",
-        u64::from(disk_consistent_lsn),
-        header_size,
-        ARCHIVE_EXTENSION = ARCHIVE_EXTENSION,
-    );
-    archive_name
-}
-
-async fn uncompress_with_header(
-    files_to_skip: &BTreeSet<PathBuf>,
-    destination_dir: &Path,
-    header: ArchiveHeader,
-    archive_after_header: impl io::AsyncRead + Send + Sync + Unpin,
-) -> anyhow::Result<()> {
-    debug!("Uncompressing archive into {}", destination_dir.display());
-    let mut archive = ZstdDecoder::new(io::BufReader::new(archive_after_header));
-
-    if !destination_dir.exists() {
-        fs::create_dir_all(&destination_dir)
-            .await
-            .with_context(|| {
-                format!(
-                    "Failed to create target directory at {}",
-                    destination_dir.display()
-                )
-            })?;
-    } else if !destination_dir.is_dir() {
-        bail!(
-            "Destination path '{}' is not a valid directory",
-            destination_dir.display()
-        );
-    }
-    debug!("Will extract {} files from the archive", header.files.len());
-    for entry in header.files {
-        uncompress_entry(
-            &mut archive,
-            &entry.subpath.as_path(destination_dir),
-            entry.size,
-            files_to_skip,
-        )
-        .await
-        .with_context(|| format!("Failed to uncompress archive entry {:?}", entry))?;
-    }
-    uncompress_entry(
-        &mut archive,
-        &destination_dir.join(METADATA_FILE_NAME),
-        header.metadata_file_size,
-        files_to_skip,
-    )
-    .await
-    .context("Failed to uncompress the metadata entry")?;
-    Ok(())
-}
-
-async fn uncompress_entry(
-    archive: &mut ZstdDecoder<io::BufReader<impl io::AsyncRead + Send + Sync + Unpin>>,
-    destination_path: &Path,
-    entry_size: u64,
-    files_to_skip: &BTreeSet<PathBuf>,
-) -> anyhow::Result<()> {
-    if let Some(parent) = destination_path.parent() {
-        fs::create_dir_all(parent).await.with_context(|| {
-            format!(
-                "Failed to create parent directory for {}",
-                destination_path.display()
-            )
-        })?;
-    };
-
-    if files_to_skip.contains(destination_path) {
-        debug!("Skipping {}", destination_path.display());
-        copy_n_bytes(entry_size, archive, &mut io::sink())
-            .await
-            .context("Failed to skip bytes in the archive")?;
-        return Ok(());
-    }
-
-    let mut destination =
-        io::BufWriter::new(fs::File::create(&destination_path).await.with_context(|| {
-            format!(
-                "Failed to open file {} for extraction",
-                destination_path.display()
-            )
-        })?);
-    copy_n_bytes(entry_size, archive, &mut destination)
-        .await
-        .with_context(|| {
-            format!(
-                "Failed to write extracted archive contents into file {}",
-                destination_path.display()
-            )
-        })?;
-    destination
-        .flush()
-        .await
-        .context("Failed to flush the streaming archive bytes")?;
-    Ok(())
-}
-
-async fn write_archive_contents(
-    source_dir: PathBuf,
-    header: ArchiveHeader,
-    metadata_bytes: Vec<u8>,
-    mut archive_input: io::DuplexStream,
-) -> anyhow::Result<()> {
-    debug!("Starting writing files into archive");
-    for file_entry in header.files {
-        let path = file_entry.subpath.as_path(&source_dir);
-        let mut source_file =
-            io::BufReader::new(fs::File::open(&path).await.with_context(|| {
-                format!(
-                    "Failed to open file for archiving to path {}",
-                    path.display()
-                )
-            })?);
-        let bytes_written = io::copy(&mut source_file, &mut archive_input)
-            .await
-            .with_context(|| {
-                format!(
-                    "Failed to open add a file into archive, file path {}",
-                    path.display()
-                )
-            })?;
-        ensure!(
-            file_entry.size == bytes_written,
-            "File {} was written to the archive incompletely",
-            path.display()
-        );
-        trace!(
-            "Added file '{}' ({} bytes) into the archive",
-            path.display(),
-            bytes_written
-        );
-    }
-    let metadata_bytes_written = io::copy(&mut metadata_bytes.as_slice(), &mut archive_input)
-        .await
-        .with_context(|| "Failed to add metadata into the archive")?;
-    ensure!(
-        header.metadata_file_size == metadata_bytes_written,
-        "Metadata file was written to the archive incompletely",
-    );
-
-    archive_input
-        .shutdown()
-        .await
-        .context("Failed to finalize the archive")?;
-    debug!("Successfully streamed all files into the archive");
-    Ok(())
-}
-
-async fn prepare_header(
-    source_dir: &Path,
-    files: impl Iterator<Item = &PathBuf>,
-    metadata_bytes: &[u8],
-) -> anyhow::Result<(ArchiveHeader, Vec<u8>)> {
-    let mut archive_files = Vec::new();
-    for file_path in files {
-        let file_metadata = fs::metadata(file_path).await.with_context(|| {
-            format!(
-                "Failed to read metadata during archive indexing for {}",
-                file_path.display()
-            )
-        })?;
-        ensure!(
-            file_metadata.is_file(),
-            "Archive indexed path {} is not a file",
-            file_path.display()
-        );
-
-        if file_path.file_name().and_then(|name| name.to_str()) != Some(METADATA_FILE_NAME) {
-            let entry = FileEntry {
-                subpath: RelativePath::new(source_dir, file_path).with_context(|| {
-                    format!(
-                        "File '{}' does not belong to pageserver workspace",
-                        file_path.display()
-                    )
-                })?,
-                size: file_metadata.len(),
-            };
-            archive_files.push(entry);
-        }
-    }
-
-    let header = ArchiveHeader {
-        files: archive_files,
-        metadata_file_size: metadata_bytes.len() as u64,
-    };
-
-    debug!("Appending a header for {} files", header.files.len());
-    let header_bytes = header.ser().context("Failed to serialize a header")?;
-    debug!("Header bytes len {}", header_bytes.len());
-    let mut compressed_header_bytes = Vec::new();
-    ZstdEncoder::new(io::BufReader::new(header_bytes.as_slice()))
-        .read_to_end(&mut compressed_header_bytes)
-        .await
-        .context("Failed to compress header bytes")?;
-    debug!(
-        "Compressed header bytes len {}",
-        compressed_header_bytes.len()
-    );
-    Ok((header, compressed_header_bytes))
-}
-
-async fn copy_n_bytes(
-    n: u64,
-    from: &mut (impl io::AsyncRead + Send + Sync + Unpin),
-    into: &mut (impl io::AsyncWrite + Send + Sync + Unpin),
-) -> anyhow::Result<()> {
-    let bytes_written = io::copy(&mut from.take(n), into).await?;
-    ensure!(
-        bytes_written == n,
-        "Failed to read exactly {} bytes from the input, bytes written: {}",
-        n,
-        bytes_written,
-    );
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use tokio::{fs, io::AsyncSeekExt};
-
-    use crate::repository::repo_harness::{RepoHarness, TIMELINE_ID};
-
-    use super::*;
-
-    #[tokio::test]
-    async fn compress_and_uncompress() -> anyhow::Result<()> {
-        let repo_harness = RepoHarness::create("compress_and_uncompress")?;
-        let timeline_dir = repo_harness.timeline_path(&TIMELINE_ID);
-        init_directory(
-            &timeline_dir,
-            vec![
-                ("first", "first_contents"),
-                ("second", "second_contents"),
-                (METADATA_FILE_NAME, "wrong_metadata"),
-            ],
-        )
-        .await?;
-        let timeline_files = list_file_paths_with_contents(&timeline_dir).await?;
-        assert_eq!(
-            timeline_files,
-            vec![
-                (
-                    timeline_dir.join("first"),
-                    FileContents::Text("first_contents".to_string())
-                ),
-                (
-                    timeline_dir.join(METADATA_FILE_NAME),
-                    FileContents::Text("wrong_metadata".to_string())
-                ),
-                (
-                    timeline_dir.join("second"),
-                    FileContents::Text("second_contents".to_string())
-                ),
-            ],
-            "Initial timeline contents should contain two normal files and a wrong metadata file"
-        );
-
-        let metadata = TimelineMetadata::new(Lsn(0x30), None, None, Lsn(0), Lsn(0), Lsn(0));
-        let paths_to_archive = timeline_files
-            .into_iter()
-            .map(|(path, _)| path)
-            .collect::<Vec<_>>();
-
-        let tempdir = tempfile::tempdir()?;
-        let base_path = tempdir.path().to_path_buf();
-        let (header, header_size, archive_target) = archive_files_as_stream(
-            &timeline_dir,
-            paths_to_archive.iter(),
-            &metadata,
-            move |mut archive_streamer, archive_name| async move {
-                let archive_target = base_path.join(&archive_name);
-                let mut archive_file = fs::File::create(&archive_target).await?;
-                io::copy(&mut archive_streamer, &mut archive_file).await?;
-                Ok(archive_target)
-            },
-        )
-        .await?;
-
-        let mut file = fs::File::open(&archive_target).await?;
-        file.seek(io::SeekFrom::Start(header_size)).await?;
-        let target_dir = tempdir.path().join("extracted");
-        uncompress_with_header(&BTreeSet::new(), &target_dir, header, file).await?;
-
-        let extracted_files = list_file_paths_with_contents(&target_dir).await?;
-
-        assert_eq!(
-            extracted_files,
-            vec![
-                (
-                    target_dir.join("first"),
-                    FileContents::Text("first_contents".to_string())
-                ),
-                (
-                    target_dir.join(METADATA_FILE_NAME),
-                    FileContents::Binary(metadata.to_bytes()?)
-                ),
-                (
-                    target_dir.join("second"),
-                    FileContents::Text("second_contents".to_string())
-                ),
-            ],
-            "Extracted files should contain all local timeline files besides its metadata, which should be taken from the arguments"
-        );
-
-        Ok(())
-    }
-
-    async fn init_directory(
-        root: &Path,
-        files_with_contents: Vec<(&str, &str)>,
-    ) -> anyhow::Result<()> {
-        fs::create_dir_all(root).await?;
-        for (file_name, contents) in files_with_contents {
-            fs::File::create(root.join(file_name))
-                .await?
-                .write_all(contents.as_bytes())
-                .await?;
-        }
-        Ok(())
-    }
-
-    #[derive(PartialEq, Eq, PartialOrd, Ord)]
-    enum FileContents {
-        Text(String),
-        Binary(Vec<u8>),
-    }
-
-    impl std::fmt::Debug for FileContents {
-        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-            match self {
-                Self::Text(text) => f.debug_tuple("Text").field(text).finish(),
-                Self::Binary(bytes) => f
-                    .debug_tuple("Binary")
-                    .field(&format!("{} bytes", bytes.len()))
-                    .finish(),
-            }
-        }
-    }
-
-    async fn list_file_paths_with_contents(
-        root: &Path,
-    ) -> anyhow::Result<Vec<(PathBuf, FileContents)>> {
-        let mut file_paths = Vec::new();
-
-        let mut dir_listings = vec![fs::read_dir(root).await?];
-        while let Some(mut dir_listing) = dir_listings.pop() {
-            while let Some(entry) = dir_listing.next_entry().await? {
-                let entry_path = entry.path();
-                if entry_path.is_file() {
-                    let contents = match String::from_utf8(fs::read(&entry_path).await?) {
-                        Ok(text) => FileContents::Text(text),
-                        Err(e) => FileContents::Binary(e.into_bytes()),
-                    };
-                    file_paths.push((entry_path, contents));
-                } else if entry_path.is_dir() {
-                    dir_listings.push(fs::read_dir(entry_path).await?);
-                } else {
-                    info!(
-                        "Skipping path '{}' as it's not a file or a directory",
-                        entry_path.display()
-                    );
-                }
-            }
-        }
-
-        file_paths.sort();
-        Ok(file_paths)
-    }
-}

pageserver/src/remote_storage/storage_sync/download.rs

@@ -1,421 +0,0 @@
-//! Timeline synchrnonization logic to put files from archives on remote storage into pageserver's local directory.
-//! Currently, tenant branch files are also downloaded, but this does not appear final.
-
-use std::{borrow::Cow, collections::BTreeSet, path::PathBuf, sync::Arc};
-
-use anyhow::{anyhow, ensure, Context};
-use futures::{stream::FuturesUnordered, StreamExt};
-use tokio::{fs, sync::RwLock};
-use tracing::{debug, error, trace, warn};
-use zenith_utils::{lsn::Lsn, zid::ZTenantId};
-
-use crate::{
-    config::PageServerConf,
-    layered_repository::metadata::{metadata_path, TimelineMetadata},
-    remote_storage::{
-        storage_sync::{
-            compression, index::TimelineIndexEntry, sync_queue, tenant_branch_files,
-            update_index_description, SyncKind, SyncTask,
-        },
-        RemoteStorage, TimelineSyncId,
-    },
-};
-
-use super::{
-    index::{ArchiveId, RemoteTimeline, RemoteTimelineIndex},
-    TimelineDownload,
-};
-
-/// Timeline download result, with extra data, needed for downloading.
-pub(super) enum DownloadedTimeline {
-    /// Remote timeline data is either absent or corrupt, no download possible.
-    Abort,
-    /// Remote timeline data is found, its latest checkpoint's metadata contents (disk_consistent_lsn) is known.
-    /// Initial download failed due to some error, the download task is rescheduled for another retry.
-    FailedAndRescheduled { disk_consistent_lsn: Lsn },
-    /// Remote timeline data is found, its latest checkpoint's metadata contents (disk_consistent_lsn) is known.
-    /// Initial download successful.
-    Successful { disk_consistent_lsn: Lsn },
-}
-
-/// Attempts to download and uncompress files from all remote archives for the timeline given.
-/// Timeline files that already exist locally are skipped during the download, but the local metadata file is
-/// updated in the end of every checkpoint archive extraction.
-///
-/// Before any archives are considered, the branch files are checked locally and remotely, all remote-only files are downloaded.
-///
-/// On an error, bumps the retries count and reschedules the download, with updated archive skip list
-/// (for any new successful archive downloads and extractions).
-pub(super) async fn download_timeline<
-    P: std::fmt::Debug + Send + Sync + 'static,
-    S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
->(
-    conf: &'static PageServerConf,
-    remote_assets: Arc<(S, RwLock<RemoteTimelineIndex>)>,
-    sync_id: TimelineSyncId,
-    mut download: TimelineDownload,
-    retries: u32,
-) -> DownloadedTimeline {
-    debug!("Downloading layers for sync id {}", sync_id);
-
-    let TimelineSyncId(tenant_id, timeline_id) = sync_id;
-    let index_read = remote_assets.1.read().await;
-    let remote_timeline = match index_read.timeline_entry(&sync_id) {
-        None => {
-            error!("Cannot download: no timeline is present in the index for given ids");
-            return DownloadedTimeline::Abort;
-        }
-        Some(index_entry) => match index_entry {
-            TimelineIndexEntry::Full(remote_timeline) => Cow::Borrowed(remote_timeline),
-            TimelineIndexEntry::Description(_) => {
-                let remote_disk_consistent_lsn = index_entry.disk_consistent_lsn();
-                drop(index_read);
-                debug!("Found timeline description for the given ids, downloading the full index");
-                match update_index_description(
-                    remote_assets.as_ref(),
-                    &conf.timeline_path(&timeline_id, &tenant_id),
-                    sync_id,
-                )
-                .await
-                {
-                    Ok(remote_timeline) => Cow::Owned(remote_timeline),
-                    Err(e) => {
-                        error!("Failed to download full timeline index: {:#}", e);
-                        return match remote_disk_consistent_lsn {
-                            Some(disk_consistent_lsn) => {
-                                sync_queue::push(SyncTask::new(
-                                    sync_id,
-                                    retries,
-                                    SyncKind::Download(download),
-                                ));
-                                DownloadedTimeline::FailedAndRescheduled {
-                                    disk_consistent_lsn,
-                                }
-                            }
-                            None => {
-                                error!("Cannot download: no disk consistent Lsn is present for the index entry");
-                                DownloadedTimeline::Abort
-                            }
-                        };
-                    }
-                }
-            }
-        },
-    };
-    let disk_consistent_lsn = match remote_timeline.checkpoints().max() {
-        Some(lsn) => lsn,
-        None => {
-            debug!("Cannot download: no disk consistent Lsn is present for the remote timeline");
-            return DownloadedTimeline::Abort;
-        }
-    };
-
-    if let Err(e) = download_missing_branches(conf, remote_assets.as_ref(), sync_id.0).await {
-        error!(
-            "Failed to download missing branches for sync id {}: {:#}",
-            sync_id, e
-        );
-        sync_queue::push(SyncTask::new(
-            sync_id,
-            retries,
-            SyncKind::Download(download),
-        ));
-        return DownloadedTimeline::FailedAndRescheduled {
-            disk_consistent_lsn,
-        };
-    }
-
-    debug!("Downloading timeline archives");
-    let archives_to_download = remote_timeline
-        .checkpoints()
-        .map(ArchiveId)
-        .filter(|remote_archive| !download.archives_to_skip.contains(remote_archive))
-        .collect::<Vec<_>>();
-
-    let archives_total = archives_to_download.len();
-    debug!("Downloading {} archives of a timeline", archives_total);
-    trace!("Archives to download: {:?}", archives_to_download);
-
-    for (archives_downloaded, archive_id) in archives_to_download.into_iter().enumerate() {
-        match try_download_archive(
-            conf,
-            sync_id,
-            Arc::clone(&remote_assets),
-            remote_timeline.as_ref(),
-            archive_id,
-            Arc::clone(&download.files_to_skip),
-        )
-        .await
-        {
-            Err(e) => {
-                let archives_left = archives_total - archives_downloaded;
-                error!(
-                    "Failed to download archive {:?} (archives downloaded: {}; archives left: {}) for tenant {} timeline {}, requeueing the download: {:#}",
-                    archive_id, archives_downloaded, archives_left, tenant_id, timeline_id, e
-                );
-                sync_queue::push(SyncTask::new(
-                    sync_id,
-                    retries,
-                    SyncKind::Download(download),
-                ));
-                return DownloadedTimeline::FailedAndRescheduled {
-                    disk_consistent_lsn,
-                };
-            }
-            Ok(()) => {
-                debug!("Successfully downloaded archive {:?}", archive_id);
-                download.archives_to_skip.insert(archive_id);
-            }
-        }
-    }
-
-    debug!("Finished downloading all timeline's archives");
-    DownloadedTimeline::Successful {
-        disk_consistent_lsn,
-    }
-}
-
-async fn try_download_archive<
-    P: Send + Sync + 'static,
-    S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
->(
-    conf: &'static PageServerConf,
-    TimelineSyncId(tenant_id, timeline_id): TimelineSyncId,
-    remote_assets: Arc<(S, RwLock<RemoteTimelineIndex>)>,
-    remote_timeline: &RemoteTimeline,
-    archive_id: ArchiveId,
-    files_to_skip: Arc<BTreeSet<PathBuf>>,
-) -> anyhow::Result<()> {
-    debug!("Downloading archive {:?}", archive_id);
-    let archive_to_download = remote_timeline
-        .archive_data(archive_id)
-        .ok_or_else(|| anyhow!("Archive {:?} not found in remote storage", archive_id))?;
-    let (archive_header, header_size) = remote_timeline
-        .restore_header(archive_id)
-        .context("Failed to restore header when downloading an archive")?;
-
-    match read_local_metadata(conf, timeline_id, tenant_id).await {
-        Ok(local_metadata) => ensure!(
-            // need to allow `<=` instead of `<` due to cases when a failed archive can be redownloaded
-            local_metadata.disk_consistent_lsn() <= archive_to_download.disk_consistent_lsn(),
-            "Cannot download archive with Lsn {} since it's earlier than local Lsn {}",
-            archive_to_download.disk_consistent_lsn(),
-            local_metadata.disk_consistent_lsn()
-        ),
-        Err(e) => warn!("Failed to read local metadata file, assuing it's safe to override its with the download. Read: {:#}", e),
-    }
-    compression::uncompress_file_stream_with_index(
-        conf.timeline_path(&timeline_id, &tenant_id),
-        files_to_skip,
-        archive_to_download.disk_consistent_lsn(),
-        archive_header,
-        header_size,
-        move |mut archive_target, archive_name| async move {
-            let archive_local_path = conf
-                .timeline_path(&timeline_id, &tenant_id)
-                .join(&archive_name);
-            let remote_storage = &remote_assets.0;
-            remote_storage
-                .download_range(
-                    &remote_storage.storage_path(&archive_local_path)?,
-                    header_size,
-                    None,
-                    &mut archive_target,
-                )
-                .await
-        },
-    )
-    .await?;
-
-    Ok(())
-}
-
-async fn read_local_metadata(
-    conf: &'static PageServerConf,
-    timeline_id: zenith_utils::zid::ZTimelineId,
-    tenant_id: ZTenantId,
-) -> anyhow::Result<TimelineMetadata> {
-    let local_metadata_path = metadata_path(conf, timeline_id, tenant_id);
-    let local_metadata_bytes = fs::read(&local_metadata_path)
-        .await
-        .context("Failed to read local metadata file bytes")?;
-    Ok(TimelineMetadata::from_bytes(&local_metadata_bytes)
-        .context("Failed to read local metadata files bytes")?)
-}
-
-async fn download_missing_branches<
-    P: std::fmt::Debug + Send + Sync + 'static,
-    S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
->(
-    conf: &'static PageServerConf,
-    (storage, index): &(S, RwLock<RemoteTimelineIndex>),
-    tenant_id: ZTenantId,
-) -> anyhow::Result<()> {
-    let local_branches = tenant_branch_files(conf, tenant_id)
-        .await
-        .context("Failed to list local branch files for the tenant")?;
-    let local_branches_dir = conf.branches_path(&tenant_id);
-    if !local_branches_dir.exists() {
-        fs::create_dir_all(&local_branches_dir)
-            .await
-            .with_context(|| {
-                format!(
-                    "Failed to create local branches directory at path '{}'",
-                    local_branches_dir.display()
-                )
-            })?;
-    }
-
-    if let Some(remote_branches) = index.read().await.branch_files(tenant_id) {
-        let mut remote_only_branches_downloads = remote_branches
-            .difference(&local_branches)
-            .map(|remote_only_branch| async move {
-                let branches_dir = conf.branches_path(&tenant_id);
-                let remote_branch_path = remote_only_branch.as_path(&branches_dir);
-                let storage_path =
-                    storage.storage_path(&remote_branch_path).with_context(|| {
-                        format!(
-                            "Failed to derive a storage path for branch with local path '{}'",
-                            remote_branch_path.display()
-                        )
-                    })?;
-                let mut target_file = fs::OpenOptions::new()
-                    .write(true)
-                    .create_new(true)
-                    .open(&remote_branch_path)
-                    .await
-                    .with_context(|| {
-                        format!(
-                            "Failed to create local branch file at '{}'",
-                            remote_branch_path.display()
-                        )
-                    })?;
-                storage
-                    .download(&storage_path, &mut target_file)
-                    .await
-                    .with_context(|| {
-                        format!(
-                            "Failed to download branch file from the remote path {:?}",
-                            storage_path
-                        )
-                    })?;
-                Ok::<_, anyhow::Error>(())
-            })
-            .collect::<FuturesUnordered<_>>();
-
-        let mut branch_downloads_failed = false;
-        while let Some(download_result) = remote_only_branches_downloads.next().await {
-            if let Err(e) = download_result {
-                branch_downloads_failed = true;
-                error!("Failed to download a branch file: {:#}", e);
-            }
-        }
-        ensure!(
-            !branch_downloads_failed,
-            "Failed to download all branch files"
-        );
-    }
-
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use std::collections::BTreeSet;
-
-    use tempfile::tempdir;
-    use tokio::fs;
-    use zenith_utils::lsn::Lsn;
-
-    use crate::{
-        remote_storage::{
-            local_fs::LocalFs,
-            storage_sync::test_utils::{
-                assert_index_descriptions, assert_timeline_files_match, create_local_timeline,
-                dummy_metadata, ensure_correct_timeline_upload, expect_timeline,
-            },
-        },
-        repository::repo_harness::{RepoHarness, TIMELINE_ID},
-    };
-
-    use super::*;
-
-    #[tokio::test]
-    async fn test_download_timeline() -> anyhow::Result<()> {
-        let repo_harness = RepoHarness::create("test_download_timeline")?;
-        let sync_id = TimelineSyncId(repo_harness.tenant_id, TIMELINE_ID);
-        let storage = LocalFs::new(tempdir()?.path().to_owned(), &repo_harness.conf.workdir)?;
-        let index = RwLock::new(RemoteTimelineIndex::try_parse_descriptions_from_paths(
-            repo_harness.conf,
-            storage
-                .list()
-                .await?
-                .into_iter()
-                .map(|storage_path| storage.local_path(&storage_path).unwrap()),
-        ));
-        let remote_assets = Arc::new((storage, index));
-        let storage = &remote_assets.0;
-        let index = &remote_assets.1;
-
-        let regular_timeline_path = repo_harness.timeline_path(&TIMELINE_ID);
-        let regular_timeline = create_local_timeline(
-            &repo_harness,
-            TIMELINE_ID,
-            &["a", "b"],
-            dummy_metadata(Lsn(0x30)),
-        )?;
-        ensure_correct_timeline_upload(
-            &repo_harness,
-            Arc::clone(&remote_assets),
-            TIMELINE_ID,
-            regular_timeline,
-        )
-        .await;
-        // upload multiple checkpoints for the same timeline
-        let regular_timeline = create_local_timeline(
-            &repo_harness,
-            TIMELINE_ID,
-            &["c", "d"],
-            dummy_metadata(Lsn(0x40)),
-        )?;
-        ensure_correct_timeline_upload(
-            &repo_harness,
-            Arc::clone(&remote_assets),
-            TIMELINE_ID,
-            regular_timeline,
-        )
-        .await;
-
-        fs::remove_dir_all(&regular_timeline_path).await?;
-        let remote_regular_timeline = expect_timeline(index, sync_id).await;
-
-        download_timeline(
-            repo_harness.conf,
-            Arc::clone(&remote_assets),
-            sync_id,
-            TimelineDownload {
-                files_to_skip: Arc::new(BTreeSet::new()),
-                archives_to_skip: BTreeSet::new(),
-            },
-            0,
-        )
-        .await;
-        assert_index_descriptions(
-            index,
-            RemoteTimelineIndex::try_parse_descriptions_from_paths(
-                repo_harness.conf,
-                remote_assets
-                    .0
-                    .list()
-                    .await
-                    .unwrap()
-                    .into_iter()
-                    .map(|storage_path| storage.local_path(&storage_path).unwrap()),
-            ),
-        )
-        .await;
-        assert_timeline_files_match(&repo_harness, TIMELINE_ID, remote_regular_timeline);
-
-        Ok(())
-    }
-}

pageserver/src/remote_storage/storage_sync/index.rs

@@ -1,422 +0,0 @@
-//! In-memory index to track the tenant files on the remote strorage, mitigating the storage format differences between the local and remote files.
-//! Able to restore itself from the storage archive data and reconstruct archive indices on demand.
-//!
-//! The index is intended to be portable, so deliberately does not store any local paths inside.
-//! This way in the future, the index could be restored fast from its serialized stored form.
-
-use std::{
-    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
-    path::{Path, PathBuf},
-};
-
-use anyhow::{anyhow, bail, ensure, Context};
-use serde::{Deserialize, Serialize};
-use tracing::debug;
-use zenith_utils::{
-    lsn::Lsn,
-    zid::{ZTenantId, ZTimelineId},
-};
-
-use crate::{
-    config::PageServerConf,
-    layered_repository::TIMELINES_SEGMENT_NAME,
-    remote_storage::{
-        storage_sync::compression::{parse_archive_name, FileEntry},
-        TimelineSyncId,
-    },
-};
-
-use super::compression::ArchiveHeader;
-
-/// A part of the filesystem path, that needs a root to become a path again.
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
-pub struct RelativePath(String);
-
-impl RelativePath {
-    /// Attempts to strip off the base from path, producing a relative path or an error.
-    pub fn new<P: AsRef<Path>>(base: &Path, path: P) -> anyhow::Result<Self> {
-        let relative = path
-            .as_ref()
-            .strip_prefix(base)
-            .context("path is not relative to base")?;
-        Ok(RelativePath(relative.to_string_lossy().to_string()))
-    }
-
-    /// Joins the relative path with the base path.
-    pub fn as_path(&self, base: &Path) -> PathBuf {
-        base.join(&self.0)
-    }
-}
-
-/// An index to track tenant files that exist on the remote storage.
-/// Currently, timeline archives and branch files are tracked.
-#[derive(Debug, Clone)]
-pub struct RemoteTimelineIndex {
-    branch_files: HashMap<ZTenantId, HashSet<RelativePath>>,
-    timeline_files: HashMap<TimelineSyncId, TimelineIndexEntry>,
-}
-
-impl RemoteTimelineIndex {
-    /// Attempts to parse file paths (not checking the file contents) and find files
-    /// that can be tracked wiht the index.
-    /// On parse falures, logs the error and continues, so empty index can be created from not suitable paths.
-    pub fn try_parse_descriptions_from_paths<P: AsRef<Path>>(
-        conf: &'static PageServerConf,
-        paths: impl Iterator<Item = P>,
-    ) -> Self {
-        let mut index = Self {
-            branch_files: HashMap::new(),
-            timeline_files: HashMap::new(),
-        };
-        for path in paths {
-            if let Err(e) = try_parse_index_entry(&mut index, conf, path.as_ref()) {
-                debug!(
-                    "Failed to parse path '{}' as index entry: {:#}",
-                    path.as_ref().display(),
-                    e
-                );
-            }
-        }
-        index
-    }
-
-    pub fn timeline_entry(&self, id: &TimelineSyncId) -> Option<&TimelineIndexEntry> {
-        self.timeline_files.get(id)
-    }
-
-    pub fn timeline_entry_mut(&mut self, id: &TimelineSyncId) -> Option<&mut TimelineIndexEntry> {
-        self.timeline_files.get_mut(id)
-    }
-
-    pub fn add_timeline_entry(&mut self, id: TimelineSyncId, entry: TimelineIndexEntry) {
-        self.timeline_files.insert(id, entry);
-    }
-
-    pub fn all_sync_ids(&self) -> impl Iterator<Item = TimelineSyncId> + '_ {
-        self.timeline_files.keys().copied()
-    }
-
-    pub fn add_branch_file(&mut self, tenant_id: ZTenantId, path: RelativePath) {
-        self.branch_files
-            .entry(tenant_id)
-            .or_insert_with(HashSet::new)
-            .insert(path);
-    }
-
-    pub fn branch_files(&self, tenant_id: ZTenantId) -> Option<&HashSet<RelativePath>> {
-        self.branch_files.get(&tenant_id)
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum TimelineIndexEntry {
-    /// An archive found on the remote storage, but not yet downloaded, only a metadata from its storage path is available, without archive contents.
-    Description(BTreeMap<ArchiveId, ArchiveDescription>),
-    /// Full archive metadata, including the file list, parsed from the archive header.
-    Full(RemoteTimeline),
-}
-
-impl TimelineIndexEntry {
-    pub fn uploaded_checkpoints(&self) -> BTreeSet<Lsn> {
-        match self {
-            Self::Description(description) => {
-                description.keys().map(|archive_id| archive_id.0).collect()
-            }
-            Self::Full(remote_timeline) => remote_timeline
-                .checkpoint_archives
-                .keys()
-                .map(|archive_id| archive_id.0)
-                .collect(),
-        }
-    }
-
-    /// Gets latest uploaded checkpoint's disk consisten Lsn for the corresponding timeline.
-    pub fn disk_consistent_lsn(&self) -> Option<Lsn> {
-        match self {
-            Self::Description(description) => {
-                description.keys().map(|archive_id| archive_id.0).max()
-            }
-            Self::Full(remote_timeline) => remote_timeline
-                .checkpoint_archives
-                .keys()
-                .map(|archive_id| archive_id.0)
-                .max(),
-        }
-    }
-}
-
-/// Checkpoint archive's id, corresponding to the `disk_consistent_lsn` from the timeline's metadata file during checkpointing.
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
-pub struct ArchiveId(pub(super) Lsn);
-
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
-struct FileId(ArchiveId, ArchiveEntryNumber);
-
-type ArchiveEntryNumber = usize;
-
-/// All archives and files in them, representing a certain timeline.
-/// Uses file and archive IDs to reference those without ownership issues.
-#[derive(Debug, PartialEq, Eq, Clone)]
-pub struct RemoteTimeline {
-    timeline_files: BTreeMap<FileId, FileEntry>,
-    checkpoint_archives: BTreeMap<ArchiveId, CheckpointArchive>,
-}
-
-/// Archive metadata, enough to restore a header with the timeline data.
-#[derive(Debug, PartialEq, Eq, Clone)]
-pub struct CheckpointArchive {
-    disk_consistent_lsn: Lsn,
-    metadata_file_size: u64,
-    files: BTreeSet<FileId>,
-    archive_header_size: u64,
-}
-
-impl CheckpointArchive {
-    pub fn disk_consistent_lsn(&self) -> Lsn {
-        self.disk_consistent_lsn
-    }
-}
-
-impl RemoteTimeline {
-    pub fn empty() -> Self {
-        Self {
-            timeline_files: BTreeMap::new(),
-            checkpoint_archives: BTreeMap::new(),
-        }
-    }
-
-    pub fn checkpoints(&self) -> impl Iterator<Item = Lsn> + '_ {
-        self.checkpoint_archives
-            .values()
-            .map(CheckpointArchive::disk_consistent_lsn)
-    }
-
-    /// Lists all relish files in the given remote timeline. Omits the metadata file.
-    pub fn stored_files(&self, timeline_dir: &Path) -> BTreeSet<PathBuf> {
-        self.timeline_files
-            .values()
-            .map(|file_entry| file_entry.subpath.as_path(timeline_dir))
-            .collect()
-    }
-
-    pub fn contains_checkpoint_at(&self, disk_consistent_lsn: Lsn) -> bool {
-        self.checkpoint_archives
-            .contains_key(&ArchiveId(disk_consistent_lsn))
-    }
-
-    pub fn archive_data(&self, archive_id: ArchiveId) -> Option<&CheckpointArchive> {
-        self.checkpoint_archives.get(&archive_id)
-    }
-
-    /// Restores a header of a certain remote archive from the memory data.
-    /// Returns the header and its compressed size in the archive, both can be used to uncompress that archive.
-    pub fn restore_header(&self, archive_id: ArchiveId) -> anyhow::Result<(ArchiveHeader, u64)> {
-        let archive = self
-            .checkpoint_archives
-            .get(&archive_id)
-            .ok_or_else(|| anyhow!("Archive {:?} not found", archive_id))?;
-
-        let mut header_files = Vec::with_capacity(archive.files.len());
-        for (expected_archive_position, archive_file) in archive.files.iter().enumerate() {
-            let &FileId(archive_id, archive_position) = archive_file;
-            ensure!(
-                expected_archive_position == archive_position,
-                "Archive header is corrupt, file # {} from archive {:?} header is missing",
-                expected_archive_position,
-                archive_id,
-            );
-
-            let timeline_file = self.timeline_files.get(archive_file).ok_or_else(|| {
-                anyhow!(
-                    "File with id {:?} not found for archive {:?}",
-                    archive_file,
-                    archive_id
-                )
-            })?;
-            header_files.push(timeline_file.clone());
-        }
-
-        Ok((
-            ArchiveHeader {
-                files: header_files,
-                metadata_file_size: archive.metadata_file_size,
-            },
-            archive.archive_header_size,
-        ))
-    }
-
-    /// Updates (creates, if necessary) the data about certain archive contents.
-    pub fn update_archive_contents(
-        &mut self,
-        disk_consistent_lsn: Lsn,
-        header: ArchiveHeader,
-        header_size: u64,
-    ) {
-        let archive_id = ArchiveId(disk_consistent_lsn);
-        let mut common_archive_files = BTreeSet::new();
-        for (file_index, file_entry) in header.files.into_iter().enumerate() {
-            let file_id = FileId(archive_id, file_index);
-            self.timeline_files.insert(file_id, file_entry);
-            common_archive_files.insert(file_id);
-        }
-
-        let metadata_file_size = header.metadata_file_size;
-        self.checkpoint_archives
-            .entry(archive_id)
-            .or_insert_with(|| CheckpointArchive {
-                metadata_file_size,
-                files: BTreeSet::new(),
-                archive_header_size: header_size,
-                disk_consistent_lsn,
-            })
-            .files
-            .extend(common_archive_files.into_iter());
-    }
-}
-
-/// Metadata abput timeline checkpoint archive, parsed from its remote storage path.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ArchiveDescription {
-    pub header_size: u64,
-    pub disk_consistent_lsn: Lsn,
-    pub archive_name: String,
-}
-
-fn try_parse_index_entry(
-    index: &mut RemoteTimelineIndex,
-    conf: &'static PageServerConf,
-    path: &Path,
-) -> anyhow::Result<()> {
-    let tenants_dir = conf.tenants_path();
-    let tenant_id = path
-        .strip_prefix(&tenants_dir)
-        .with_context(|| {
-            format!(
-                "Path '{}' does not belong to tenants directory '{}'",
-                path.display(),
-                tenants_dir.display(),
-            )
-        })?
-        .iter()
-        .next()
-        .ok_or_else(|| anyhow!("Found no tenant id in path '{}'", path.display()))?
-        .to_string_lossy()
-        .parse::<ZTenantId>()
-        .with_context(|| format!("Failed to parse tenant id from path '{}'", path.display()))?;
-
-    let branches_path = conf.branches_path(&tenant_id);
-    let timelines_path = conf.timelines_path(&tenant_id);
-    match (
-        RelativePath::new(&branches_path, &path),
-        path.strip_prefix(&timelines_path),
-    ) {
-        (Ok(_), Ok(_)) => bail!(
-            "Path '{}' cannot start with both branches '{}' and the timelines '{}' prefixes",
-            path.display(),
-            branches_path.display(),
-            timelines_path.display()
-        ),
-        (Ok(branches_entry), Err(_)) => index.add_branch_file(tenant_id, branches_entry),
-        (Err(_), Ok(timelines_subpath)) => {
-            let mut segments = timelines_subpath.iter();
-            let timeline_id = segments
-                .next()
-                .ok_or_else(|| {
-                    anyhow!(
-                        "{} directory of tenant {} (path '{}') is not an index entry",
-                        TIMELINES_SEGMENT_NAME,
-                        tenant_id,
-                        path.display()
-                    )
-                })?
-                .to_string_lossy()
-                .parse::<ZTimelineId>()
-                .with_context(|| {
-                    format!("Failed to parse timeline id from path '{}'", path.display())
-                })?;
-
-            let (disk_consistent_lsn, header_size) =
-                parse_archive_name(path).with_context(|| {
-                    format!(
-                        "Failed to parse archive name out in path '{}'",
-                        path.display()
-                    )
-                })?;
-
-            let archive_name = path
-                .file_name()
-                .ok_or_else(|| anyhow!("Archive '{}' has no file name", path.display()))?
-                .to_string_lossy()
-                .to_string();
-
-            let sync_id = TimelineSyncId(tenant_id, timeline_id);
-            let timeline_index_entry = index
-                .timeline_files
-                .entry(sync_id)
-                .or_insert_with(|| TimelineIndexEntry::Description(BTreeMap::new()));
-            match timeline_index_entry {
-                TimelineIndexEntry::Description(descriptions) => {
-                    descriptions.insert(
-                        ArchiveId(disk_consistent_lsn),
-                        ArchiveDescription {
-                            header_size,
-                            disk_consistent_lsn,
-                            archive_name,
-                        },
-                    );
-                }
-                TimelineIndexEntry::Full(_) => {
-                    bail!("Cannot add parsed archive description to its full context in index with sync id {}", sync_id)
-                }
-            }
-        }
-        (Err(branches_error), Err(timelines_strip_error)) => {
-            bail!(
-                "Path '{}' is not an index entry: it's neither parsable as a branch entry '{:#}' nor as an archive entry '{}'",
-                path.display(),
-                branches_error,
-                timelines_strip_error,
-            )
-        }
-    }
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn header_restoration_preserves_file_order() {
-        let header = ArchiveHeader {
-            files: vec![
-                FileEntry {
-                    size: 5,
-                    subpath: RelativePath("one".to_string()),
-                },
-                FileEntry {
-                    size: 1,
-                    subpath: RelativePath("two".to_string()),
-                },
-                FileEntry {
-                    size: 222,
-                    subpath: RelativePath("zero".to_string()),
-                },
-            ],
-            metadata_file_size: 5,
-        };
-
-        let lsn = Lsn(1);
-        let mut remote_timeline = RemoteTimeline::empty();
-        remote_timeline.update_archive_contents(lsn, header.clone(), 15);
-
-        let (restored_header, _) = remote_timeline
-            .restore_header(ArchiveId(lsn))
-            .expect("Should be able to restore header from a valid remote timeline");
-
-        assert_eq!(
-            header, restored_header,
-            "Header restoration should preserve file order"
-        );
-    }
-}

pageserver/src/remote_storage/storage_sync/upload.rs

@@ -1,566 +0,0 @@
-//! Timeline synchronization logic to compress and upload to the remote storage all new timeline files from the checkpoints.
-//! Currently, tenant branch files are also uploaded, but this does not appear final.
-
-use std::{borrow::Cow, collections::BTreeSet, path::PathBuf, sync::Arc};
-
-use anyhow::{ensure, Context};
-use futures::{stream::FuturesUnordered, StreamExt};
-use tokio::{fs, sync::RwLock};
-use tracing::{debug, error, warn};
-use zenith_utils::zid::ZTenantId;
-
-use crate::{
-    config::PageServerConf,
-    remote_storage::{
-        storage_sync::{
-            compression,
-            index::{RemoteTimeline, TimelineIndexEntry},
-            sync_queue, tenant_branch_files, update_index_description, SyncKind, SyncTask,
-        },
-        RemoteStorage, TimelineSyncId,
-    },
-};
-
-use super::{compression::ArchiveHeader, index::RemoteTimelineIndex, NewCheckpoint};
-
-/// Attempts to compress and upload given checkpoint files.
-/// No extra checks for overlapping files is made: download takes care of that, ensuring no non-metadata local timeline files are overwritten.
-///
-/// Before the checkpoint files are uploaded, branch files are uploaded, if any local ones are missing remotely.
-///
-/// On an error, bumps the retries count and reschedules the entire task.
-/// On success, populates index data with new downloads.
-pub(super) async fn upload_timeline_checkpoint<
-    P: std::fmt::Debug + Send + Sync + 'static,
-    S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
->(
-    config: &'static PageServerConf,
-    remote_assets: Arc<(S, RwLock<RemoteTimelineIndex>)>,
-    sync_id: TimelineSyncId,
-    new_checkpoint: NewCheckpoint,
-    retries: u32,
-) -> Option<bool> {
-    debug!("Uploading checkpoint for sync id {}", sync_id);
-    if let Err(e) = upload_missing_branches(config, remote_assets.as_ref(), sync_id.0).await {
-        error!(
-            "Failed to upload missing branches for sync id {}: {:#}",
-            sync_id, e
-        );
-        sync_queue::push(SyncTask::new(
-            sync_id,
-            retries,
-            SyncKind::Upload(new_checkpoint),
-        ));
-        return Some(false);
-    }
-    let new_upload_lsn = new_checkpoint.metadata.disk_consistent_lsn();
-
-    let index = &remote_assets.1;
-
-    let TimelineSyncId(tenant_id, timeline_id) = sync_id;
-    let timeline_dir = config.timeline_path(&timeline_id, &tenant_id);
-
-    let index_read = index.read().await;
-    let remote_timeline = match index_read.timeline_entry(&sync_id) {
-        None => None,
-        Some(TimelineIndexEntry::Full(remote_timeline)) => Some(Cow::Borrowed(remote_timeline)),
-        Some(TimelineIndexEntry::Description(_)) => {
-            debug!("Found timeline description for the given ids, downloading the full index");
-            match update_index_description(remote_assets.as_ref(), &timeline_dir, sync_id).await {
-                Ok(remote_timeline) => Some(Cow::Owned(remote_timeline)),
-                Err(e) => {
-                    error!("Failed to download full timeline index: {:#}", e);
-                    sync_queue::push(SyncTask::new(
-                        sync_id,
-                        retries,
-                        SyncKind::Upload(new_checkpoint),
-                    ));
-                    return Some(false);
-                }
-            }
-        }
-    };
-
-    let already_contains_upload_lsn = remote_timeline
-        .as_ref()
-        .map(|remote_timeline| remote_timeline.contains_checkpoint_at(new_upload_lsn))
-        .unwrap_or(false);
-    if already_contains_upload_lsn {
-        warn!(
-            "Received a checkpoint with Lsn {} that's already been uploaded to remote storage, skipping the upload.",
-            new_upload_lsn
-        );
-        return None;
-    }
-
-    let already_uploaded_files = remote_timeline
-        .map(|timeline| timeline.stored_files(&timeline_dir))
-        .unwrap_or_default();
-    drop(index_read);
-
-    match try_upload_checkpoint(
-        config,
-        Arc::clone(&remote_assets),
-        sync_id,
-        &new_checkpoint,
-        already_uploaded_files,
-    )
-    .await
-    {
-        Ok((archive_header, header_size)) => {
-            let mut index_write = index.write().await;
-            match index_write.timeline_entry_mut(&sync_id) {
-                Some(TimelineIndexEntry::Full(remote_timeline)) => {
-                    remote_timeline.update_archive_contents(
-                        new_checkpoint.metadata.disk_consistent_lsn(),
-                        archive_header,
-                        header_size,
-                    );
-                }
-                None | Some(TimelineIndexEntry::Description(_)) => {
-                    let mut new_timeline = RemoteTimeline::empty();
-                    new_timeline.update_archive_contents(
-                        new_checkpoint.metadata.disk_consistent_lsn(),
-                        archive_header,
-                        header_size,
-                    );
-                    index_write.add_timeline_entry(sync_id, TimelineIndexEntry::Full(new_timeline));
-                }
-            }
-            debug!("Checkpoint uploaded successfully");
-            Some(true)
-        }
-        Err(e) => {
-            error!(
-                "Failed to upload checkpoint: {:#}, requeueing the upload",
-                e
-            );
-            sync_queue::push(SyncTask::new(
-                sync_id,
-                retries,
-                SyncKind::Upload(new_checkpoint),
-            ));
-            Some(false)
-        }
-    }
-}
-
-async fn try_upload_checkpoint<
-    P: Send + Sync + 'static,
-    S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
->(
-    config: &'static PageServerConf,
-    remote_assets: Arc<(S, RwLock<RemoteTimelineIndex>)>,
-    sync_id: TimelineSyncId,
-    new_checkpoint: &NewCheckpoint,
-    files_to_skip: BTreeSet<PathBuf>,
-) -> anyhow::Result<(ArchiveHeader, u64)> {
-    let TimelineSyncId(tenant_id, timeline_id) = sync_id;
-    let timeline_dir = config.timeline_path(&timeline_id, &tenant_id);
-
-    let files_to_upload = new_checkpoint
-        .layers
-        .iter()
-        .filter(|&path_to_upload| {
-            if files_to_skip.contains(path_to_upload) {
-                error!(
-                    "Skipping file upload '{}', since it was already uploaded",
-                    path_to_upload.display()
-                );
-                false
-            } else {
-                true
-            }
-        })
-        .collect::<Vec<_>>();
-    ensure!(!files_to_upload.is_empty(), "No files to upload");
-
-    compression::archive_files_as_stream(
-        &timeline_dir,
-        files_to_upload.into_iter(),
-        &new_checkpoint.metadata,
-        move |archive_streamer, archive_name| async move {
-            let timeline_dir = config.timeline_path(&timeline_id, &tenant_id);
-            let remote_storage = &remote_assets.0;
-            remote_storage
-                .upload(
-                    archive_streamer,
-                    &remote_storage.storage_path(&timeline_dir.join(&archive_name))?,
-                )
-                .await
-        },
-    )
-    .await
-    .map(|(header, header_size, _)| (header, header_size))
-}
-
-async fn upload_missing_branches<
-    P: std::fmt::Debug + Send + Sync + 'static,
-    S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
->(
-    config: &'static PageServerConf,
-    (storage, index): &(S, RwLock<RemoteTimelineIndex>),
-    tenant_id: ZTenantId,
-) -> anyhow::Result<()> {
-    let local_branches = tenant_branch_files(config, tenant_id)
-        .await
-        .context("Failed to list local branch files for the tenant")?;
-    let index_read = index.read().await;
-    let remote_branches = index_read
-        .branch_files(tenant_id)
-        .cloned()
-        .unwrap_or_default();
-    drop(index_read);
-
-    let mut branch_uploads = local_branches
-        .difference(&remote_branches)
-        .map(|local_only_branch| async move {
-            let local_branch_path = local_only_branch.as_path(&config.branches_path(&tenant_id));
-            let storage_path = storage.storage_path(&local_branch_path).with_context(|| {
-                format!(
-                    "Failed to derive a storage path for branch with local path '{}'",
-                    local_branch_path.display()
-                )
-            })?;
-            let local_branch_file = fs::OpenOptions::new()
-                .read(true)
-                .open(&local_branch_path)
-                .await
-                .with_context(|| {
-                    format!(
-                        "Failed to open local branch file {} for reading",
-                        local_branch_path.display()
-                    )
-                })?;
-            storage
-                .upload(local_branch_file, &storage_path)
-                .await
-                .with_context(|| {
-                    format!(
-                        "Failed to upload branch file to the remote path {:?}",
-                        storage_path
-                    )
-                })?;
-            Ok::<_, anyhow::Error>(local_only_branch)
-        })
-        .collect::<FuturesUnordered<_>>();
-
-    let mut branch_uploads_failed = false;
-    while let Some(upload_result) = branch_uploads.next().await {
-        match upload_result {
-            Ok(local_only_branch) => index
-                .write()
-                .await
-                .add_branch_file(tenant_id, local_only_branch.clone()),
-            Err(e) => {
-                error!("Failed to upload branch file: {:#}", e);
-                branch_uploads_failed = true;
-            }
-        }
-    }
-
-    ensure!(!branch_uploads_failed, "Failed to upload all branch files");
-
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use tempfile::tempdir;
-    use zenith_utils::lsn::Lsn;
-
-    use crate::{
-        remote_storage::{
-            local_fs::LocalFs,
-            storage_sync::{
-                index::ArchiveId,
-                test_utils::{
-                    assert_index_descriptions, create_local_timeline, dummy_metadata,
-                    ensure_correct_timeline_upload, expect_timeline,
-                },
-            },
-        },
-        repository::repo_harness::{RepoHarness, TIMELINE_ID},
-    };
-
-    use super::*;
-
-    #[tokio::test]
-    async fn reupload_timeline() -> anyhow::Result<()> {
-        let repo_harness = RepoHarness::create("reupload_timeline")?;
-        let sync_id = TimelineSyncId(repo_harness.tenant_id, TIMELINE_ID);
-        let storage = LocalFs::new(tempdir()?.path().to_owned(), &repo_harness.conf.workdir)?;
-        let index = RwLock::new(RemoteTimelineIndex::try_parse_descriptions_from_paths(
-            repo_harness.conf,
-            storage
-                .list()
-                .await?
-                .into_iter()
-                .map(|storage_path| storage.local_path(&storage_path).unwrap()),
-        ));
-        let remote_assets = Arc::new((storage, index));
-        let index = &remote_assets.1;
-
-        let first_upload_metadata = dummy_metadata(Lsn(0x10));
-        let first_checkpoint = create_local_timeline(
-            &repo_harness,
-            TIMELINE_ID,
-            &["a", "b"],
-            first_upload_metadata.clone(),
-        )?;
-        let local_timeline_path = repo_harness.timeline_path(&TIMELINE_ID);
-        ensure_correct_timeline_upload(
-            &repo_harness,
-            Arc::clone(&remote_assets),
-            TIMELINE_ID,
-            first_checkpoint,
-        )
-        .await;
-
-        let uploaded_timeline = expect_timeline(index, sync_id).await;
-        let uploaded_archives = uploaded_timeline
-            .checkpoints()
-            .map(ArchiveId)
-            .collect::<Vec<_>>();
-        assert_eq!(
-            uploaded_archives.len(),
-            1,
-            "Only one archive is expected after a first upload"
-        );
-        let first_uploaded_archive = uploaded_archives.first().copied().unwrap();
-        assert_eq!(
-            uploaded_timeline.checkpoints().last(),
-            Some(first_upload_metadata.disk_consistent_lsn()),
-            "Metadata that was uploaded, should have its Lsn stored"
-        );
-        assert_eq!(
-            uploaded_timeline
-                .archive_data(uploaded_archives.first().copied().unwrap())
-                .unwrap()
-                .disk_consistent_lsn(),
-            first_upload_metadata.disk_consistent_lsn(),
-            "Uploaded archive should have corresponding Lsn"
-        );
-        assert_eq!(
-            uploaded_timeline.stored_files(&local_timeline_path),
-            vec![local_timeline_path.join("a"), local_timeline_path.join("b")]
-                .into_iter()
-                .collect(),
-            "Should have all files from the first checkpoint"
-        );
-
-        let second_upload_metadata = dummy_metadata(Lsn(0x40));
-        let second_checkpoint = create_local_timeline(
-            &repo_harness,
-            TIMELINE_ID,
-            &["b", "c"],
-            second_upload_metadata.clone(),
-        )?;
-        assert!(
-            first_upload_metadata.disk_consistent_lsn()
-                < second_upload_metadata.disk_consistent_lsn()
-        );
-        ensure_correct_timeline_upload(
-            &repo_harness,
-            Arc::clone(&remote_assets),
-            TIMELINE_ID,
-            second_checkpoint,
-        )
-        .await;
-
-        let updated_timeline = expect_timeline(index, sync_id).await;
-        let mut updated_archives = updated_timeline
-            .checkpoints()
-            .map(ArchiveId)
-            .collect::<Vec<_>>();
-        assert_eq!(
-            updated_archives.len(),
-            2,
-            "Two archives are expected after a successful update of the upload"
-        );
-        updated_archives.retain(|archive_id| archive_id != &first_uploaded_archive);
-        assert_eq!(
-            updated_archives.len(),
-            1,
-            "Only one new archive is expected among the uploaded"
-        );
-        let second_uploaded_archive = updated_archives.last().copied().unwrap();
-        assert_eq!(
-            updated_timeline.checkpoints().max(),
-            Some(second_upload_metadata.disk_consistent_lsn()),
-            "Metadata that was uploaded, should have its Lsn stored"
-        );
-        assert_eq!(
-            updated_timeline
-                .archive_data(second_uploaded_archive)
-                .unwrap()
-                .disk_consistent_lsn(),
-            second_upload_metadata.disk_consistent_lsn(),
-            "Uploaded archive should have corresponding Lsn"
-        );
-        assert_eq!(
-            updated_timeline.stored_files(&local_timeline_path),
-            vec![
-                local_timeline_path.join("a"),
-                local_timeline_path.join("b"),
-                local_timeline_path.join("c"),
-            ]
-            .into_iter()
-            .collect(),
-            "Should have all files from both checkpoints without duplicates"
-        );
-
-        let third_upload_metadata = dummy_metadata(Lsn(0x20));
-        let third_checkpoint = create_local_timeline(
-            &repo_harness,
-            TIMELINE_ID,
-            &["d"],
-            third_upload_metadata.clone(),
-        )?;
-        assert_ne!(
-            third_upload_metadata.disk_consistent_lsn(),
-            first_upload_metadata.disk_consistent_lsn()
-        );
-        assert!(
-            third_upload_metadata.disk_consistent_lsn()
-                < second_upload_metadata.disk_consistent_lsn()
-        );
-        ensure_correct_timeline_upload(
-            &repo_harness,
-            Arc::clone(&remote_assets),
-            TIMELINE_ID,
-            third_checkpoint,
-        )
-        .await;
-
-        let updated_timeline = expect_timeline(index, sync_id).await;
-        let mut updated_archives = updated_timeline
-            .checkpoints()
-            .map(ArchiveId)
-            .collect::<Vec<_>>();
-        assert_eq!(
-            updated_archives.len(),
-            3,
-            "Three archives are expected after two successful updates of the upload"
-        );
-        updated_archives.retain(|archive_id| {
-            archive_id != &first_uploaded_archive && archive_id != &second_uploaded_archive
-        });
-        assert_eq!(
-            updated_archives.len(),
-            1,
-            "Only one new archive is expected among the uploaded"
-        );
-        let third_uploaded_archive = updated_archives.last().copied().unwrap();
-        assert!(
-            updated_timeline.checkpoints().max().unwrap()
-                > third_upload_metadata.disk_consistent_lsn(),
-            "Should not influence the last lsn by uploading an older checkpoint"
-        );
-        assert_eq!(
-            updated_timeline
-                .archive_data(third_uploaded_archive)
-                .unwrap()
-                .disk_consistent_lsn(),
-            third_upload_metadata.disk_consistent_lsn(),
-            "Uploaded archive should have corresponding Lsn"
-        );
-        assert_eq!(
-            updated_timeline.stored_files(&local_timeline_path),
-            vec![
-                local_timeline_path.join("a"),
-                local_timeline_path.join("b"),
-                local_timeline_path.join("c"),
-                local_timeline_path.join("d"),
-            ]
-            .into_iter()
-            .collect(),
-            "Should have all files from three checkpoints without duplicates"
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn reupload_timeline_rejected() -> anyhow::Result<()> {
-        let repo_harness = RepoHarness::create("reupload_timeline_rejected")?;
-        let sync_id = TimelineSyncId(repo_harness.tenant_id, TIMELINE_ID);
-        let storage = LocalFs::new(tempdir()?.path().to_owned(), &repo_harness.conf.workdir)?;
-        let index = RwLock::new(RemoteTimelineIndex::try_parse_descriptions_from_paths(
-            repo_harness.conf,
-            storage
-                .list()
-                .await?
-                .into_iter()
-                .map(|storage_path| storage.local_path(&storage_path).unwrap()),
-        ));
-        let remote_assets = Arc::new((storage, index));
-        let storage = &remote_assets.0;
-        let index = &remote_assets.1;
-
-        let first_upload_metadata = dummy_metadata(Lsn(0x10));
-        let first_checkpoint = create_local_timeline(
-            &repo_harness,
-            TIMELINE_ID,
-            &["a", "b"],
-            first_upload_metadata.clone(),
-        )?;
-        ensure_correct_timeline_upload(
-            &repo_harness,
-            Arc::clone(&remote_assets),
-            TIMELINE_ID,
-            first_checkpoint,
-        )
-        .await;
-        let after_first_uploads = RemoteTimelineIndex::try_parse_descriptions_from_paths(
-            repo_harness.conf,
-            remote_assets
-                .0
-                .list()
-                .await
-                .unwrap()
-                .into_iter()
-                .map(|storage_path| storage.local_path(&storage_path).unwrap()),
-        );
-
-        let normal_upload_metadata = dummy_metadata(Lsn(0x20));
-        assert_ne!(
-            normal_upload_metadata.disk_consistent_lsn(),
-            first_upload_metadata.disk_consistent_lsn()
-        );
-
-        let checkpoint_with_no_files = create_local_timeline(
-            &repo_harness,
-            TIMELINE_ID,
-            &[],
-            normal_upload_metadata.clone(),
-        )?;
-        upload_timeline_checkpoint(
-            repo_harness.conf,
-            Arc::clone(&remote_assets),
-            sync_id,
-            checkpoint_with_no_files,
-            0,
-        )
-        .await;
-        assert_index_descriptions(index, after_first_uploads.clone()).await;
-
-        let checkpoint_with_uploaded_lsn = create_local_timeline(
-            &repo_harness,
-            TIMELINE_ID,
-            &["something", "new"],
-            first_upload_metadata.clone(),
-        )?;
-        upload_timeline_checkpoint(
-            repo_harness.conf,
-            Arc::clone(&remote_assets),
-            sync_id,
-            checkpoint_with_uploaded_lsn,
-            0,
-        )
-        .await;
-        assert_index_descriptions(index, after_first_uploads.clone()).await;
-
-        Ok(())
-    }
-}

pageserver/src/repository.rs

@@ -1,9 +1,7 @@
 use crate::relish::*;
-use crate::walrecord::MultiXactMember;
 use crate::CheckpointConfig;
 use anyhow::Result;
-use bytes::Bytes;
-use postgres_ffi::{MultiXactId, MultiXactOffset, TransactionId};
+use bytes::{Buf, BufMut, Bytes, BytesMut};
 use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
 use std::ops::{AddAssign, Deref};
@@ -12,43 +10,23 @@ use std::time::Duration;
 use zenith_utils::lsn::{Lsn, RecordLsn};
 use zenith_utils::zid::ZTimelineId;
 
-/// Block number within a relish. This matches PostgreSQL's BlockNumber type.
-pub type BlockNumber = u32;
-
 ///
 /// A repository corresponds to one .zenith directory. One repository holds multiple
 /// timelines, forked off from the same initial call to 'initdb'.
 pub trait Repository: Send + Sync {
     fn shutdown(&self) -> Result<()>;
 
-    /// Updates timeline based on the new sync state, received from the remote storage synchronization.
-    /// See [`crate::remote_storage`] for more details about the synchronization.
-    fn set_timeline_state(
-        &self,
-        timeline_id: ZTimelineId,
-        new_state: TimelineSyncState,
-    ) -> Result<()>;
-
-    /// Gets current synchronization state of the timeline.
-    /// See [`crate::remote_storage`] for more details about the synchronization.
-    fn get_timeline_state(&self, timeline_id: ZTimelineId) -> Option<TimelineSyncState>;
-
     /// Get Timeline handle for given zenith timeline ID.
-    fn get_timeline(&self, timelineid: ZTimelineId) -> Result<RepositoryTimeline>;
+    fn get_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>>;
 
     /// Create a new, empty timeline. The caller is responsible for loading data into it
-    /// Initdb lsn is provided for timeline impl to be able to perform checks for some operations against it.
-    fn create_empty_timeline(
-        &self,
-        timelineid: ZTimelineId,
-        initdb_lsn: Lsn,
-    ) -> Result<Arc<dyn Timeline>>;
+    fn create_empty_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>>;
 
     /// Branch a timeline
     fn branch_timeline(&self, src: ZTimelineId, dst: ZTimelineId, start_lsn: Lsn) -> Result<()>;
 
     /// perform one garbage collection iteration, removing old data files from disk.
-    /// this function is periodically called by gc thread.
+    /// this funtion is periodically called by gc thread.
     /// also it can be explicitly requested through page server api 'do_gc' command.
     ///
     /// 'timelineid' specifies the timeline to GC, or None for all.
@@ -68,63 +46,6 @@ pub trait Repository: Send + Sync {
     fn checkpoint_iteration(&self, cconf: CheckpointConfig) -> Result<()>;
 }
 
-/// A timeline, that belongs to the current repository.
-pub enum RepositoryTimeline {
-    /// Timeline, with its files present locally in pageserver's working directory.
-    /// Loaded into pageserver's memory and ready to be used.
-    Local(Arc<dyn Timeline>),
-    /// Timeline, found on the pageserver's remote storage, but not yet downloaded locally.
-    Remote {
-        id: ZTimelineId,
-        /// metadata contents of the latest successfully uploaded checkpoint
-        disk_consistent_lsn: Lsn,
-    },
-}
-
-impl RepositoryTimeline {
-    pub fn local_timeline(&self) -> Option<Arc<dyn Timeline>> {
-        if let Self::Local(local_timeline) = self {
-            Some(Arc::clone(local_timeline))
-        } else {
-            None
-        }
-    }
-}
-
-/// A state of the timeline synchronization with the remote storage.
-/// Contains `disk_consistent_lsn` of the corresponding remote timeline (latest checkpoint's disk_consistent_lsn).
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
-pub enum TimelineSyncState {
-    /// No further downloads from the remote storage are needed.
-    /// The timeline state is up-to-date or ahead of the remote storage one,
-    /// ready to be used in any pageserver operation.
-    Ready(Lsn),
-    /// Timeline is scheduled for downloading, but its current local state is not up to date with the remote storage.
-    /// The timeline is not ready to be used in any pageserver operations, otherwise it might diverge its local state from the remote version,
-    /// making it impossible to sync it further.
-    AwaitsDownload(Lsn),
-    /// Timeline was not in the pageserver's local working directory, but was found on the remote storage, ready to be downloaded.
-    /// Cannot be used in any pageserver operations due to complete absence locally.
-    CloudOnly(Lsn),
-    /// Timeline was evicted from the pageserver's local working directory due to conflicting remote and local states or too many errors during the synchronization.
-    /// Such timelines cannot have their state synchronized further and may not have the data about remote timeline's disk_consistent_lsn, since eviction may happen
-    /// due to errors before the remote timeline contents is known.
-    Evicted(Option<Lsn>),
-}
-
-impl TimelineSyncState {
-    pub fn remote_disk_consistent_lsn(&self) -> Option<Lsn> {
-        Some(match self {
-            TimelineSyncState::Evicted(None) => return None,
-            TimelineSyncState::Ready(lsn) => lsn,
-            TimelineSyncState::AwaitsDownload(lsn) => lsn,
-            TimelineSyncState::CloudOnly(lsn) => lsn,
-            TimelineSyncState::Evicted(Some(lsn)) => lsn,
-        })
-        .copied()
-    }
-}
-
 ///
 /// Result of performing GC
 ///
@@ -185,10 +106,10 @@ pub trait Timeline: Send + Sync {
     fn wait_lsn(&self, lsn: Lsn) -> Result<()>;
 
     /// Look up given page version.
-    fn get_page_at_lsn(&self, tag: RelishTag, blknum: BlockNumber, lsn: Lsn) -> Result<Bytes>;
+    fn get_page_at_lsn(&self, tag: RelishTag, blknum: u32, lsn: Lsn) -> Result<Bytes>;
 
     /// Get size of a relish
-    fn get_relish_size(&self, tag: RelishTag, lsn: Lsn) -> Result<Option<BlockNumber>>;
+    fn get_relish_size(&self, tag: RelishTag, lsn: Lsn) -> Result<Option<u32>>;
 
     /// Does relation exist?
     fn get_rel_exists(&self, tag: RelishTag, lsn: Lsn) -> Result<bool>;
@@ -203,9 +124,6 @@ pub trait Timeline: Send + Sync {
     /// Get a list of all existing non-relational objects
     fn list_nonrels(&self, lsn: Lsn) -> Result<HashSet<RelishTag>>;
 
-    /// Get the ancestor's timeline id
-    fn get_ancestor_timeline_id(&self) -> Option<ZTimelineId>;
-
     /// Get the LSN where this branch was created
     fn get_ancestor_lsn(&self) -> Lsn;
 
@@ -233,10 +151,6 @@ pub trait Timeline: Send + Sync {
     /// know anything about them here in the repository.
     fn checkpoint(&self, cconf: CheckpointConfig) -> Result<()>;
 
-    ///
-    /// Check that it is valid to request operations with that lsn.
-    fn check_lsn_is_in_scope(&self, lsn: Lsn) -> Result<()>;
-
     /// Retrieve current logical size of the timeline
     ///
     /// NOTE: counted incrementally, includes ancestors,
@@ -246,9 +160,6 @@ pub trait Timeline: Send + Sync {
     /// Does the same as get_current_logical_size but counted on demand.
     /// Used in tests to ensure thet incremental and non incremental variants match.
     fn get_current_logical_size_non_incremental(&self, lsn: Lsn) -> Result<usize>;
-
-    /// An escape hatch to allow "casting" a generic Timeline to LayeredTimeline.
-    fn upgrade_to_layered_timeline(&self) -> &crate::layered_repository::LayeredTimeline;
 }
 
 /// Various functions to mutate the timeline.
@@ -260,25 +171,13 @@ pub trait TimelineWriter: Deref<Target = dyn Timeline> {
     ///
     /// This will implicitly extend the relation, if the page is beyond the
     /// current end-of-file.
-    fn put_wal_record(
-        &self,
-        lsn: Lsn,
-        tag: RelishTag,
-        blknum: BlockNumber,
-        rec: ZenithWalRecord,
-    ) -> Result<()>;
+    fn put_wal_record(&self, lsn: Lsn, tag: RelishTag, blknum: u32, rec: WALRecord) -> Result<()>;
 
     /// Like put_wal_record, but with ready-made image of the page.
-    fn put_page_image(
-        &self,
-        tag: RelishTag,
-        blknum: BlockNumber,
-        lsn: Lsn,
-        img: Bytes,
-    ) -> Result<()>;
+    fn put_page_image(&self, tag: RelishTag, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()>;
 
     /// Truncate relation
-    fn put_truncation(&self, rel: RelishTag, lsn: Lsn, nblocks: BlockNumber) -> Result<()>;
+    fn put_truncation(&self, rel: RelishTag, lsn: Lsn, nblocks: u32) -> Result<()>;
 
     /// This method is used for marking dropped relations and truncated SLRU files and aborted two phase records
     fn drop_relish(&self, tag: RelishTag, lsn: Lsn) -> Result<()>;
@@ -290,53 +189,44 @@ pub trait TimelineWriter: Deref<Target = dyn Timeline> {
     fn advance_last_record_lsn(&self, lsn: Lsn);
 }
 
-/// Each update to a page is represented by a ZenithWalRecord. It can be a wrapper
-/// around a PostgreSQL WAL record, or a custom zenith-specific "record".
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub enum ZenithWalRecord {
-    /// Native PostgreSQL WAL record
-    Postgres { will_init: bool, rec: Bytes },
-
-    /// Set bits in heap visibility map. (heap blkno, flag bits to clear)
-    ClearVisibilityMapFlags { heap_blkno: u32, flags: u8 },
-    /// Mark transaction IDs as committed on a CLOG page
-    ClogSetCommitted { xids: Vec<TransactionId> },
-    /// Mark transaction IDs as aborted on a CLOG page
-    ClogSetAborted { xids: Vec<TransactionId> },
-    /// Extend multixact offsets SLRU
-    MultixactOffsetCreate {
-        mid: MultiXactId,
-        moff: MultiXactOffset,
-    },
-    /// Extend multixact members SLRU.
-    MultixactMembersCreate {
-        moff: MultiXactOffset,
-        members: Vec<MultiXactMember>,
-    },
+pub struct WALRecord {
+    pub will_init: bool,
+    pub rec: Bytes,
+    // Remember the offset of main_data in rec,
+    // so that we don't have to parse the record again.
+    // If record has no main_data, this offset equals rec.len().
+    pub main_data_offset: u32,
 }
 
-impl ZenithWalRecord {
-    /// Does replaying this WAL record initialize the page from scratch, or does
-    /// it need to be applied over the previous image of the page?
-    pub fn will_init(&self) -> bool {
-        match self {
-            ZenithWalRecord::Postgres { will_init, rec: _ } => *will_init,
-
-            // None of the special zenith record types currently initialize the page
-            _ => false,
+impl WALRecord {
+    pub fn pack(&self, buf: &mut BytesMut) {
+        buf.put_u8(self.will_init as u8);
+        buf.put_u32(self.main_data_offset);
+        buf.put_u32(self.rec.len() as u32);
+        buf.put_slice(&self.rec[..]);
+    }
+    pub fn unpack(buf: &mut Bytes) -> WALRecord {
+        let will_init = buf.get_u8() != 0;
+        let main_data_offset = buf.get_u32();
+        let rec_len = buf.get_u32() as usize;
+        let rec = buf.split_to(rec_len);
+        WALRecord {
+            will_init,
+            rec,
+            main_data_offset,
         }
     }
 }
 
 #[cfg(test)]
 pub mod repo_harness {
-    use bytes::BytesMut;
     use std::{fs, path::PathBuf};
 
     use crate::{
-        config::PageServerConf,
         layered_repository::{LayeredRepository, TIMELINES_SEGMENT_NAME},
         walredo::{WalRedoError, WalRedoManager},
+        PageServerConf,
     };
 
     use super::*;
@@ -377,7 +267,6 @@ pub mod repo_harness {
 
             let tenant_id = ZTenantId::generate();
             fs::create_dir_all(conf.tenant_path(&tenant_id))?;
-            fs::create_dir_all(conf.branches_path(&tenant_id))?;
 
             Ok(Self { conf, tenant_id })
         }
@@ -405,10 +294,10 @@ pub mod repo_harness {
         fn request_redo(
             &self,
             rel: RelishTag,
-            blknum: BlockNumber,
+            blknum: u32,
             lsn: Lsn,
             base_img: Option<Bytes>,
-            records: Vec<(Lsn, ZenithWalRecord)>,
+            records: Vec<(Lsn, WALRecord)>,
         ) -> Result<Bytes, WalRedoError> {
             let s = format!(
                 "redo for {} blk {} to get to {}, with {} and {} records",
@@ -439,16 +328,14 @@ mod tests {
     use super::repo_harness::*;
     use super::*;
     use postgres_ffi::{pg_constants, xlog_utils::SIZEOF_CHECKPOINT};
-    use std::fs;
 
     /// Arbitrary relation tag, for testing.
-    const TESTREL_A_REL_TAG: RelTag = RelTag {
+    const TESTREL_A: RelishTag = RelishTag::Relation(RelTag {
         spcnode: 0,
         dbnode: 111,
         relnode: 1000,
         forknum: 0,
-    };
-    const TESTREL_A: RelishTag = RelishTag::Relation(TESTREL_A_REL_TAG);
+    });
     const TESTREL_B: RelishTag = RelishTag::Relation(RelTag {
         spcnode: 0,
         dbnode: 111,
@@ -474,7 +361,7 @@ mod tests {
         //repo.get_timeline("11223344556677881122334455667788");
 
         // Create timeline to work on
-        let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?;
+        let tline = repo.create_empty_timeline(TIMELINE_ID)?;
         let writer = tline.writer();
 
         writer.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?;
@@ -592,7 +479,7 @@ mod tests {
         let repo = RepoHarness::create("test_drop_extend")?.load();
 
         // Create timeline to work on
-        let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?;
+        let tline = repo.create_empty_timeline(TIMELINE_ID)?;
         let writer = tline.writer();
 
         writer.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?;
@@ -629,7 +516,7 @@ mod tests {
         let repo = RepoHarness::create("test_truncate_extend")?.load();
 
         // Create timeline to work on
-        let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?;
+        let tline = repo.create_empty_timeline(TIMELINE_ID)?;
         let writer = tline.writer();
 
         //from storage_layer.rs
@@ -728,14 +615,14 @@ mod tests {
     #[test]
     fn test_large_rel() -> Result<()> {
         let repo = RepoHarness::create("test_large_rel")?.load();
-        let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?;
+        let tline = repo.create_empty_timeline(TIMELINE_ID)?;
         let writer = tline.writer();
 
         let mut lsn = 0x10;
         for blknum in 0..pg_constants::RELSEG_SIZE + 1 {
             let img = TEST_IMG(&format!("foo blk {} at {}", blknum, Lsn(lsn)));
             lsn += 0x10;
-            writer.put_page_image(TESTREL_A, blknum as BlockNumber, Lsn(lsn), img)?;
+            writer.put_page_image(TESTREL_A, blknum as u32, Lsn(lsn), img)?;
         }
         writer.advance_last_record_lsn(Lsn(lsn));
 
@@ -771,11 +658,11 @@ mod tests {
         let mut size: i32 = 3000;
         while size >= 0 {
             lsn += 0x10;
-            writer.put_truncation(TESTREL_A, Lsn(lsn), size as BlockNumber)?;
+            writer.put_truncation(TESTREL_A, Lsn(lsn), size as u32)?;
             writer.advance_last_record_lsn(Lsn(lsn));
             assert_eq!(
                 tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(),
-                size as BlockNumber
+                size as u32
             );
 
             size -= 1;
@@ -791,7 +678,7 @@ mod tests {
     #[test]
     fn test_list_rels_drop() -> Result<()> {
         let repo = RepoHarness::create("test_list_rels_drop")?.load();
-        let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?;
+        let tline = repo.create_empty_timeline(TIMELINE_ID)?;
         let writer = tline.writer();
         const TESTDB: u32 = 111;
 
@@ -811,10 +698,7 @@ mod tests {
 
         // Create a branch, check that the relation is visible there
         repo.branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Lsn(0x30))?;
-        let newtline = match repo.get_timeline(NEW_TIMELINE_ID)?.local_timeline() {
-            Some(timeline) => timeline,
-            None => panic!("Should have a local timeline"),
-        };
+        let newtline = repo.get_timeline(NEW_TIMELINE_ID)?;
         let new_writer = newtline.writer();
 
         assert!(newtline
@@ -852,7 +736,7 @@ mod tests {
     #[test]
     fn test_branch() -> Result<()> {
         let repo = RepoHarness::create("test_branch")?.load();
-        let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?;
+        let tline = repo.create_empty_timeline(TIMELINE_ID)?;
         let writer = tline.writer();
 
         // Import initial dummy checkpoint record, otherwise the get_timeline() call
@@ -872,10 +756,7 @@ mod tests {
 
         // Branch the history, modify relation differently on the new timeline
         repo.branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Lsn(0x30))?;
-        let newtline = match repo.get_timeline(NEW_TIMELINE_ID)?.local_timeline() {
-            Some(timeline) => timeline,
-            None => panic!("Should have a local timeline"),
-        };
+        let newtline = repo.get_timeline(NEW_TIMELINE_ID)?;
         let new_writer = newtline.writer();
 
         new_writer.put_page_image(TESTREL_A, 0, Lsn(0x40), TEST_IMG("bar blk 0 at 4"))?;
@@ -904,230 +785,13 @@ mod tests {
         Ok(())
     }
 
-    fn make_some_layers(tline: &Arc<dyn Timeline>, start_lsn: Lsn) -> Result<()> {
-        let mut lsn = start_lsn;
-        {
-            let writer = tline.writer();
-            // Create a relation on the timeline
-            writer.put_page_image(
-                TESTREL_A,
-                0,
-                lsn,
-                TEST_IMG(&format!("foo blk 0 at {}", lsn)),
-            )?;
-            lsn += 0x10;
-            writer.put_page_image(
-                TESTREL_A,
-                0,
-                lsn,
-                TEST_IMG(&format!("foo blk 0 at {}", lsn)),
-            )?;
-            writer.advance_last_record_lsn(lsn);
-        }
-        tline.checkpoint(CheckpointConfig::Forced)?;
-        {
-            let writer = tline.writer();
-            lsn += 0x10;
-            writer.put_page_image(
-                TESTREL_A,
-                0,
-                lsn,
-                TEST_IMG(&format!("foo blk 0 at {}", lsn)),
-            )?;
-            lsn += 0x10;
-            writer.put_page_image(
-                TESTREL_A,
-                0,
-                lsn,
-                TEST_IMG(&format!("foo blk 0 at {}", lsn)),
-            )?;
-            writer.advance_last_record_lsn(lsn);
-        }
-        tline.checkpoint(CheckpointConfig::Forced)
-    }
-
-    #[test]
-    fn test_prohibit_branch_creation_on_garbage_collected_data() -> Result<()> {
-        let repo =
-            RepoHarness::create("test_prohibit_branch_creation_on_garbage_collected_data")?.load();
-
-        let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?;
-        make_some_layers(&tline, Lsn(0x20))?;
-
-        // this removes layers before lsn 40 (50 minus 10), so there are two remaining layers, image and delta for 31-50
-        repo.gc_iteration(Some(TIMELINE_ID), 0x10, false)?;
-
-        // try to branch at lsn 25, should fail because we already garbage collected the data
-        match repo.branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Lsn(0x25)) {
-            Ok(_) => panic!("branching should have failed"),
-            Err(err) => {
-                assert!(err.to_string().contains("invalid branch start lsn"));
-                assert!(err
-                    .source()
-                    .unwrap()
-                    .to_string()
-                    .contains("we might've already garbage collected needed data"))
-            }
-        }
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_prohibit_branch_creation_on_pre_initdb_lsn() -> Result<()> {
-        let repo = RepoHarness::create("test_prohibit_branch_creation_on_pre_initdb_lsn")?.load();
-
-        repo.create_empty_timeline(TIMELINE_ID, Lsn(0x50))?;
-        // try to branch at lsn 0x25, should fail because initdb lsn is 0x50
-        match repo.branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Lsn(0x25)) {
-            Ok(_) => panic!("branching should have failed"),
-            Err(err) => {
-                assert!(&err.to_string().contains("invalid branch start lsn"));
-                assert!(&err
-                    .source()
-                    .unwrap()
-                    .to_string()
-                    .contains("is earlier than initdb lsn"));
-            }
-        }
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_prohibit_get_page_at_lsn_for_garbage_collected_pages() -> Result<()> {
-        let repo =
-            RepoHarness::create("test_prohibit_get_page_at_lsn_for_garbage_collected_pages")?
-                .load();
-
-        let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?;
-        make_some_layers(&tline, Lsn(0x20))?;
-
-        repo.gc_iteration(Some(TIMELINE_ID), 0x10, false)?;
-
-        match tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x25)) {
-            Ok(_) => panic!("request for page should have failed"),
-            Err(err) => assert!(err
-                .to_string()
-                .contains("tried to request a page version that was garbage collected")),
-        }
-        Ok(())
-    }
-
-    #[test]
-    fn test_retain_data_in_parent_which_is_needed_for_child() -> Result<()> {
-        let repo =
-            RepoHarness::create("test_retain_data_in_parent_which_is_needed_for_child")?.load();
-        let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?;
-
-        make_some_layers(&tline, Lsn(0x20))?;
-
-        repo.branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Lsn(0x40))?;
-        let newtline = match repo.get_timeline(NEW_TIMELINE_ID)?.local_timeline() {
-            Some(timeline) => timeline,
-            None => panic!("Should have a local timeline"),
-        };
-
-        // this removes layers before lsn 40 (50 minus 10), so there are two remaining layers, image and delta for 31-50
-        repo.gc_iteration(Some(TIMELINE_ID), 0x10, false)?;
-        assert!(newtline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x25)).is_ok());
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_parent_keeps_data_forever_after_branching() -> Result<()> {
-        let harness = RepoHarness::create("test_parent_keeps_data_forever_after_branching")?;
-        let repo = harness.load();
-        let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?;
-
-        make_some_layers(&tline, Lsn(0x20))?;
-
-        repo.branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Lsn(0x40))?;
-        let newtline = match repo.get_timeline(NEW_TIMELINE_ID)?.local_timeline() {
-            Some(timeline) => timeline,
-            None => panic!("Should have a local timeline"),
-        };
-
-        make_some_layers(&newtline, Lsn(0x60))?;
-
-        // run gc on parent
-        repo.gc_iteration(Some(TIMELINE_ID), 0x10, false)?;
-
-        // check that the layer in parent before the branching point is still there
-        let tline_dir = harness.conf.timeline_path(&TIMELINE_ID, &harness.tenant_id);
-
-        let expected_image_layer_path = tline_dir.join(format!(
-            "rel_{}_{}_{}_{}_{}_{:016X}_{:016X}",
-            TESTREL_A_REL_TAG.spcnode,
-            TESTREL_A_REL_TAG.dbnode,
-            TESTREL_A_REL_TAG.relnode,
-            TESTREL_A_REL_TAG.forknum,
-            0, // seg is 0
-            0x20,
-            0x30,
-        ));
-        assert!(fs::metadata(&expected_image_layer_path).is_ok());
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_read_beyond_eof() -> Result<()> {
-        let harness = RepoHarness::create("test_read_beyond_eof")?;
-        let repo = harness.load();
-        let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?;
-
-        make_some_layers(&tline, Lsn(0x20))?;
-        {
-            let writer = tline.writer();
-            writer.put_page_image(
-                TESTREL_A,
-                0,
-                Lsn(0x60),
-                TEST_IMG(&format!("foo blk 0 at {}", Lsn(0x50))),
-            )?;
-            writer.advance_last_record_lsn(Lsn(0x60));
-        }
-
-        // Test read before rel creation. Should error out.
-        assert!(tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x10)).is_err());
-
-        // Read block beyond end of relation at different points in time.
-        // These reads should fall into different delta, image, and in-memory layers.
-        assert_eq!(tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x20))?, ZERO_PAGE);
-        assert_eq!(tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x25))?, ZERO_PAGE);
-        assert_eq!(tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x30))?, ZERO_PAGE);
-        assert_eq!(tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x35))?, ZERO_PAGE);
-        assert_eq!(tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x40))?, ZERO_PAGE);
-        assert_eq!(tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x45))?, ZERO_PAGE);
-        assert_eq!(tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x50))?, ZERO_PAGE);
-        assert_eq!(tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x55))?, ZERO_PAGE);
-        assert_eq!(tline.get_page_at_lsn(TESTREL_A, 1, Lsn(0x60))?, ZERO_PAGE);
-
-        // Test on an in-memory layer with no preceding layer
-        {
-            let writer = tline.writer();
-            writer.put_page_image(
-                TESTREL_B,
-                0,
-                Lsn(0x70),
-                TEST_IMG(&format!("foo blk 0 at {}", Lsn(0x70))),
-            )?;
-            writer.advance_last_record_lsn(Lsn(0x70));
-        }
-        assert_eq!(tline.get_page_at_lsn(TESTREL_B, 1, Lsn(0x70))?, ZERO_PAGE);
-
-        Ok(())
-    }
-
     #[test]
     fn corrupt_metadata() -> Result<()> {
         const TEST_NAME: &str = "corrupt_metadata";
         let harness = RepoHarness::create(TEST_NAME)?;
         let repo = harness.load();
 
-        repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?;
+        repo.create_empty_timeline(TIMELINE_ID)?;
         drop(repo);
 
         let metadata_path = harness.timeline_path(&TIMELINE_ID).join(METADATA_FILE_NAME);
@@ -1141,11 +805,7 @@ mod tests {
 
         let new_repo = harness.load();
         let err = new_repo.get_timeline(TIMELINE_ID).err().unwrap();
-        assert_eq!(err.to_string(), "failed to load metadata");
-        assert_eq!(
-            err.source().unwrap().to_string(),
-            "metadata checksum mismatch"
-        );
+        assert!(err.to_string().contains("checksum"));
 
         Ok(())
     }
@@ -1156,12 +816,7 @@ mod tests {
         let harness = RepoHarness::create(TEST_NAME)?;
         let repo = harness.load();
 
-        // Create a timeline with disk_consistent_lsn = 8000
-        let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0x8000))?;
-        let writer = tline.writer();
-        writer.advance_last_record_lsn(Lsn(0x8000));
-        drop(writer);
-        repo.checkpoint_iteration(CheckpointConfig::Forced)?;
+        repo.create_empty_timeline(TIMELINE_ID)?;
         drop(repo);
 
         let timeline_path = harness.timeline_path(&TIMELINE_ID);
@@ -1175,85 +830,38 @@ mod tests {
             Ok(())
         };
 
-        // Helper function to check that a relation file exists, and a corresponding
-        // <filename>.0.old file does not.
-        let assert_exists = |filename: &str| {
-            let path = timeline_path.join(filename);
-            assert!(path.exists(), "file {} was removed", filename);
+        let image_filename = format!("pg_control_0_{:016X}", 8000);
+        let delta_filename = format!("pg_control_0_{:016X}_{:016X}", 8000, 8008);
 
-            // Check that there is no .old file
-            let backup_path = timeline_path.join(format!("{}.0.old", filename));
-            assert!(
-                !backup_path.exists(),
-                "unexpected backup file {}",
-                backup_path.display()
-            );
-        };
+        make_empty_file(&image_filename)?;
+        make_empty_file(&delta_filename)?;
 
-        // Helper function to check that a relation file does *not* exists, and a corresponding
-        // <filename>.<num>.old file does.
-        let assert_is_renamed = |filename: &str, num: u32| {
+        let new_repo = harness.load();
+        new_repo.get_timeline(TIMELINE_ID).unwrap();
+        drop(new_repo);
+
+        let check_old = |filename: &str, num: u32| {
             let path = timeline_path.join(filename);
-            assert!(
-                !path.exists(),
-                "file {} was not removed as expected",
-                filename
-            );
+            assert!(!path.exists());
 
             let backup_path = timeline_path.join(format!("{}.{}.old", filename, num));
-            assert!(
-                backup_path.exists(),
-                "backup file {} was not created",
-                backup_path.display()
-            );
+            assert!(backup_path.exists());
         };
 
-        // These files are considered to be in the future and will be renamed out
-        // of the way
-        let future_filenames = vec![
-            format!("pg_control_0_{:016X}", 0x8001),
-            format!("pg_control_0_{:016X}_{:016X}", 0x8001, 0x8008),
-        ];
-        // But these are not:
-        let past_filenames = vec![
-            format!("pg_control_0_{:016X}", 0x8000),
-            format!("pg_control_0_{:016X}_{:016X}", 0x7000, 0x8001),
-        ];
+        check_old(&image_filename, 0);
+        check_old(&delta_filename, 0);
 
-        for filename in future_filenames.iter().chain(past_filenames.iter()) {
-            make_empty_file(filename)?;
-        }
-
-        // Load the timeline. This will cause the files in the "future" to be renamed
-        // away.
-        let new_repo = harness.load();
-        new_repo.get_timeline(TIMELINE_ID).unwrap();
-        drop(new_repo);
-
-        for filename in future_filenames.iter() {
-            assert_is_renamed(filename, 0);
-        }
-        for filename in past_filenames.iter() {
-            assert_exists(filename);
-        }
-
-        // Create the future files again, and load again. They should be renamed to
-        // *.1.old this time.
-        for filename in future_filenames.iter() {
-            make_empty_file(filename)?;
-        }
+        make_empty_file(&image_filename)?;
+        make_empty_file(&delta_filename)?;
 
         let new_repo = harness.load();
         new_repo.get_timeline(TIMELINE_ID).unwrap();
         drop(new_repo);
 
-        for filename in future_filenames.iter() {
-            assert_is_renamed(filename, 0);
-            assert_is_renamed(filename, 1);
-        }
-        for filename in past_filenames.iter() {
-            assert_exists(filename);
-        }
+        check_old(&image_filename, 0);
+        check_old(&delta_filename, 0);
+        check_old(&image_filename, 1);
+        check_old(&delta_filename, 1);
 
         Ok(())
     }

pageserver/src/tenant_mgr.rs

@@ -2,17 +2,19 @@
 //! page server.
 
 use crate::branches;
-use crate::config::PageServerConf;
 use crate::layered_repository::LayeredRepository;
-use crate::repository::{Repository, Timeline, TimelineSyncState};
+use crate::repository::{Repository, Timeline};
 use crate::tenant_threads;
 use crate::walredo::PostgresRedoManager;
+use crate::PageServerConf;
 use anyhow::{anyhow, bail, Context, Result};
 use lazy_static::lazy_static;
-use log::*;
+use log::{debug, info};
 use serde::{Deserialize, Serialize};
-use std::collections::{hash_map, HashMap};
+use std::collections::HashMap;
 use std::fmt;
+use std::fs;
+use std::str::FromStr;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex, MutexGuard};
 use zenith_utils::zid::{ZTenantId, ZTimelineId};
@@ -28,6 +30,11 @@ struct Tenant {
 
 #[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)]
 pub enum TenantState {
+    // This tenant only exists in cloud storage. It cannot be accessed.
+    CloudOnly,
+    // This tenant exists in cloud storage, and we are currently downloading it to local disk.
+    // It cannot be accessed yet, not until it's been fully downloaded to local disk.
+    Downloading,
     // All data for this tenant is complete on local disk, but we haven't loaded the Repository,
     // Timeline and Layer structs into memory yet, so it cannot be accessed yet.
     //Ready,
@@ -45,6 +52,8 @@ pub enum TenantState {
 impl fmt::Display for TenantState {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
+            TenantState::CloudOnly => f.write_str("CloudOnly"),
+            TenantState::Downloading => f.write_str("Downloading"),
             TenantState::Active => f.write_str("Active"),
             TenantState::Idle => f.write_str("Idle"),
             TenantState::Stopping => f.write_str("Stopping"),
@@ -58,69 +67,86 @@ fn access_tenants() -> MutexGuard<'static, HashMap<ZTenantId, Tenant>> {
 
 static SHUTDOWN_REQUESTED: AtomicBool = AtomicBool::new(false);
 
-/// Updates tenants' repositories, changing their timelines state in memory.
-pub fn set_timeline_states(
-    conf: &'static PageServerConf,
-    timeline_states: HashMap<ZTenantId, HashMap<ZTimelineId, TimelineSyncState>>,
-) {
-    if timeline_states.is_empty() {
-        debug!("no timeline state updates to perform");
-        return;
-    }
+pub fn init(conf: &'static PageServerConf) {
+    for dir_entry in fs::read_dir(conf.tenants_path()).unwrap() {
+        let tenantid =
+            ZTenantId::from_str(dir_entry.unwrap().file_name().to_str().unwrap()).unwrap();
 
-    info!("Updating states for {} timelines", timeline_states.len());
-    trace!("States: {:?}", timeline_states);
-
-    let mut m = access_tenants();
-    for (tenant_id, timeline_states) in timeline_states {
-        let tenant = m.entry(tenant_id).or_insert_with(|| Tenant {
-            state: TenantState::Idle,
-            repo: None,
-        });
-        if let Err(e) = put_timelines_into_tenant(conf, tenant, tenant_id, timeline_states) {
-            error!(
-                "Failed to update timeline states for tenant {}: {:#}",
-                tenant_id, e
-            );
+        {
+            let mut m = access_tenants();
+            let tenant = Tenant {
+                state: TenantState::CloudOnly,
+                repo: None,
+            };
+            m.insert(tenantid, tenant);
         }
+
+        init_repo(conf, tenantid);
+        info!("initialized storage for tenant: {}", &tenantid);
     }
 }
 
-fn put_timelines_into_tenant(
+fn init_repo(conf: &'static PageServerConf, tenant_id: ZTenantId) {
+    // Set up a WAL redo manager, for applying WAL records.
+    let walredo_mgr = PostgresRedoManager::new(conf, tenant_id);
+
+    // Set up an object repository, for actual data storage.
+    let repo = Arc::new(LayeredRepository::new(
+        conf,
+        Arc::new(walredo_mgr),
+        tenant_id,
+        true,
+    ));
+
+    let mut m = access_tenants();
+    let tenant = m.get_mut(&tenant_id).unwrap();
+    tenant.repo = Some(repo);
+    tenant.state = TenantState::Active;
+
+    // TODO Start these threads only if tenant actively receives some WAL
+    tenant_threads::start_tenant_threads(conf, tenant_id);
+}
+
+pub fn register_relish_download(
     conf: &'static PageServerConf,
-    tenant: &mut Tenant,
     tenant_id: ZTenantId,
-    timeline_states: HashMap<ZTimelineId, TimelineSyncState>,
-) -> anyhow::Result<()> {
-    let repo = match tenant.repo.as_ref() {
-        Some(repo) => Arc::clone(repo),
-        None => {
-            // Set up a WAL redo manager, for applying WAL records.
-            let walredo_mgr = PostgresRedoManager::new(conf, tenant_id);
+    timeline_id: ZTimelineId,
+) {
+    log::info!(
+        "Registering new download, tenant id {}, timeline id: {}",
+        tenant_id,
+        timeline_id
+    );
 
-            // Set up an object repository, for actual data storage.
-            let repo: Arc<dyn Repository> = Arc::new(LayeredRepository::new(
-                conf,
-                Arc::new(walredo_mgr),
-                tenant_id,
-                conf.remote_storage_config.is_some(),
-            ));
-            tenant.repo = Some(Arc::clone(&repo));
-            repo
+    {
+        let mut m = access_tenants();
+        let tenant = m.entry(tenant_id).or_insert_with(|| Tenant {
+            state: TenantState::Downloading,
+            repo: None,
+        });
+        tenant.state = TenantState::Downloading;
+        match &tenant.repo {
+            Some(repo) => {
+                init_timeline(repo.as_ref(), timeline_id);
+                tenant.state = TenantState::Active;
+                return;
+            }
+            None => log::warn!("Initialize new repo"),
         }
-    };
-
-    for (timeline_id, timeline_state) in timeline_states {
-        repo.set_timeline_state(timeline_id, timeline_state)
-            .with_context(|| {
-                format!(
-                    "Failed to update timeline {} state to {:?}",
-                    timeline_id, timeline_state
-                )
-            })?;
+        tenant.state = TenantState::Active;
     }
 
-    Ok(())
+    // init repo updates Tenant state
+    init_repo(conf, tenant_id);
+    let new_repo = get_repository_for_tenant(tenant_id).unwrap();
+    init_timeline(new_repo.as_ref(), timeline_id);
+}
+
+fn init_timeline(repo: &dyn Repository, timeline_id: ZTimelineId) {
+    match repo.get_timeline(timeline_id) {
+        Ok(_timeline) => log::info!("Successfully initialized timeline {}", timeline_id),
+        Err(e) => log::error!("Failed to init timeline {}, reason: {:#}", timeline_id, e),
+    }
 }
 
 // Check this flag in the thread loops to know when to exit
@@ -152,41 +178,54 @@ pub fn create_repository_for_tenant(
     conf: &'static PageServerConf,
     tenantid: ZTenantId,
 ) -> Result<()> {
-    let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenantid));
-    let repo = Some(branches::create_repo(conf, tenantid, wal_redo_manager)?);
-
-    match access_tenants().entry(tenantid) {
-        hash_map::Entry::Occupied(_) => bail!("tenant {} already exists", tenantid),
-        hash_map::Entry::Vacant(v) => {
-            v.insert(Tenant {
-                state: TenantState::Idle,
-                repo,
-            });
+    {
+        let mut m = access_tenants();
+        // First check that the tenant doesn't exist already
+        if m.get(&tenantid).is_some() {
+            bail!("tenant {} already exists", tenantid);
         }
+        let tenant = Tenant {
+            state: TenantState::CloudOnly,
+            repo: None,
+        };
+        m.insert(tenantid, tenant);
     }
 
+    let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenantid));
+    let repo = branches::create_repo(conf, tenantid, wal_redo_manager)?;
+
+    let mut m = access_tenants();
+    let tenant = m.get_mut(&tenantid).unwrap();
+    tenant.repo = Some(repo);
+    tenant.state = TenantState::Active;
+
     Ok(())
 }
 
-pub fn get_tenant_state(tenantid: ZTenantId) -> Option<TenantState> {
-    Some(access_tenants().get(&tenantid)?.state)
+// If tenant is not found in the repository, return CloudOnly state
+pub fn get_tenant_state(tenantid: ZTenantId) -> TenantState {
+    let m = access_tenants();
+    match m.get(&tenantid) {
+        Some(tenant) => tenant.state,
+        None => TenantState::CloudOnly,
+    }
 }
 
-pub fn set_tenant_state(tenantid: ZTenantId, newstate: TenantState) -> Result<TenantState> {
+pub fn set_tenant_state(tenantid: ZTenantId, state: TenantState) -> Result<TenantState> {
     let mut m = access_tenants();
     let tenant = m.get_mut(&tenantid);
 
     match tenant {
         Some(tenant) => {
-            if newstate == TenantState::Idle && tenant.state != TenantState::Active {
+            if state == TenantState::Idle && tenant.state != TenantState::Active {
                 // Only Active tenant can become Idle
                 return Ok(tenant.state);
             }
-            info!("set_tenant_state: {} -> {}", tenant.state, newstate);
-            tenant.state = newstate;
+            info!("set_tenant_state: {} -> {}", tenant.state, state);
+            tenant.state = state;
             Ok(tenant.state)
         }
-        None => bail!("Tenant not found for id {}", tenantid),
+        None => bail!("Tenant not found for tenant {}", tenantid),
     }
 }
 
@@ -207,14 +246,13 @@ pub fn get_timeline_for_tenant(
     timelineid: ZTimelineId,
 ) -> Result<Arc<dyn Timeline>> {
     get_repository_for_tenant(tenantid)?
-        .get_timeline(timelineid)?
-        .local_timeline()
-        .ok_or_else(|| anyhow!("cannot fetch timeline {}", timelineid))
+        .get_timeline(timelineid)
+        .with_context(|| format!("cannot fetch timeline {}", timelineid))
 }
 
 fn list_tenantids() -> Result<Vec<ZTenantId>> {
-    access_tenants()
-        .iter()
+    let m = access_tenants();
+    m.iter()
         .map(|v| {
             let (tenantid, _) = v;
             Ok(*tenantid)
@@ -230,8 +268,8 @@ pub struct TenantInfo {
 }
 
 pub fn list_tenants() -> Result<Vec<TenantInfo>> {
-    access_tenants()
-        .iter()
+    let m = access_tenants();
+    m.iter()
         .map(|v| {
             let (id, tenant) = v;
             Ok(TenantInfo {

pageserver/src/tenant_threads.rs

@@ -1,9 +1,9 @@
 //! This module contains functions to serve per-tenant background processes,
 //! such as checkpointer and GC
-use crate::config::PageServerConf;
 use crate::tenant_mgr;
 use crate::tenant_mgr::TenantState;
 use crate::CheckpointConfig;
+use crate::PageServerConf;
 use anyhow::Result;
 use lazy_static::lazy_static;
 use std::collections::HashMap;
@@ -11,7 +11,6 @@ use std::sync::Mutex;
 use std::thread::JoinHandle;
 use std::time::Duration;
 use tracing::*;
-use zenith_metrics::{register_int_gauge_vec, IntGaugeVec};
 use zenith_utils::zid::ZTenantId;
 
 struct TenantHandleEntry {
@@ -26,44 +25,33 @@ lazy_static! {
         Mutex::new(HashMap::new());
 }
 
-lazy_static! {
-    static ref TENANT_THREADS_COUNT: IntGaugeVec = register_int_gauge_vec!(
-        "tenant_threads_count",
-        "Number of live tenant threads",
-        &["tenant_thread_type"]
-    )
-    .expect("failed to define a metric");
-}
-
-// Launch checkpointer and GC for the tenant.
-// It's possible that the threads are running already,
-// if so, just don't spawn new ones.
 pub fn start_tenant_threads(conf: &'static PageServerConf, tenantid: ZTenantId) {
+    //ensure that old threads are stopeed
+    wait_for_tenant_threads_to_stop(tenantid);
+
+    let checkpointer_handle = std::thread::Builder::new()
+        .name("Checkpointer thread".into())
+        .spawn(move || {
+            checkpoint_loop(tenantid, conf).expect("Checkpointer thread died");
+        })
+        .ok();
+
+    let gc_handle = std::thread::Builder::new()
+        .name("GC thread".into())
+        .spawn(move || {
+            gc_loop(tenantid, conf).expect("GC thread died");
+        })
+        .ok();
+
+    // TODO handle thread errors if any
+
     let mut handles = TENANT_HANDLES.lock().unwrap();
-    let h = handles
-        .entry(tenantid)
-        .or_insert_with(|| TenantHandleEntry {
-            checkpointer_handle: None,
-            gc_handle: None,
-        });
+    let h = TenantHandleEntry {
+        checkpointer_handle,
+        gc_handle,
+    };
 
-    if h.checkpointer_handle.is_none() {
-        h.checkpointer_handle = std::thread::Builder::new()
-            .name("Checkpointer thread".into())
-            .spawn(move || {
-                checkpoint_loop(tenantid, conf).expect("Checkpointer thread died");
-            })
-            .ok();
-    }
-
-    if h.gc_handle.is_none() {
-        h.gc_handle = std::thread::Builder::new()
-            .name("GC thread".into())
-            .spawn(move || {
-                gc_loop(tenantid, conf).expect("GC thread died");
-            })
-            .ok();
-    }
+    handles.insert(tenantid, h);
 }
 
 pub fn wait_for_tenant_threads_to_stop(tenantid: ZTenantId) {
@@ -81,14 +69,8 @@ pub fn wait_for_tenant_threads_to_stop(tenantid: ZTenantId) {
 /// Checkpointer thread's main loop
 ///
 fn checkpoint_loop(tenantid: ZTenantId, conf: &'static PageServerConf) -> Result<()> {
-    let gauge = TENANT_THREADS_COUNT.with_label_values(&["checkpointer"]);
-    gauge.inc();
-    scopeguard::defer! {
-        gauge.dec();
-    }
-
     loop {
-        if tenant_mgr::get_tenant_state(tenantid) != Some(TenantState::Active) {
+        if tenant_mgr::get_tenant_state(tenantid) != TenantState::Active {
             break;
         }
 
@@ -102,7 +84,7 @@ fn checkpoint_loop(tenantid: ZTenantId, conf: &'static PageServerConf) -> Result
     }
 
     trace!(
-        "checkpointer thread stopped for tenant {} state is {:?}",
+        "checkpointer thread stopped for tenant {} state is {}",
         tenantid,
         tenant_mgr::get_tenant_state(tenantid)
     );
@@ -113,14 +95,8 @@ fn checkpoint_loop(tenantid: ZTenantId, conf: &'static PageServerConf) -> Result
 /// GC thread's main loop
 ///
 fn gc_loop(tenantid: ZTenantId, conf: &'static PageServerConf) -> Result<()> {
-    let gauge = TENANT_THREADS_COUNT.with_label_values(&["gc"]);
-    gauge.inc();
-    scopeguard::defer! {
-        gauge.dec();
-    }
-
     loop {
-        if tenant_mgr::get_tenant_state(tenantid) != Some(TenantState::Active) {
+        if tenant_mgr::get_tenant_state(tenantid) != TenantState::Active {
             break;
         }
 
@@ -135,14 +111,13 @@ fn gc_loop(tenantid: ZTenantId, conf: &'static PageServerConf) -> Result<()> {
         // TODO Write it in more adequate way using
         // condvar.wait_timeout() or something
         let mut sleep_time = conf.gc_period.as_secs();
-        while sleep_time > 0 && tenant_mgr::get_tenant_state(tenantid) == Some(TenantState::Active)
-        {
+        while sleep_time > 0 && tenant_mgr::get_tenant_state(tenantid) == TenantState::Active {
             sleep_time -= 1;
             std::thread::sleep(Duration::from_secs(1));
         }
     }
     trace!(
-        "GC thread stopped for tenant {} state is {:?}",
+        "GC thread stopped for tenant {} state is {}",
         tenantid,
         tenant_mgr::get_tenant_state(tenantid)
     );

pageserver/src/virtual_file.rs

@@ -1,619 +0,0 @@
-//!
-//! VirtualFile is like a normal File, but it's not bound directly to
-//! a file descriptor. Instead, the file is opened when it's read from,
-//! and if too many files are open globally in the system, least-recently
-//! used ones are closed.
-//!
-//! To track which files have been recently used, we use the clock algorithm
-//! with a 'recently_used' flag on each slot.
-//!
-//! This is similar to PostgreSQL's virtual file descriptor facility in
-//! src/backend/storage/file/fd.c
-//!
-use std::fs::{File, OpenOptions};
-use std::io::{Error, ErrorKind, Read, Seek, SeekFrom, Write};
-use std::os::unix::fs::FileExt;
-use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
-use std::sync::{RwLock, RwLockWriteGuard};
-
-use once_cell::sync::OnceCell;
-
-///
-/// A virtual file descriptor. You can use this just like std::fs::File, but internally
-/// the underlying file is closed if the system is low on file descriptors,
-/// and re-opened when it's accessed again.
-///
-/// Like with std::fs::File, multiple threads can read/write the file concurrently,
-/// holding just a shared reference the same VirtualFile, using the read_at() / write_at()
-/// functions from the FileExt trait. But the functions from the Read/Write/Seek traits
-/// require a mutable reference, because they modify the "current position".
-///
-/// Each VirtualFile has a physical file descriptor in the global OPEN_FILES array, at the
-/// slot that 'handle points to, if the underlying file is currently open. If it's not
-/// currently open, the 'handle' can still point to the slot where it was last kept. The
-/// 'tag' field is used to detect whether the handle still is valid or not.
-///
-pub struct VirtualFile {
-    /// Lazy handle to the global file descriptor cache. The slot that this points to
-    /// might contain our File, or it may be empty, or it may contain a File that
-    /// belongs to a different VirtualFile.
-    handle: RwLock<SlotHandle>,
-
-    /// Current file position
-    pos: u64,
-
-    /// File path and options to use to open it.
-    ///
-    /// Note: this only contains the options needed to re-open it. For example,
-    /// if a new file is created, we only pass the create flag when it's initially
-    /// opened, in the VirtualFile::create() function, and strip the flag before
-    /// storing it here.
-    pub path: PathBuf,
-    open_options: OpenOptions,
-}
-
-#[derive(PartialEq, Clone, Copy)]
-struct SlotHandle {
-    /// Index into OPEN_FILES.slots
-    index: usize,
-
-    /// Value of 'tag' in the slot. If slot's tag doesn't match, then the slot has
-    /// been recycled and no longer contains the FD for this virtual file.
-    tag: u64,
-}
-
-/// OPEN_FILES is the global array that holds the physical file descriptors that
-/// are currently open. Each slot in the array is protected by a separate lock,
-/// so that different files can be accessed independently. The lock must be held
-/// in write mode to replace the slot with a different file, but a read mode
-/// is enough to operate on the file, whether you're reading or writing to it.
-///
-/// OPEN_FILES starts in uninitialized state, and it's initialized by
-/// the virtual_file::init() function. It must be called exactly once at page
-/// server startup.
-static OPEN_FILES: OnceCell<OpenFiles> = OnceCell::new();
-
-struct OpenFiles {
-    slots: &'static [Slot],
-
-    /// clock arm for the clock algorithm
-    next: AtomicUsize,
-}
-
-struct Slot {
-    inner: RwLock<SlotInner>,
-
-    /// has this file been used since last clock sweep?
-    recently_used: AtomicBool,
-}
-
-struct SlotInner {
-    /// Counter that's incremented every time a different file is stored here.
-    /// To avoid the ABA problem.
-    tag: u64,
-
-    /// the underlying file
-    file: Option<File>,
-}
-
-impl OpenFiles {
-    /// Find a slot to use, evicting an existing file descriptor if needed.
-    ///
-    /// On return, we hold a lock on the slot, and its 'tag' has been updated
-    /// recently_used has been set. It's all ready for reuse.
-    fn find_victim_slot(&self) -> (SlotHandle, RwLockWriteGuard<SlotInner>) {
-        //
-        // Run the clock algorithm to find a slot to replace.
-        //
-        let num_slots = self.slots.len();
-        let mut retries = 0;
-        let mut slot;
-        let mut slot_guard;
-        let index;
-        loop {
-            let next = self.next.fetch_add(1, Ordering::AcqRel) % num_slots;
-            slot = &self.slots[next];
-
-            // If the recently_used flag on this slot is set, continue the clock
-            // sweep. Otherwise try to use this slot. If we cannot acquire the
-            // lock, also continue the clock sweep.
-            //
-            // We only continue in this manner for a while, though. If we loop
-            // through the array twice without finding a victim, just pick the
-            // next slot and wait until we can reuse it. This way, we avoid
-            // spinning in the extreme case that all the slots are busy with an
-            // I/O operation.
-            if retries < num_slots * 2 {
-                if !slot.recently_used.swap(false, Ordering::Release) {
-                    if let Ok(guard) = slot.inner.try_write() {
-                        slot_guard = guard;
-                        index = next;
-                        break;
-                    }
-                }
-                retries += 1;
-            } else {
-                slot_guard = slot.inner.write().unwrap();
-                index = next;
-                break;
-            }
-        }
-
-        //
-        // We now have the victim slot locked. If it was in use previously, close the
-        // old file.
-        //
-        if let Some(old_file) = slot_guard.file.take() {
-            drop(old_file);
-        }
-
-        // Prepare the slot for reuse and return it
-        slot_guard.tag += 1;
-        slot.recently_used.store(true, Ordering::Relaxed);
-        (
-            SlotHandle {
-                index,
-                tag: slot_guard.tag,
-            },
-            slot_guard,
-        )
-    }
-}
-
-impl VirtualFile {
-    /// Open a file in read-only mode. Like File::open.
-    pub fn open(path: &Path) -> Result<VirtualFile, std::io::Error> {
-        Self::open_with_options(path, OpenOptions::new().read(true))
-    }
-
-    /// Create a new file for writing. If the file exists, it will be truncated.
-    /// Like File::create.
-    pub fn create(path: &Path) -> Result<VirtualFile, std::io::Error> {
-        Self::open_with_options(
-            path,
-            OpenOptions::new().write(true).create(true).truncate(true),
-        )
-    }
-
-    /// Open a file with given options.
-    ///
-    /// Note: If any custom flags were set in 'open_options' through OpenOptionsExt,
-    /// they will be applied also when the file is subsequently re-opened, not only
-    /// on the first time. Make sure that's sane!
-    pub fn open_with_options(
-        path: &Path,
-        open_options: &OpenOptions,
-    ) -> Result<VirtualFile, std::io::Error> {
-        let (handle, mut slot_guard) = get_open_files().find_victim_slot();
-
-        let file = open_options.open(path)?;
-
-        // Strip all options other than read and write.
-        //
-        // It would perhaps be nicer to check just for the read and write flags
-        // explicitly, but OpenOptions doesn't contain any functions to read flags,
-        // only to set them.
-        let mut reopen_options = open_options.clone();
-        reopen_options.create(false);
-        reopen_options.create_new(false);
-        reopen_options.truncate(false);
-
-        let vfile = VirtualFile {
-            handle: RwLock::new(handle),
-            pos: 0,
-            path: path.to_path_buf(),
-            open_options: reopen_options,
-        };
-
-        slot_guard.file.replace(file);
-
-        Ok(vfile)
-    }
-
-    /// Call File::sync_all() on the underlying File.
-    pub fn sync_all(&self) -> Result<(), Error> {
-        self.with_file(|file| file.sync_all())?
-    }
-
-    /// Helper function that looks up the underlying File for this VirtualFile,
-    /// opening it and evicting some other File if necessary. It calls 'func'
-    /// with the physical File.
-    fn with_file<F, R>(&self, mut func: F) -> Result<R, Error>
-    where
-        F: FnMut(&File) -> R,
-    {
-        let open_files = get_open_files();
-
-        let mut handle_guard = {
-            // Read the cached slot handle, and see if the slot that it points to still
-            // contains our File.
-            //
-            // We only need to hold the handle lock while we read the current handle. If
-            // another thread closes the file and recycles the slot for a different file,
-            // we will notice that the handle we read is no longer valid and retry.
-            let mut handle = *self.handle.read().unwrap();
-            loop {
-                // Check if the slot contains our File
-                {
-                    let slot = &open_files.slots[handle.index];
-                    let slot_guard = slot.inner.read().unwrap();
-                    if slot_guard.tag == handle.tag {
-                        if let Some(file) = &slot_guard.file {
-                            // Found a cached file descriptor.
-                            slot.recently_used.store(true, Ordering::Relaxed);
-                            return Ok(func(file));
-                        }
-                    }
-                }
-
-                // The slot didn't contain our File. We will have to open it ourselves,
-                // but before that, grab a write lock on handle in the VirtualFile, so
-                // that no other thread will try to concurrently open the same file.
-                let handle_guard = self.handle.write().unwrap();
-
-                // If another thread changed the handle while we were not holding the lock,
-                // then the handle might now be valid again. Loop back to retry.
-                if *handle_guard != handle {
-                    handle = *handle_guard;
-                    continue;
-                }
-                break handle_guard;
-            }
-        };
-
-        // We need to open the file ourselves. The handle in the VirtualFile is
-        // now locked in write-mode. Find a free slot to put it in.
-        let (handle, mut slot_guard) = open_files.find_victim_slot();
-
-        // Open the physical file
-        let file = self.open_options.open(&self.path)?;
-
-        // Perform the requested operation on it
-        //
-        // TODO: We could downgrade the locks to read mode before calling
-        // 'func', to allow a little bit more concurrency, but the standard
-        // library RwLock doesn't allow downgrading without releasing the lock,
-        // and that doesn't seem worth the trouble. (parking_lot RwLock would
-        // allow it)
-        let result = func(&file);
-
-        // Store the File in the slot and update the handle in the VirtualFile
-        // to point to it.
-        slot_guard.file.replace(file);
-
-        *handle_guard = handle;
-
-        Ok(result)
-    }
-}
-
-impl Drop for VirtualFile {
-    /// If a VirtualFile is dropped, close the underlying file if it was open.
-    fn drop(&mut self) {
-        let handle = self.handle.get_mut().unwrap();
-
-        // We could check with a read-lock first, to avoid waiting on an
-        // unrelated I/O.
-        let slot = &get_open_files().slots[handle.index];
-        let mut slot_guard = slot.inner.write().unwrap();
-        if slot_guard.tag == handle.tag {
-            slot.recently_used.store(false, Ordering::Relaxed);
-            slot_guard.file.take();
-        }
-    }
-}
-
-impl Read for VirtualFile {
-    fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error> {
-        let pos = self.pos;
-        let n = self.read_at(buf, pos)?;
-        self.pos += n as u64;
-        Ok(n)
-    }
-}
-
-impl Write for VirtualFile {
-    fn write(&mut self, buf: &[u8]) -> Result<usize, std::io::Error> {
-        let pos = self.pos;
-        let n = self.write_at(buf, pos)?;
-        self.pos += n as u64;
-        Ok(n)
-    }
-
-    fn flush(&mut self) -> Result<(), std::io::Error> {
-        // flush is no-op for File (at least on unix), so we don't need to do
-        // anything here either.
-        Ok(())
-    }
-}
-
-impl Seek for VirtualFile {
-    fn seek(&mut self, pos: SeekFrom) -> Result<u64, Error> {
-        match pos {
-            SeekFrom::Start(offset) => {
-                self.pos = offset;
-            }
-            SeekFrom::End(offset) => {
-                self.pos = self.with_file(|mut file| file.seek(SeekFrom::End(offset)))??
-            }
-            SeekFrom::Current(offset) => {
-                let pos = self.pos as i128 + offset as i128;
-                if pos < 0 {
-                    return Err(Error::new(
-                        ErrorKind::InvalidInput,
-                        "offset would be negative",
-                    ));
-                }
-                if pos > u64::MAX as i128 {
-                    return Err(Error::new(ErrorKind::InvalidInput, "offset overflow"));
-                }
-                self.pos = pos as u64;
-            }
-        }
-        Ok(self.pos)
-    }
-}
-
-impl FileExt for VirtualFile {
-    fn read_at(&self, buf: &mut [u8], offset: u64) -> Result<usize, Error> {
-        self.with_file(|file| file.read_at(buf, offset))?
-    }
-
-    fn write_at(&self, buf: &[u8], offset: u64) -> Result<usize, Error> {
-        self.with_file(|file| file.write_at(buf, offset))?
-    }
-}
-
-impl OpenFiles {
-    fn new(num_slots: usize) -> OpenFiles {
-        let mut slots = Box::new(Vec::with_capacity(num_slots));
-        for _ in 0..num_slots {
-            let slot = Slot {
-                recently_used: AtomicBool::new(false),
-                inner: RwLock::new(SlotInner { tag: 0, file: None }),
-            };
-            slots.push(slot);
-        }
-
-        OpenFiles {
-            next: AtomicUsize::new(0),
-            slots: Box::leak(slots),
-        }
-    }
-}
-
-///
-/// Initialize the virtual file module. This must be called once at page
-/// server startup.
-///
-pub fn init(num_slots: usize) {
-    if OPEN_FILES.set(OpenFiles::new(num_slots)).is_err() {
-        panic!("virtual_file::init called twice");
-    }
-}
-
-const TEST_MAX_FILE_DESCRIPTORS: usize = 10;
-
-// Get a handle to the global slots array.
-fn get_open_files() -> &'static OpenFiles {
-    //
-    // In unit tests, page server startup doesn't happen and no one calls
-    // virtual_file::init(). Initialize it here, with a small array.
-    //
-    // This applies to the virtual file tests below, but all other unit
-    // tests too, so the virtual file facility is always usable in
-    // unit tests.
-    //
-    if cfg!(test) {
-        OPEN_FILES.get_or_init(|| OpenFiles::new(TEST_MAX_FILE_DESCRIPTORS))
-    } else {
-        OPEN_FILES.get().expect("virtual_file::init not called yet")
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use rand::seq::SliceRandom;
-    use rand::thread_rng;
-    use rand::Rng;
-    use std::sync::Arc;
-    use std::thread;
-
-    // Helper function to slurp contents of a file, starting at the current position,
-    // into a string
-    fn read_string<FD>(vfile: &mut FD) -> Result<String, Error>
-    where
-        FD: Read,
-    {
-        let mut buf = String::new();
-        vfile.read_to_string(&mut buf)?;
-        Ok(buf)
-    }
-
-    // Helper function to slurp a portion of a file into a string
-    fn read_string_at<FD>(vfile: &mut FD, pos: u64, len: usize) -> Result<String, Error>
-    where
-        FD: FileExt,
-    {
-        let mut buf = Vec::new();
-        buf.resize(len, 0);
-        vfile.read_exact_at(&mut buf, pos)?;
-        Ok(String::from_utf8(buf).unwrap())
-    }
-
-    #[test]
-    fn test_virtual_files() -> Result<(), Error> {
-        // The real work is done in the test_files() helper function. This
-        // allows us to run the same set of tests against a native File, and
-        // VirtualFile. We trust the native Files and wouldn't need to test them,
-        // but this allows us to verify that the operations return the same
-        // results with VirtualFiles as with native Files. (Except that with
-        // native files, you will run out of file descriptors if the ulimit
-        // is low enough.)
-        test_files("virtual_files", |path, open_options| {
-            VirtualFile::open_with_options(path, open_options)
-        })
-    }
-
-    #[test]
-    fn test_physical_files() -> Result<(), Error> {
-        test_files("physical_files", |path, open_options| {
-            open_options.open(path)
-        })
-    }
-
-    fn test_files<OF, FD>(testname: &str, openfunc: OF) -> Result<(), Error>
-    where
-        FD: Read + Write + Seek + FileExt,
-        OF: Fn(&Path, &OpenOptions) -> Result<FD, std::io::Error>,
-    {
-        let testdir = crate::config::PageServerConf::test_repo_dir(testname);
-        std::fs::create_dir_all(&testdir)?;
-
-        let path_a = testdir.join("file_a");
-        let mut file_a = openfunc(
-            &path_a,
-            OpenOptions::new().write(true).create(true).truncate(true),
-        )?;
-        file_a.write_all(b"foobar")?;
-
-        // cannot read from a file opened in write-only mode
-        assert!(read_string(&mut file_a).is_err());
-
-        // Close the file and re-open for reading
-        let mut file_a = openfunc(&path_a, OpenOptions::new().read(true))?;
-
-        // cannot write to a file opened in read-only mode
-        assert!(file_a.write(b"bar").is_err());
-
-        // Try simple read
-        assert_eq!("foobar", read_string(&mut file_a)?);
-
-        // It's positioned at the EOF now.
-        assert_eq!("", read_string(&mut file_a)?);
-
-        // Test seeks.
-        assert_eq!(file_a.seek(SeekFrom::Start(1))?, 1);
-        assert_eq!("oobar", read_string(&mut file_a)?);
-
-        assert_eq!(file_a.seek(SeekFrom::End(-2))?, 4);
-        assert_eq!("ar", read_string(&mut file_a)?);
-
-        assert_eq!(file_a.seek(SeekFrom::Start(1))?, 1);
-        assert_eq!(file_a.seek(SeekFrom::Current(2))?, 3);
-        assert_eq!("bar", read_string(&mut file_a)?);
-
-        assert_eq!(file_a.seek(SeekFrom::Current(-5))?, 1);
-        assert_eq!("oobar", read_string(&mut file_a)?);
-
-        // Test erroneous seeks to before byte 0
-        assert!(file_a.seek(SeekFrom::End(-7)).is_err());
-        assert_eq!(file_a.seek(SeekFrom::Start(1))?, 1);
-        assert!(file_a.seek(SeekFrom::Current(-2)).is_err());
-
-        // the erroneous seek should have left the position unchanged
-        assert_eq!("oobar", read_string(&mut file_a)?);
-
-        // Create another test file, and try FileExt functions on it.
-        let path_b = testdir.join("file_b");
-        let mut file_b = openfunc(
-            &path_b,
-            OpenOptions::new()
-                .read(true)
-                .write(true)
-                .create(true)
-                .truncate(true),
-        )?;
-        file_b.write_all_at(b"BAR", 3)?;
-        file_b.write_all_at(b"FOO", 0)?;
-
-        assert_eq!(read_string_at(&mut file_b, 2, 3)?, "OBA");
-
-        // Open a lot of files, enough to cause some evictions. (Or to be precise,
-        // open the same file many times. The effect is the same.)
-        //
-        // leave file_a positioned at offset 1 before we start
-        assert_eq!(file_a.seek(SeekFrom::Start(1))?, 1);
-
-        let mut vfiles = Vec::new();
-        for _ in 0..100 {
-            let mut vfile = openfunc(&path_b, OpenOptions::new().read(true))?;
-            assert_eq!("FOOBAR", read_string(&mut vfile)?);
-            vfiles.push(vfile);
-        }
-
-        // make sure we opened enough files to definitely cause evictions.
-        assert!(vfiles.len() > TEST_MAX_FILE_DESCRIPTORS * 2);
-
-        // The underlying file descriptor for 'file_a' should be closed now. Try to read
-        // from it again. We left the file positioned at offset 1 above.
-        assert_eq!("oobar", read_string(&mut file_a)?);
-
-        // Check that all the other FDs still work too. Use them in random order for
-        // good measure.
-        vfiles.as_mut_slice().shuffle(&mut thread_rng());
-        for vfile in vfiles.iter_mut() {
-            assert_eq!("OOBAR", read_string_at(vfile, 1, 5)?);
-        }
-
-        Ok(())
-    }
-
-    /// Test using VirtualFiles from many threads concurrently. This tests both using
-    /// a lot of VirtualFiles concurrently, causing evictions, and also using the same
-    /// VirtualFile from multiple threads concurrently.
-    #[test]
-    fn test_vfile_concurrency() -> Result<(), Error> {
-        const SIZE: usize = 8 * 1024;
-        const VIRTUAL_FILES: usize = 100;
-        const THREADS: usize = 100;
-        const SAMPLE: [u8; SIZE] = [0xADu8; SIZE];
-
-        let testdir = crate::config::PageServerConf::test_repo_dir("vfile_concurrency");
-        std::fs::create_dir_all(&testdir)?;
-
-        // Create a test file.
-        let test_file_path = testdir.join("concurrency_test_file");
-        {
-            let file = File::create(&test_file_path)?;
-            file.write_all_at(&SAMPLE, 0)?;
-        }
-
-        // Open the file many times.
-        let mut files = Vec::new();
-        for _ in 0..VIRTUAL_FILES {
-            let f = VirtualFile::open_with_options(&test_file_path, OpenOptions::new().read(true))?;
-            files.push(f);
-        }
-        let files = Arc::new(files);
-
-        // Launch many threads, and use the virtual files concurrently in random order.
-        let mut threads = Vec::new();
-        for threadno in 0..THREADS {
-            let builder =
-                thread::Builder::new().name(format!("test_vfile_concurrency thread {}", threadno));
-
-            let files = files.clone();
-            let thread = builder
-                .spawn(move || {
-                    let mut buf = [0u8; SIZE];
-                    let mut rng = rand::thread_rng();
-                    for _ in 1..1000 {
-                        let f = &files[rng.gen_range(0..files.len())];
-                        f.read_exact_at(&mut buf, 0).unwrap();
-                        assert!(buf == SAMPLE);
-                    }
-                })
-                .unwrap();
-            threads.push(thread);
-        }
-
-        for thread in threads {
-            thread.join().unwrap();
-        }
-
-        Ok(())
-    }
-}

pageserver/src/waldecoder.rs

@@ -1,18 +1,220 @@
 //!
-//! Functions for parsing WAL records.
+//! WAL decoder. For each WAL record, it decodes the record to figure out which data blocks
+//! the record affects, so that they can be stored in repository.
 //!
-use bytes::{Buf, Bytes};
+use bytes::{Buf, BufMut, Bytes, BytesMut};
+use crc32c::*;
+use log::*;
 use postgres_ffi::pg_constants;
-use postgres_ffi::xlog_utils::{TimestampTz, XLOG_SIZE_OF_XLOG_RECORD};
+use postgres_ffi::xlog_utils::*;
+use postgres_ffi::XLogLongPageHeaderData;
+use postgres_ffi::XLogPageHeaderData;
 use postgres_ffi::XLogRecord;
 use postgres_ffi::{BlockNumber, OffsetNumber};
 use postgres_ffi::{MultiXactId, MultiXactOffset, MultiXactStatus, Oid, TransactionId};
-use serde::{Deserialize, Serialize};
-use tracing::*;
+use std::cmp::min;
+use thiserror::Error;
+use zenith_utils::lsn::Lsn;
 
-use crate::repository::ZenithWalRecord;
+#[allow(dead_code)]
+pub struct WalStreamDecoder {
+    lsn: Lsn,
 
-/// DecodedBkpBlock represents per-page data contained in a WAL record.
+    startlsn: Lsn, // LSN where this record starts
+    contlen: u32,
+    padlen: u32,
+
+    inputbuf: BytesMut,
+
+    recordbuf: BytesMut,
+}
+
+#[derive(Error, Debug, Clone)]
+#[error("{msg} at {lsn}")]
+pub struct WalDecodeError {
+    msg: String,
+    lsn: Lsn,
+}
+
+//
+// WalRecordStream is a Stream that returns a stream of WAL records
+// FIXME: This isn't a proper rust stream
+//
+impl WalStreamDecoder {
+    pub fn new(lsn: Lsn) -> WalStreamDecoder {
+        WalStreamDecoder {
+            lsn,
+
+            startlsn: Lsn(0),
+            contlen: 0,
+            padlen: 0,
+
+            inputbuf: BytesMut::new(),
+            recordbuf: BytesMut::new(),
+        }
+    }
+
+    // The latest LSN position fed to the decoder.
+    pub fn available(&self) -> Lsn {
+        self.lsn + self.inputbuf.remaining() as u64
+    }
+
+    pub fn feed_bytes(&mut self, buf: &[u8]) {
+        self.inputbuf.extend_from_slice(buf);
+    }
+
+    /// Attempt to decode another WAL record from the input that has been fed to the
+    /// decoder so far.
+    ///
+    /// Returns one of the following:
+    ///     Ok((Lsn, Bytes)): a tuple containing the LSN of next record, and the record itself
+    ///     Ok(None): there is not enough data in the input buffer. Feed more by calling the `feed_bytes` function
+    ///     Err(WalDecodeError): an error occured while decoding, meaning the input was invalid.
+    ///
+    pub fn poll_decode(&mut self) -> Result<Option<(Lsn, Bytes)>, WalDecodeError> {
+        let recordbuf;
+
+        // Run state machine that validates page headers, and reassembles records
+        // that cross page boundaries.
+        loop {
+            // parse and verify page boundaries as we go
+            if self.lsn.segment_offset(pg_constants::WAL_SEGMENT_SIZE) == 0 {
+                // parse long header
+
+                if self.inputbuf.remaining() < XLOG_SIZE_OF_XLOG_LONG_PHD {
+                    return Ok(None);
+                }
+
+                let hdr = XLogLongPageHeaderData::from_bytes(&mut self.inputbuf);
+
+                if hdr.std.xlp_pageaddr != self.lsn.0 {
+                    return Err(WalDecodeError {
+                        msg: "invalid xlog segment header".into(),
+                        lsn: self.lsn,
+                    });
+                }
+                // TODO: verify the remaining fields in the header
+
+                self.lsn += XLOG_SIZE_OF_XLOG_LONG_PHD as u64;
+                continue;
+            } else if self.lsn.block_offset() == 0 {
+                if self.inputbuf.remaining() < XLOG_SIZE_OF_XLOG_SHORT_PHD {
+                    return Ok(None);
+                }
+
+                let hdr = XLogPageHeaderData::from_bytes(&mut self.inputbuf);
+
+                if hdr.xlp_pageaddr != self.lsn.0 {
+                    return Err(WalDecodeError {
+                        msg: "invalid xlog page header".into(),
+                        lsn: self.lsn,
+                    });
+                }
+                // TODO: verify the remaining fields in the header
+
+                self.lsn += XLOG_SIZE_OF_XLOG_SHORT_PHD as u64;
+                continue;
+            } else if self.padlen > 0 {
+                if self.inputbuf.remaining() < self.padlen as usize {
+                    return Ok(None);
+                }
+
+                // skip padding
+                self.inputbuf.advance(self.padlen as usize);
+                self.lsn += self.padlen as u64;
+                self.padlen = 0;
+            } else if self.contlen == 0 {
+                assert!(self.recordbuf.is_empty());
+
+                // need to have at least the xl_tot_len field
+                if self.inputbuf.remaining() < 4 {
+                    return Ok(None);
+                }
+
+                // peek xl_tot_len at the beginning of the record.
+                // FIXME: assumes little-endian
+                self.startlsn = self.lsn;
+                let xl_tot_len = (&self.inputbuf[0..4]).get_u32_le();
+                if (xl_tot_len as usize) < XLOG_SIZE_OF_XLOG_RECORD {
+                    return Err(WalDecodeError {
+                        msg: format!("invalid xl_tot_len {}", xl_tot_len),
+                        lsn: self.lsn,
+                    });
+                }
+
+                // Fast path for the common case that the whole record fits on the page.
+                let pageleft = self.lsn.remaining_in_block() as u32;
+                if self.inputbuf.remaining() >= xl_tot_len as usize && xl_tot_len <= pageleft {
+                    // Take the record from the 'inputbuf', and validate it.
+                    recordbuf = self.inputbuf.copy_to_bytes(xl_tot_len as usize);
+                    self.lsn += xl_tot_len as u64;
+                    break;
+                } else {
+                    // Need to assemble the record from pieces. Remember the size of the
+                    // record, and loop back. On next iteration, we will reach the 'else'
+                    // branch below, and copy the part of the record that was on this page
+                    // to 'recordbuf'.  Subsequent iterations will skip page headers, and
+                    // append the continuations from the next pages to 'recordbuf'.
+                    self.recordbuf.reserve(xl_tot_len as usize);
+                    self.contlen = xl_tot_len;
+                    continue;
+                }
+            } else {
+                // we're continuing a record, possibly from previous page.
+                let pageleft = self.lsn.remaining_in_block() as u32;
+
+                // read the rest of the record, or as much as fits on this page.
+                let n = min(self.contlen, pageleft) as usize;
+
+                if self.inputbuf.remaining() < n {
+                    return Ok(None);
+                }
+
+                self.recordbuf.put(self.inputbuf.split_to(n));
+                self.lsn += n as u64;
+                self.contlen -= n as u32;
+
+                if self.contlen == 0 {
+                    // The record is now complete.
+                    recordbuf = std::mem::replace(&mut self.recordbuf, BytesMut::new()).freeze();
+                    break;
+                }
+                continue;
+            }
+        }
+
+        // We now have a record in the 'recordbuf' local variable.
+        let xlogrec = XLogRecord::from_slice(&recordbuf[0..XLOG_SIZE_OF_XLOG_RECORD]);
+
+        let mut crc = 0;
+        crc = crc32c_append(crc, &recordbuf[XLOG_RECORD_CRC_OFFS + 4..]);
+        crc = crc32c_append(crc, &recordbuf[0..XLOG_RECORD_CRC_OFFS]);
+        if crc != xlogrec.xl_crc {
+            return Err(WalDecodeError {
+                msg: "WAL record crc mismatch".into(),
+                lsn: self.lsn,
+            });
+        }
+
+        // XLOG_SWITCH records are special. If we see one, we need to skip
+        // to the next WAL segment.
+        if xlogrec.is_xlog_switch_record() {
+            trace!("saw xlog switch record at {}", self.lsn);
+            self.padlen = self.lsn.calc_padding(pg_constants::WAL_SEGMENT_SIZE as u64) as u32;
+        } else {
+            // Pad to an 8-byte boundary
+            self.padlen = self.lsn.calc_padding(8u32) as u32;
+        }
+
+        // Always align resulting LSN on 0x8 boundary -- that is important for getPage()
+        // and WalReceiver integration. Since this code is used both for WalReceiver and
+        // initial WAL import let's force alignment right here.
+        let result = (self.lsn.align(), recordbuf);
+        Ok(Some(result))
+    }
+}
+
+#[allow(dead_code)]
 #[derive(Default)]
 pub struct DecodedBkpBlock {
     /* Is this block ref in use? */
@@ -27,18 +229,17 @@ pub struct DecodedBkpBlock {
     pub blkno: u32,
 
     /* copy of the fork_flags field from the XLogRecordBlockHeader */
-    pub flags: u8,
+    flags: u8,
 
     /* Information on full-page image, if any */
-    pub has_image: bool,   /* has image, even for consistency checking */
+    has_image: bool,       /* has image, even for consistency checking */
     pub apply_image: bool, /* has image that should be restored */
     pub will_init: bool,   /* record doesn't need previous page version to apply */
     //char	   *bkp_image;
-    pub hole_offset: u16,
-    pub hole_length: u16,
-    pub bimg_offset: u32,
-    pub bimg_len: u16,
-    pub bimg_info: u8,
+    hole_offset: u16,
+    hole_length: u16,
+    bimg_len: u16,
+    bimg_info: u8,
 
     /* Buffer holding the rmgr-specific data associated with this block */
     has_data: bool,
@@ -354,7 +555,7 @@ impl XlClogTruncate {
 }
 
 #[repr(C)]
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug)]
 pub struct MultiXactMember {
     pub xid: TransactionId,
     pub status: MultiXactStatus,
@@ -658,19 +859,8 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
     }
 
     // 3. Decode blocks.
-    let mut ptr = record.len() - buf.remaining();
-    for blk in blocks.iter_mut() {
-        if blk.has_image {
-            blk.bimg_offset = ptr as u32;
-            ptr += blk.bimg_len as usize;
-        }
-        if blk.has_data {
-            ptr += blk.data_len as usize;
-        }
-    }
     // We don't need them, so just skip blocks_total_len bytes
     buf.advance(blocks_total_len as usize);
-    assert_eq!(ptr, record.len() - buf.remaining());
 
     let main_data_offset = (xlogrec.xl_tot_len - main_data_len) as usize;
 
@@ -679,6 +869,97 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
         assert_eq!(buf.remaining(), main_data_len as usize);
     }
 
+    // 5. Handle a few special record types that modify blocks without registering
+    // them with the standard mechanism.
+    if xlogrec.xl_rmid == pg_constants::RM_HEAP_ID {
+        let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
+        let blkno = blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32;
+        if info == pg_constants::XLOG_HEAP_INSERT {
+            let xlrec = XlHeapInsert::decode(&mut buf);
+            assert_eq!(0, buf.remaining());
+            if (xlrec.flags
+                & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
+                    | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
+                != 0
+            {
+                let mut blk = DecodedBkpBlock::new();
+                blk.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
+                blk.blkno = blkno;
+                blk.rnode_spcnode = blocks[0].rnode_spcnode;
+                blk.rnode_dbnode = blocks[0].rnode_dbnode;
+                blk.rnode_relnode = blocks[0].rnode_relnode;
+                blocks.push(blk);
+            }
+        } else if info == pg_constants::XLOG_HEAP_DELETE {
+            let xlrec = XlHeapDelete::decode(&mut buf);
+            assert_eq!(0, buf.remaining());
+            if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 {
+                let mut blk = DecodedBkpBlock::new();
+                blk.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
+                blk.blkno = blkno;
+                blk.rnode_spcnode = blocks[0].rnode_spcnode;
+                blk.rnode_dbnode = blocks[0].rnode_dbnode;
+                blk.rnode_relnode = blocks[0].rnode_relnode;
+                blocks.push(blk);
+            }
+        } else if info == pg_constants::XLOG_HEAP_UPDATE
+            || info == pg_constants::XLOG_HEAP_HOT_UPDATE
+        {
+            let xlrec = XlHeapUpdate::decode(&mut buf);
+            // the size of tuple data is inferred from the size of the record.
+            // we can't validate the remaining number of bytes without parsing
+            // the tuple data.
+            if (xlrec.flags & pg_constants::XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0 {
+                let mut blk = DecodedBkpBlock::new();
+                blk.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
+                blk.blkno = blkno;
+                blk.rnode_spcnode = blocks[0].rnode_spcnode;
+                blk.rnode_dbnode = blocks[0].rnode_dbnode;
+                blk.rnode_relnode = blocks[0].rnode_relnode;
+                blocks.push(blk);
+            }
+            if (xlrec.flags & pg_constants::XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) != 0
+                && blocks.len() > 1
+            {
+                let mut blk = DecodedBkpBlock::new();
+                blk.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
+                blk.blkno = blocks[1].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32;
+                blk.rnode_spcnode = blocks[1].rnode_spcnode;
+                blk.rnode_dbnode = blocks[1].rnode_dbnode;
+                blk.rnode_relnode = blocks[1].rnode_relnode;
+                blocks.push(blk);
+            }
+        }
+    } else if xlogrec.xl_rmid == pg_constants::RM_HEAP2_ID {
+        let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
+        if info == pg_constants::XLOG_HEAP2_MULTI_INSERT {
+            let xlrec = XlHeapMultiInsert::decode(&mut buf);
+
+            let offset_array_len = if xlogrec.xl_info & pg_constants::XLOG_HEAP_INIT_PAGE > 0 {
+                // the offsets array is omitted if XLOG_HEAP_INIT_PAGE is set
+                0
+            } else {
+                std::mem::size_of::<u16>() * xlrec.ntuples as usize
+            };
+            assert_eq!(offset_array_len, buf.remaining());
+
+            if (xlrec.flags
+                & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
+                    | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
+                != 0
+            {
+                let mut blk = DecodedBkpBlock::new();
+                let blkno = blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32;
+                blk.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
+                blk.blkno = blkno;
+                blk.rnode_spcnode = blocks[0].rnode_spcnode;
+                blk.rnode_dbnode = blocks[0].rnode_dbnode;
+                blk.rnode_relnode = blocks[0].rnode_relnode;
+                blocks.push(blk);
+            }
+        }
+    }
+
     DecodedWALRecord {
         xl_xid: xlogrec.xl_xid,
         xl_info: xlogrec.xl_info,
@@ -693,20 +974,7 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
 /// Build a human-readable string to describe a WAL record
 ///
 /// For debugging purposes
-pub fn describe_wal_record(rec: &ZenithWalRecord) -> String {
-    match rec {
-        ZenithWalRecord::Postgres { will_init, rec } => {
-            format!(
-                "will_init: {}, {}",
-                will_init,
-                describe_postgres_wal_record(rec)
-            )
-        }
-        _ => format!("{:?}", rec),
-    }
-}
-
-fn describe_postgres_wal_record(record: &Bytes) -> String {
+pub fn describe_wal_record(record: &Bytes) -> String {
     // TODO: It would be nice to use the PostgreSQL rmgrdesc infrastructure for this.
     // Maybe use the postgres wal redo process, the same used for replaying WAL records?
     // Or could we compile the rmgrdesc routines into the dump_layer_file() binary directly,

pageserver/src/walingest.rs

@@ -1,906 +0,0 @@
-//!
-//! Parse PostgreSQL WAL records and store them in a zenith Timeline.
-//!
-//! The pipeline for ingesting WAL looks like this:
-//!
-//! WAL receiver  ->   WalIngest  ->   Repository
-//!
-//! The WAL receiver receives a stream of WAL from the WAL safekeepers,
-//! and decodes it to individual WAL records. It feeds the WAL records
-//! to WalIngest, which parses them and stores them in the Repository.
-//!
-//! The zenith Repository can store page versions in two formats: as
-//! page images, or a WAL records. WalIngest::ingest_record() extracts
-//! page images out of some WAL records, but most it stores as WAL
-//! records. If a WAL record modifies multple pages, WalIngest
-//! will call Repository::put_wal_record or put_page_image functions
-//! separately for each modified page.
-//!
-//! To reconstruct a page using a WAL record, the Repository calls the
-//! code in walredo.rs. walredo.rs passes most WAL records to the WAL
-//! redo Postgres process, but some records it can handle directly with
-//! bespoken Rust code.
-
-use postgres_ffi::nonrelfile_utils::clogpage_precedes;
-use postgres_ffi::nonrelfile_utils::slru_may_delete_clogsegment;
-use std::cmp::min;
-
-use anyhow::Result;
-use bytes::{Buf, Bytes, BytesMut};
-use tracing::*;
-
-use crate::relish::*;
-use crate::repository::*;
-use crate::walrecord::*;
-use postgres_ffi::nonrelfile_utils::mx_offset_to_member_segment;
-use postgres_ffi::xlog_utils::*;
-use postgres_ffi::TransactionId;
-use postgres_ffi::{pg_constants, CheckPoint};
-use zenith_utils::lsn::Lsn;
-
-static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
-
-pub struct WalIngest {
-    checkpoint: CheckPoint,
-    checkpoint_modified: bool,
-}
-
-impl WalIngest {
-    pub fn new(timeline: &dyn Timeline, startpoint: Lsn) -> Result<WalIngest> {
-        // Fetch the latest checkpoint into memory, so that we can compare with it
-        // quickly in `ingest_record` and update it when it changes.
-        let checkpoint_bytes = timeline.get_page_at_lsn(RelishTag::Checkpoint, 0, startpoint)?;
-        let checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
-        trace!("CheckPoint.nextXid = {}", checkpoint.nextXid.value);
-
-        Ok(WalIngest {
-            checkpoint,
-            checkpoint_modified: false,
-        })
-    }
-
-    ///
-    /// Decode a PostgreSQL WAL record and store it in the repository, in the given timeline.
-    ///
-    ///
-    /// Helper function to parse a WAL record and call the Timeline's PUT functions for all the
-    /// relations/pages that the record affects.
-    ///
-    pub fn ingest_record(
-        &mut self,
-        timeline: &dyn TimelineWriter,
-        recdata: Bytes,
-        lsn: Lsn,
-    ) -> Result<()> {
-        let mut decoded = decode_wal_record(recdata);
-        let mut buf = decoded.record.clone();
-        buf.advance(decoded.main_data_offset);
-
-        assert!(!self.checkpoint_modified);
-        if self.checkpoint.update_next_xid(decoded.xl_xid) {
-            self.checkpoint_modified = true;
-        }
-
-        // Heap AM records need some special handling, because they modify VM pages
-        // without registering them with the standard mechanism.
-        if decoded.xl_rmid == pg_constants::RM_HEAP_ID
-            || decoded.xl_rmid == pg_constants::RM_HEAP2_ID
-        {
-            self.ingest_heapam_record(&mut buf, timeline, lsn, &mut decoded)?;
-        }
-        // Handle other special record types
-        if decoded.xl_rmid == pg_constants::RM_SMGR_ID
-            && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK)
-                == pg_constants::XLOG_SMGR_TRUNCATE
-        {
-            let truncate = XlSmgrTruncate::decode(&mut buf);
-            self.ingest_xlog_smgr_truncate(timeline, lsn, &truncate)?;
-        } else if decoded.xl_rmid == pg_constants::RM_DBASE_ID {
-            if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK)
-                == pg_constants::XLOG_DBASE_CREATE
-            {
-                let createdb = XlCreateDatabase::decode(&mut buf);
-                self.ingest_xlog_dbase_create(timeline, lsn, &createdb)?;
-            } else if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK)
-                == pg_constants::XLOG_DBASE_DROP
-            {
-                let dropdb = XlDropDatabase::decode(&mut buf);
-
-                // To drop the database, we need to drop all the relations in it. Like in
-                // ingest_xlog_dbase_create(), use the previous record's LSN in the list_rels() call
-                let req_lsn = min(timeline.get_last_record_lsn(), lsn);
-
-                for tablespace_id in dropdb.tablespace_ids {
-                    let rels = timeline.list_rels(tablespace_id, dropdb.db_id, req_lsn)?;
-                    for rel in rels {
-                        timeline.drop_relish(rel, lsn)?;
-                    }
-                    trace!(
-                        "Drop FileNodeMap {}, {} at lsn {}",
-                        tablespace_id,
-                        dropdb.db_id,
-                        lsn
-                    );
-                    timeline.drop_relish(
-                        RelishTag::FileNodeMap {
-                            spcnode: tablespace_id,
-                            dbnode: dropdb.db_id,
-                        },
-                        lsn,
-                    )?;
-                }
-            }
-        } else if decoded.xl_rmid == pg_constants::RM_TBLSPC_ID {
-            trace!("XLOG_TBLSPC_CREATE/DROP is not handled yet");
-        } else if decoded.xl_rmid == pg_constants::RM_CLOG_ID {
-            let info = decoded.xl_info & !pg_constants::XLR_INFO_MASK;
-            if info == pg_constants::CLOG_ZEROPAGE {
-                let pageno = buf.get_u32_le();
-                let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-                let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-                timeline.put_page_image(
-                    RelishTag::Slru {
-                        slru: SlruKind::Clog,
-                        segno,
-                    },
-                    rpageno,
-                    lsn,
-                    ZERO_PAGE.clone(),
-                )?;
-            } else {
-                assert!(info == pg_constants::CLOG_TRUNCATE);
-                let xlrec = XlClogTruncate::decode(&mut buf);
-                self.ingest_clog_truncate_record(timeline, lsn, &xlrec)?;
-            }
-        } else if decoded.xl_rmid == pg_constants::RM_XACT_ID {
-            let info = decoded.xl_info & pg_constants::XLOG_XACT_OPMASK;
-            if info == pg_constants::XLOG_XACT_COMMIT || info == pg_constants::XLOG_XACT_ABORT {
-                let parsed_xact =
-                    XlXactParsedRecord::decode(&mut buf, decoded.xl_xid, decoded.xl_info);
-                self.ingest_xact_record(
-                    timeline,
-                    lsn,
-                    &parsed_xact,
-                    info == pg_constants::XLOG_XACT_COMMIT,
-                )?;
-            } else if info == pg_constants::XLOG_XACT_COMMIT_PREPARED
-                || info == pg_constants::XLOG_XACT_ABORT_PREPARED
-            {
-                let parsed_xact =
-                    XlXactParsedRecord::decode(&mut buf, decoded.xl_xid, decoded.xl_info);
-                self.ingest_xact_record(
-                    timeline,
-                    lsn,
-                    &parsed_xact,
-                    info == pg_constants::XLOG_XACT_COMMIT_PREPARED,
-                )?;
-                // Remove twophase file. see RemoveTwoPhaseFile() in postgres code
-                trace!(
-                    "Drop twophaseFile for xid {} parsed_xact.xid {} here at {}",
-                    decoded.xl_xid,
-                    parsed_xact.xid,
-                    lsn
-                );
-                timeline.drop_relish(
-                    RelishTag::TwoPhase {
-                        xid: parsed_xact.xid,
-                    },
-                    lsn,
-                )?;
-            } else if info == pg_constants::XLOG_XACT_PREPARE {
-                timeline.put_page_image(
-                    RelishTag::TwoPhase {
-                        xid: decoded.xl_xid,
-                    },
-                    0,
-                    lsn,
-                    Bytes::copy_from_slice(&buf[..]),
-                )?;
-            }
-        } else if decoded.xl_rmid == pg_constants::RM_MULTIXACT_ID {
-            let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
-
-            if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE {
-                let pageno = buf.get_u32_le();
-                let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-                let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-                timeline.put_page_image(
-                    RelishTag::Slru {
-                        slru: SlruKind::MultiXactOffsets,
-                        segno,
-                    },
-                    rpageno,
-                    lsn,
-                    ZERO_PAGE.clone(),
-                )?;
-            } else if info == pg_constants::XLOG_MULTIXACT_ZERO_MEM_PAGE {
-                let pageno = buf.get_u32_le();
-                let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-                let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-                timeline.put_page_image(
-                    RelishTag::Slru {
-                        slru: SlruKind::MultiXactMembers,
-                        segno,
-                    },
-                    rpageno,
-                    lsn,
-                    ZERO_PAGE.clone(),
-                )?;
-            } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
-                let xlrec = XlMultiXactCreate::decode(&mut buf);
-                self.ingest_multixact_create_record(timeline, lsn, &xlrec)?;
-            } else if info == pg_constants::XLOG_MULTIXACT_TRUNCATE_ID {
-                let xlrec = XlMultiXactTruncate::decode(&mut buf);
-                self.ingest_multixact_truncate_record(timeline, lsn, &xlrec)?;
-            }
-        } else if decoded.xl_rmid == pg_constants::RM_RELMAP_ID {
-            let xlrec = XlRelmapUpdate::decode(&mut buf);
-            self.ingest_relmap_page(timeline, lsn, &xlrec, &decoded)?;
-        } else if decoded.xl_rmid == pg_constants::RM_XLOG_ID {
-            let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
-            if info == pg_constants::XLOG_NEXTOID {
-                let next_oid = buf.get_u32_le();
-                if self.checkpoint.nextOid != next_oid {
-                    self.checkpoint.nextOid = next_oid;
-                    self.checkpoint_modified = true;
-                }
-            } else if info == pg_constants::XLOG_CHECKPOINT_ONLINE
-                || info == pg_constants::XLOG_CHECKPOINT_SHUTDOWN
-            {
-                let mut checkpoint_bytes = [0u8; SIZEOF_CHECKPOINT];
-                buf.copy_to_slice(&mut checkpoint_bytes);
-                let xlog_checkpoint = CheckPoint::decode(&checkpoint_bytes).unwrap();
-                trace!(
-                    "xlog_checkpoint.oldestXid={}, checkpoint.oldestXid={}",
-                    xlog_checkpoint.oldestXid,
-                    self.checkpoint.oldestXid
-                );
-                if (self
-                    .checkpoint
-                    .oldestXid
-                    .wrapping_sub(xlog_checkpoint.oldestXid) as i32)
-                    < 0
-                {
-                    self.checkpoint.oldestXid = xlog_checkpoint.oldestXid;
-                    self.checkpoint_modified = true;
-                }
-            }
-        }
-
-        // Iterate through all the blocks that the record modifies, and
-        // "put" a separate copy of the record for each block.
-        for blk in decoded.blocks.iter() {
-            self.ingest_decoded_block(timeline, lsn, &decoded, blk)?;
-        }
-
-        // If checkpoint data was updated, store the new version in the repository
-        if self.checkpoint_modified {
-            let new_checkpoint_bytes = self.checkpoint.encode();
-
-            timeline.put_page_image(RelishTag::Checkpoint, 0, lsn, new_checkpoint_bytes)?;
-            self.checkpoint_modified = false;
-        }
-
-        // Now that this record has been fully handled, including updating the
-        // checkpoint data, let the repository know that it is up-to-date to this LSN
-        timeline.advance_last_record_lsn(lsn);
-
-        Ok(())
-    }
-
-    fn ingest_decoded_block(
-        &mut self,
-        timeline: &dyn TimelineWriter,
-        lsn: Lsn,
-        decoded: &DecodedWALRecord,
-        blk: &DecodedBkpBlock,
-    ) -> Result<()> {
-        let tag = RelishTag::Relation(RelTag {
-            spcnode: blk.rnode_spcnode,
-            dbnode: blk.rnode_dbnode,
-            relnode: blk.rnode_relnode,
-            forknum: blk.forknum as u8,
-        });
-
-        //
-        // Instead of storing full-page-image WAL record,
-        // it is better to store extracted image: we can skip wal-redo
-        // in this case. Also some FPI records may contain multiple (up to 32) pages,
-        // so them have to be copied multiple times.
-        //
-        if blk.apply_image
-            && blk.has_image
-            && decoded.xl_rmid == pg_constants::RM_XLOG_ID
-            && (decoded.xl_info == pg_constants::XLOG_FPI
-                || decoded.xl_info == pg_constants::XLOG_FPI_FOR_HINT)
-        // compression of WAL is not yet supported: fall back to storing the original WAL record
-            && (blk.bimg_info & pg_constants::BKPIMAGE_IS_COMPRESSED) == 0
-        {
-            // Extract page image from FPI record
-            let img_len = blk.bimg_len as usize;
-            let img_offs = blk.bimg_offset as usize;
-            let mut image = BytesMut::with_capacity(pg_constants::BLCKSZ as usize);
-            image.extend_from_slice(&decoded.record[img_offs..img_offs + img_len]);
-
-            if blk.hole_length != 0 {
-                let tail = image.split_off(blk.hole_offset as usize);
-                image.resize(image.len() + blk.hole_length as usize, 0u8);
-                image.unsplit(tail);
-            }
-            image[0..4].copy_from_slice(&((lsn.0 >> 32) as u32).to_le_bytes());
-            image[4..8].copy_from_slice(&(lsn.0 as u32).to_le_bytes());
-            assert_eq!(image.len(), pg_constants::BLCKSZ as usize);
-            timeline.put_page_image(tag, blk.blkno, lsn, image.freeze())?;
-        } else {
-            let rec = ZenithWalRecord::Postgres {
-                will_init: blk.will_init || blk.apply_image,
-                rec: decoded.record.clone(),
-            };
-            timeline.put_wal_record(lsn, tag, blk.blkno, rec)?;
-        }
-        Ok(())
-    }
-
-    fn ingest_heapam_record(
-        &mut self,
-        buf: &mut Bytes,
-        timeline: &dyn TimelineWriter,
-        lsn: Lsn,
-        decoded: &mut DecodedWALRecord,
-    ) -> Result<()> {
-        // Handle VM bit updates that are implicitly part of heap records.
-        if decoded.xl_rmid == pg_constants::RM_HEAP_ID {
-            let info = decoded.xl_info & pg_constants::XLOG_HEAP_OPMASK;
-            if info == pg_constants::XLOG_HEAP_INSERT {
-                let xlrec = XlHeapInsert::decode(buf);
-                assert_eq!(0, buf.remaining());
-                if (xlrec.flags
-                    & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
-                        | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
-                    != 0
-                {
-                    timeline.put_wal_record(
-                        lsn,
-                        RelishTag::Relation(RelTag {
-                            forknum: pg_constants::VISIBILITYMAP_FORKNUM,
-                            spcnode: decoded.blocks[0].rnode_spcnode,
-                            dbnode: decoded.blocks[0].rnode_dbnode,
-                            relnode: decoded.blocks[0].rnode_relnode,
-                        }),
-                        decoded.blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32,
-                        ZenithWalRecord::ClearVisibilityMapFlags {
-                            heap_blkno: decoded.blocks[0].blkno,
-                            flags: pg_constants::VISIBILITYMAP_VALID_BITS,
-                        },
-                    )?;
-                }
-            } else if info == pg_constants::XLOG_HEAP_DELETE {
-                let xlrec = XlHeapDelete::decode(buf);
-                assert_eq!(0, buf.remaining());
-                if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 {
-                    timeline.put_wal_record(
-                        lsn,
-                        RelishTag::Relation(RelTag {
-                            forknum: pg_constants::VISIBILITYMAP_FORKNUM,
-                            spcnode: decoded.blocks[0].rnode_spcnode,
-                            dbnode: decoded.blocks[0].rnode_dbnode,
-                            relnode: decoded.blocks[0].rnode_relnode,
-                        }),
-                        decoded.blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32,
-                        ZenithWalRecord::ClearVisibilityMapFlags {
-                            heap_blkno: decoded.blocks[0].blkno,
-                            flags: pg_constants::VISIBILITYMAP_VALID_BITS,
-                        },
-                    )?;
-                }
-            } else if info == pg_constants::XLOG_HEAP_UPDATE
-                || info == pg_constants::XLOG_HEAP_HOT_UPDATE
-            {
-                let xlrec = XlHeapUpdate::decode(buf);
-                // the size of tuple data is inferred from the size of the record.
-                // we can't validate the remaining number of bytes without parsing
-                // the tuple data.
-                if (xlrec.flags & pg_constants::XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0 {
-                    timeline.put_wal_record(
-                        lsn,
-                        RelishTag::Relation(RelTag {
-                            forknum: pg_constants::VISIBILITYMAP_FORKNUM,
-                            spcnode: decoded.blocks[0].rnode_spcnode,
-                            dbnode: decoded.blocks[0].rnode_dbnode,
-                            relnode: decoded.blocks[0].rnode_relnode,
-                        }),
-                        decoded.blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32,
-                        ZenithWalRecord::ClearVisibilityMapFlags {
-                            heap_blkno: decoded.blocks[0].blkno,
-                            flags: pg_constants::VISIBILITYMAP_VALID_BITS,
-                        },
-                    )?;
-                }
-                if (xlrec.flags & pg_constants::XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) != 0
-                    && decoded.blocks.len() > 1
-                {
-                    timeline.put_wal_record(
-                        lsn,
-                        RelishTag::Relation(RelTag {
-                            forknum: pg_constants::VISIBILITYMAP_FORKNUM,
-                            spcnode: decoded.blocks[1].rnode_spcnode,
-                            dbnode: decoded.blocks[1].rnode_dbnode,
-                            relnode: decoded.blocks[1].rnode_relnode,
-                        }),
-                        decoded.blocks[1].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32,
-                        ZenithWalRecord::ClearVisibilityMapFlags {
-                            heap_blkno: decoded.blocks[1].blkno,
-                            flags: pg_constants::VISIBILITYMAP_VALID_BITS,
-                        },
-                    )?;
-                }
-            }
-        } else if decoded.xl_rmid == pg_constants::RM_HEAP2_ID {
-            let info = decoded.xl_info & pg_constants::XLOG_HEAP_OPMASK;
-            if info == pg_constants::XLOG_HEAP2_MULTI_INSERT {
-                let xlrec = XlHeapMultiInsert::decode(buf);
-
-                let offset_array_len = if decoded.xl_info & pg_constants::XLOG_HEAP_INIT_PAGE > 0 {
-                    // the offsets array is omitted if XLOG_HEAP_INIT_PAGE is set
-                    0
-                } else {
-                    std::mem::size_of::<u16>() * xlrec.ntuples as usize
-                };
-                assert_eq!(offset_array_len, buf.remaining());
-
-                // FIXME: why also ALL_FROZEN_SET?
-                if (xlrec.flags
-                    & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
-                        | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
-                    != 0
-                {
-                    timeline.put_wal_record(
-                        lsn,
-                        RelishTag::Relation(RelTag {
-                            forknum: pg_constants::VISIBILITYMAP_FORKNUM,
-                            spcnode: decoded.blocks[0].rnode_spcnode,
-                            dbnode: decoded.blocks[0].rnode_dbnode,
-                            relnode: decoded.blocks[0].rnode_relnode,
-                        }),
-                        decoded.blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32,
-                        ZenithWalRecord::ClearVisibilityMapFlags {
-                            heap_blkno: decoded.blocks[0].blkno,
-                            flags: pg_constants::VISIBILITYMAP_VALID_BITS,
-                        },
-                    )?;
-                }
-            }
-        }
-
-        // FIXME: What about XLOG_HEAP_LOCK and XLOG_HEAP2_LOCK_UPDATED?
-
-        Ok(())
-    }
-
-    /// Subroutine of ingest_record(), to handle an XLOG_DBASE_CREATE record.
-    fn ingest_xlog_dbase_create(
-        &mut self,
-        timeline: &dyn TimelineWriter,
-        lsn: Lsn,
-        rec: &XlCreateDatabase,
-    ) -> Result<()> {
-        let db_id = rec.db_id;
-        let tablespace_id = rec.tablespace_id;
-        let src_db_id = rec.src_db_id;
-        let src_tablespace_id = rec.src_tablespace_id;
-
-        // Creating a database is implemented by copying the template (aka. source) database.
-        // To copy all the relations, we need to ask for the state as of the same LSN, but we
-        // cannot pass 'lsn' to the Timeline.get_* functions, or they will block waiting for
-        // the last valid LSN to advance up to it. So we use the previous record's LSN in the
-        // get calls instead.
-        let req_lsn = min(timeline.get_last_record_lsn(), lsn);
-
-        let rels = timeline.list_rels(src_tablespace_id, src_db_id, req_lsn)?;
-
-        trace!("ingest_xlog_dbase_create: {} rels", rels.len());
-
-        let mut num_rels_copied = 0;
-        let mut num_blocks_copied = 0;
-        for rel in rels {
-            if let RelishTag::Relation(src_rel) = rel {
-                assert_eq!(src_rel.spcnode, src_tablespace_id);
-                assert_eq!(src_rel.dbnode, src_db_id);
-
-                let nblocks = timeline.get_relish_size(rel, req_lsn)?.unwrap_or(0);
-                let dst_rel = RelTag {
-                    spcnode: tablespace_id,
-                    dbnode: db_id,
-                    relnode: src_rel.relnode,
-                    forknum: src_rel.forknum,
-                };
-
-                // Copy content
-                for blknum in 0..nblocks {
-                    let content = timeline.get_page_at_lsn(rel, blknum, req_lsn)?;
-
-                    debug!("copying block {} from {} to {}", blknum, src_rel, dst_rel);
-
-                    timeline.put_page_image(RelishTag::Relation(dst_rel), blknum, lsn, content)?;
-                    num_blocks_copied += 1;
-                }
-
-                if nblocks == 0 {
-                    // make sure we have some trace of the relation, even if it's empty
-                    timeline.put_truncation(RelishTag::Relation(dst_rel), lsn, 0)?;
-                }
-
-                num_rels_copied += 1;
-            }
-        }
-
-        // Copy relfilemap
-        // TODO This implementation is very inefficient -
-        // it scans all non-rels only to find FileNodeMaps
-        for tag in timeline.list_nonrels(req_lsn)? {
-            if let RelishTag::FileNodeMap { spcnode, dbnode } = tag {
-                if spcnode == src_tablespace_id && dbnode == src_db_id {
-                    let img = timeline.get_page_at_lsn(tag, 0, req_lsn)?;
-                    let new_tag = RelishTag::FileNodeMap {
-                        spcnode: tablespace_id,
-                        dbnode: db_id,
-                    };
-                    timeline.put_page_image(new_tag, 0, lsn, img)?;
-                    break;
-                }
-            }
-        }
-        info!(
-            "Created database {}/{}, copied {} blocks in {} rels at {}",
-            tablespace_id, db_id, num_blocks_copied, num_rels_copied, lsn
-        );
-        Ok(())
-    }
-
-    /// Subroutine of ingest_record(), to handle an XLOG_SMGR_TRUNCATE record.
-    ///
-    /// This is the same logic as in PostgreSQL's smgr_redo() function.
-    fn ingest_xlog_smgr_truncate(
-        &mut self,
-        timeline: &dyn TimelineWriter,
-        lsn: Lsn,
-        rec: &XlSmgrTruncate,
-    ) -> Result<()> {
-        let spcnode = rec.rnode.spcnode;
-        let dbnode = rec.rnode.dbnode;
-        let relnode = rec.rnode.relnode;
-
-        if (rec.flags & pg_constants::SMGR_TRUNCATE_HEAP) != 0 {
-            let rel = RelTag {
-                spcnode,
-                dbnode,
-                relnode,
-                forknum: pg_constants::MAIN_FORKNUM,
-            };
-            timeline.put_truncation(RelishTag::Relation(rel), lsn, rec.blkno)?;
-        }
-        if (rec.flags & pg_constants::SMGR_TRUNCATE_FSM) != 0 {
-            let rel = RelTag {
-                spcnode,
-                dbnode,
-                relnode,
-                forknum: pg_constants::FSM_FORKNUM,
-            };
-
-            // FIXME: 'blkno' stored in the WAL record is the new size of the
-            // heap. The formula for calculating the new size of the FSM is
-            // pretty complicated (see FreeSpaceMapPrepareTruncateRel() in
-            // PostgreSQL), and we should also clear bits in the tail FSM block,
-            // and update the upper level FSM pages. None of that has been
-            // implemented. What we do instead, is always just truncate the FSM
-            // to zero blocks. That's bad for performance, but safe. (The FSM
-            // isn't needed for correctness, so we could also leave garbage in
-            // it. Seems more tidy to zap it away.)
-            if rec.blkno != 0 {
-                info!("Partial truncation of FSM is not supported");
-            }
-            let num_fsm_blocks = 0;
-            timeline.put_truncation(RelishTag::Relation(rel), lsn, num_fsm_blocks)?;
-        }
-        if (rec.flags & pg_constants::SMGR_TRUNCATE_VM) != 0 {
-            let rel = RelTag {
-                spcnode,
-                dbnode,
-                relnode,
-                forknum: pg_constants::VISIBILITYMAP_FORKNUM,
-            };
-
-            // FIXME: Like with the FSM above, the logic to truncate the VM
-            // correctly has not been implemented. Just zap it away completely,
-            // always. Unlike the FSM, the VM must never have bits incorrectly
-            // set. From a correctness point of view, it's always OK to clear
-            // bits or remove it altogether, though.
-            if rec.blkno != 0 {
-                info!("Partial truncation of VM is not supported");
-            }
-            let num_vm_blocks = 0;
-            timeline.put_truncation(RelishTag::Relation(rel), lsn, num_vm_blocks)?;
-        }
-        Ok(())
-    }
-
-    /// Subroutine of ingest_record(), to handle an XLOG_XACT_* records.
-    ///
-    fn ingest_xact_record(
-        &mut self,
-        timeline: &dyn TimelineWriter,
-        lsn: Lsn,
-        parsed: &XlXactParsedRecord,
-        is_commit: bool,
-    ) -> Result<()> {
-        // Record update of CLOG pages
-        let mut pageno = parsed.xid / pg_constants::CLOG_XACTS_PER_PAGE;
-        let mut segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-        let mut rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-        let mut page_xids: Vec<TransactionId> = vec![parsed.xid];
-
-        for subxact in &parsed.subxacts {
-            let subxact_pageno = subxact / pg_constants::CLOG_XACTS_PER_PAGE;
-            if subxact_pageno != pageno {
-                // This subxact goes to different page. Write the record
-                // for all the XIDs on the previous page, and continue
-                // accumulating XIDs on this new page.
-                timeline.put_wal_record(
-                    lsn,
-                    RelishTag::Slru {
-                        slru: SlruKind::Clog,
-                        segno,
-                    },
-                    rpageno,
-                    if is_commit {
-                        ZenithWalRecord::ClogSetCommitted { xids: page_xids }
-                    } else {
-                        ZenithWalRecord::ClogSetAborted { xids: page_xids }
-                    },
-                )?;
-                page_xids = Vec::new();
-            }
-            pageno = subxact_pageno;
-            segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-            rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-            page_xids.push(*subxact);
-        }
-        timeline.put_wal_record(
-            lsn,
-            RelishTag::Slru {
-                slru: SlruKind::Clog,
-                segno,
-            },
-            rpageno,
-            if is_commit {
-                ZenithWalRecord::ClogSetCommitted { xids: page_xids }
-            } else {
-                ZenithWalRecord::ClogSetAborted { xids: page_xids }
-            },
-        )?;
-
-        for xnode in &parsed.xnodes {
-            for forknum in pg_constants::MAIN_FORKNUM..=pg_constants::VISIBILITYMAP_FORKNUM {
-                let rel = RelTag {
-                    forknum,
-                    spcnode: xnode.spcnode,
-                    dbnode: xnode.dbnode,
-                    relnode: xnode.relnode,
-                };
-                timeline.drop_relish(RelishTag::Relation(rel), lsn)?;
-            }
-        }
-        Ok(())
-    }
-
-    fn ingest_clog_truncate_record(
-        &mut self,
-        timeline: &dyn TimelineWriter,
-        lsn: Lsn,
-        xlrec: &XlClogTruncate,
-    ) -> Result<()> {
-        info!(
-            "RM_CLOG_ID truncate pageno {} oldestXid {} oldestXidDB {} lsn {}",
-            xlrec.pageno, xlrec.oldest_xid, xlrec.oldest_xid_db, lsn
-        );
-
-        // Here we treat oldestXid and oldestXidDB
-        // differently from postgres redo routines.
-        // In postgres checkpoint.oldestXid lags behind xlrec.oldest_xid
-        // until checkpoint happens and updates the value.
-        // Here we can use the most recent value.
-        // It's just an optimization, though and can be deleted.
-        // TODO Figure out if there will be any issues with replica.
-        self.checkpoint.oldestXid = xlrec.oldest_xid;
-        self.checkpoint.oldestXidDB = xlrec.oldest_xid_db;
-        self.checkpoint_modified = true;
-
-        // TODO Treat AdvanceOldestClogXid() or write a comment why we don't need it
-
-        let latest_page_number =
-            self.checkpoint.nextXid.value as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
-
-        // Now delete all segments containing pages between xlrec.pageno
-        // and latest_page_number.
-
-        // First, make an important safety check:
-        // the current endpoint page must not be eligible for removal.
-        // See SimpleLruTruncate() in slru.c
-        if clogpage_precedes(latest_page_number, xlrec.pageno) {
-            info!("could not truncate directory pg_xact apparent wraparound");
-            return Ok(());
-        }
-
-        // Iterate via SLRU CLOG segments and drop segments that we're ready to truncate
-        // TODO This implementation is very inefficient -
-        // it scans all non-rels only to find Clog
-        //
-        // We cannot pass 'lsn' to the Timeline.list_nonrels(), or it
-        // will block waiting for the last valid LSN to advance up to
-        // it. So we use the previous record's LSN in the get calls
-        // instead.
-        let req_lsn = min(timeline.get_last_record_lsn(), lsn);
-        for obj in timeline.list_nonrels(req_lsn)? {
-            if let RelishTag::Slru { slru, segno } = obj {
-                if slru == SlruKind::Clog {
-                    let segpage = segno * pg_constants::SLRU_PAGES_PER_SEGMENT;
-                    if slru_may_delete_clogsegment(segpage, xlrec.pageno) {
-                        timeline.drop_relish(RelishTag::Slru { slru, segno }, lsn)?;
-                        trace!("Drop CLOG segment {:>04X} at lsn {}", segno, lsn);
-                    }
-                }
-            }
-        }
-
-        Ok(())
-    }
-
-    fn ingest_multixact_create_record(
-        &mut self,
-        timeline: &dyn TimelineWriter,
-        lsn: Lsn,
-        xlrec: &XlMultiXactCreate,
-    ) -> Result<()> {
-        // Create WAL record for updating the multixact-offsets page
-        let pageno = xlrec.mid / pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32;
-        let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-        let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-
-        timeline.put_wal_record(
-            lsn,
-            RelishTag::Slru {
-                slru: SlruKind::MultiXactOffsets,
-                segno,
-            },
-            rpageno,
-            ZenithWalRecord::MultixactOffsetCreate {
-                mid: xlrec.mid,
-                moff: xlrec.moff,
-            },
-        )?;
-
-        // Create WAL records for the update of each affected multixact-members page
-        let mut members = xlrec.members.iter();
-        let mut offset = xlrec.moff;
-        loop {
-            let pageno = offset / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-
-            // How many members fit on this page?
-            let page_remain = pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32
-                - offset % pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-
-            let mut this_page_members: Vec<MultiXactMember> = Vec::new();
-            for _ in 0..page_remain {
-                if let Some(m) = members.next() {
-                    this_page_members.push(m.clone());
-                } else {
-                    break;
-                }
-            }
-            if this_page_members.is_empty() {
-                // all done
-                break;
-            }
-            let n_this_page = this_page_members.len();
-
-            timeline.put_wal_record(
-                lsn,
-                RelishTag::Slru {
-                    slru: SlruKind::MultiXactMembers,
-                    segno: pageno / pg_constants::SLRU_PAGES_PER_SEGMENT,
-                },
-                pageno % pg_constants::SLRU_PAGES_PER_SEGMENT,
-                ZenithWalRecord::MultixactMembersCreate {
-                    moff: offset,
-                    members: this_page_members,
-                },
-            )?;
-
-            // Note: The multixact members can wrap around, even within one WAL record.
-            offset = offset.wrapping_add(n_this_page as u32);
-        }
-        if xlrec.mid >= self.checkpoint.nextMulti {
-            self.checkpoint.nextMulti = xlrec.mid + 1;
-            self.checkpoint_modified = true;
-        }
-        if xlrec.moff + xlrec.nmembers > self.checkpoint.nextMultiOffset {
-            self.checkpoint.nextMultiOffset = xlrec.moff + xlrec.nmembers;
-            self.checkpoint_modified = true;
-        }
-        let max_mbr_xid = xlrec.members.iter().fold(0u32, |acc, mbr| {
-            if mbr.xid.wrapping_sub(acc) as i32 > 0 {
-                mbr.xid
-            } else {
-                acc
-            }
-        });
-
-        if self.checkpoint.update_next_xid(max_mbr_xid) {
-            self.checkpoint_modified = true;
-        }
-        Ok(())
-    }
-
-    fn ingest_multixact_truncate_record(
-        &mut self,
-        timeline: &dyn TimelineWriter,
-        lsn: Lsn,
-        xlrec: &XlMultiXactTruncate,
-    ) -> Result<()> {
-        self.checkpoint.oldestMulti = xlrec.end_trunc_off;
-        self.checkpoint.oldestMultiDB = xlrec.oldest_multi_db;
-        self.checkpoint_modified = true;
-
-        // PerformMembersTruncation
-        let maxsegment: i32 = mx_offset_to_member_segment(pg_constants::MAX_MULTIXACT_OFFSET);
-        let startsegment: i32 = mx_offset_to_member_segment(xlrec.start_trunc_memb);
-        let endsegment: i32 = mx_offset_to_member_segment(xlrec.end_trunc_memb);
-        let mut segment: i32 = startsegment;
-
-        // Delete all the segments except the last one. The last segment can still
-        // contain, possibly partially, valid data.
-        while segment != endsegment {
-            timeline.drop_relish(
-                RelishTag::Slru {
-                    slru: SlruKind::MultiXactMembers,
-                    segno: segment as u32,
-                },
-                lsn,
-            )?;
-
-            /* move to next segment, handling wraparound correctly */
-            if segment == maxsegment {
-                segment = 0;
-            } else {
-                segment += 1;
-            }
-        }
-
-        // Truncate offsets
-        // FIXME: this did not handle wraparound correctly
-
-        Ok(())
-    }
-
-    fn ingest_relmap_page(
-        &mut self,
-        timeline: &dyn TimelineWriter,
-        lsn: Lsn,
-        xlrec: &XlRelmapUpdate,
-        decoded: &DecodedWALRecord,
-    ) -> Result<()> {
-        let tag = RelishTag::FileNodeMap {
-            spcnode: xlrec.tsid,
-            dbnode: xlrec.dbid,
-        };
-
-        let mut buf = decoded.record.clone();
-        buf.advance(decoded.main_data_offset);
-        // skip xl_relmap_update
-        buf.advance(12);
-
-        timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buf[..]))?;
-
-        Ok(())
-    }
-}

pageserver/src/walreceiver.rs

@@ -5,29 +5,28 @@
 //!
 //! We keep one WAL receiver active per timeline.
 
-use crate::config::PageServerConf;
+use crate::relish::*;
+use crate::restore_local_repo;
 use crate::tenant_mgr;
-use crate::tenant_mgr::TenantState;
-use crate::tenant_threads;
-use crate::walingest::WalIngest;
-use anyhow::{bail, Context, Error, Result};
+use crate::waldecoder::*;
+use crate::PageServerConf;
+use anyhow::{bail, Error, Result};
 use lazy_static::lazy_static;
-use parking_lot::Mutex;
-use postgres_ffi::waldecoder::*;
+use postgres::fallible_iterator::FallibleIterator;
+use postgres::replication::ReplicationIter;
+use postgres::{Client, NoTls, SimpleQueryMessage, SimpleQueryRow};
+use postgres_ffi::*;
 use postgres_protocol::message::backend::ReplicationMessage;
 use postgres_types::PgLsn;
 use std::cell::Cell;
 use std::collections::HashMap;
 use std::str::FromStr;
+use std::sync::Mutex;
 use std::thread;
+use std::thread::sleep;
 use std::thread::JoinHandle;
 use std::thread_local;
-use std::time::SystemTime;
-use tokio::pin;
-use tokio::sync::oneshot;
-use tokio_postgres::replication::ReplicationStream;
-use tokio_postgres::{Client, NoTls, SimpleQueryMessage, SimpleQueryRow};
-use tokio_stream::StreamExt;
+use std::time::{Duration, SystemTime};
 use tracing::*;
 use zenith_utils::lsn::Lsn;
 use zenith_utils::zid::ZTenantId;
@@ -39,8 +38,6 @@ use zenith_utils::zid::ZTimelineId;
 struct WalReceiverEntry {
     wal_producer_connstr: String,
     wal_receiver_handle: Option<JoinHandle<()>>,
-    wal_receiver_interrupt_sender: Option<oneshot::Sender<()>>,
-    tenantid: ZTenantId,
 }
 
 lazy_static! {
@@ -61,46 +58,13 @@ thread_local! {
 // per tenant/timeline to cancel inactive walreceivers.
 // TODO deal with blocking pg connections
 pub fn stop_wal_receiver(timelineid: ZTimelineId) {
-    let mut receivers = WAL_RECEIVERS.lock();
-
+    let mut receivers = WAL_RECEIVERS.lock().unwrap();
     if let Some(r) = receivers.get_mut(&timelineid) {
-        match r.wal_receiver_interrupt_sender.take() {
-            Some(s) => {
-                if s.send(()).is_err() {
-                    warn!("wal receiver interrupt signal already sent");
-                }
-            }
-            None => {
-                warn!("wal_receiver_interrupt_sender is missing, wal recever shouldn't be running")
-            }
-        }
-
-        info!("waiting for wal receiver to stop");
-        let handle = r.wal_receiver_handle.take();
-        // do not hold the lock while joining the handle (deadlock is possible otherwise)
-        drop(receivers);
-        // there is no timeout or try_join option available so in case of a bug this can hang forever
-        handle.map(JoinHandle::join);
+        r.wal_receiver_handle.take();
+        // r.wal_receiver_handle.take().map(JoinHandle::join);
     }
 }
 
-pub fn drop_wal_receiver(timelineid: ZTimelineId, tenantid: ZTenantId) {
-    let mut receivers = WAL_RECEIVERS.lock();
-    receivers.remove(&timelineid);
-
-    // Check if it was the last walreceiver of the tenant.
-    // TODO now we store one WalReceiverEntry per timeline,
-    // so this iterator looks a bit strange.
-    for (_timelineid, entry) in receivers.iter() {
-        if entry.tenantid == tenantid {
-            return;
-        }
-    }
-
-    // When last walreceiver of the tenant is gone, change state to Idle
-    tenant_mgr::set_tenant_state(tenantid, TenantState::Idle).unwrap();
-}
-
 // Launch a new WAL receiver, or tell one that's running about change in connection string
 pub fn launch_wal_receiver(
     conf: &'static PageServerConf,
@@ -108,41 +72,33 @@ pub fn launch_wal_receiver(
     wal_producer_connstr: &str,
     tenantid: ZTenantId,
 ) {
-    let mut receivers = WAL_RECEIVERS.lock();
+    let mut receivers = WAL_RECEIVERS.lock().unwrap();
 
     match receivers.get_mut(&timelineid) {
         Some(receiver) => {
             receiver.wal_producer_connstr = wal_producer_connstr.into();
         }
         None => {
-            let (tx, rx) = tokio::sync::oneshot::channel::<()>();
-
             let wal_receiver_handle = thread::Builder::new()
                 .name("WAL receiver thread".into())
                 .spawn(move || {
                     IS_WAL_RECEIVER.with(|c| c.set(true));
-                    thread_main(conf, timelineid, tenantid, rx);
+                    thread_main(conf, timelineid, tenantid);
                 })
                 .unwrap();
 
             let receiver = WalReceiverEntry {
                 wal_producer_connstr: wal_producer_connstr.into(),
                 wal_receiver_handle: Some(wal_receiver_handle),
-                wal_receiver_interrupt_sender: Some(tx),
-                tenantid,
             };
             receivers.insert(timelineid, receiver);
-
-            // Update tenant state and start tenant threads, if they are not running yet.
-            tenant_mgr::set_tenant_state(tenantid, TenantState::Active).unwrap();
-            tenant_threads::start_tenant_threads(conf, tenantid);
         }
     };
 }
 
 // Look up current WAL producer connection string in the hash table
 fn get_wal_producer_connstr(timelineid: ZTimelineId) -> String {
-    let receivers = WAL_RECEIVERS.lock();
+    let receivers = WAL_RECEIVERS.lock().unwrap();
 
     receivers
         .get(&timelineid)
@@ -154,49 +110,36 @@ fn get_wal_producer_connstr(timelineid: ZTimelineId) -> String {
 //
 // This is the entry point for the WAL receiver thread.
 //
-fn thread_main(
-    conf: &'static PageServerConf,
-    timelineid: ZTimelineId,
-    tenantid: ZTenantId,
-    interrupt_receiver: oneshot::Receiver<()>,
-) {
+fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId, tenantid: ZTenantId) {
     let _enter = info_span!("WAL receiver", timeline = %timelineid, tenant = %tenantid).entered();
     info!("WAL receiver thread started");
 
-    // Look up the current WAL producer address
-    let wal_producer_connstr = get_wal_producer_connstr(timelineid);
-
+    //
     // Make a connection to the WAL safekeeper, or directly to the primary PostgreSQL server,
-    // and start streaming WAL from it.
-    let res = walreceiver_main(
-        conf,
-        tenantid,
-        timelineid,
-        &wal_producer_connstr,
-        interrupt_receiver,
-    );
+    // and start streaming WAL from it. If the connection is lost, keep retrying.
+    //
+    while !tenant_mgr::shutdown_requested() {
+        // Look up the current WAL producer address
+        let wal_producer_connstr = get_wal_producer_connstr(timelineid);
 
-    // TODO cleanup info messages
-    if let Err(e) = res {
-        info!("WAL streaming connection failed ({})", e);
-    } else {
-        info!(
-            "walreceiver disconnected tenant {}, timelineid {}",
-            tenantid, timelineid
-        );
+        let res = walreceiver_main(conf, timelineid, &wal_producer_connstr, tenantid);
+
+        if let Err(e) = res {
+            info!(
+                "WAL streaming connection failed ({}), retrying in 1 second",
+                e
+            );
+            sleep(Duration::from_secs(1));
+        }
     }
-
-    // Drop it from list of active WAL_RECEIVERS
-    // so that next callmemaybe request launched a new thread
-    drop_wal_receiver(timelineid, tenantid);
+    debug!("WAL streaming shut down");
 }
 
 fn walreceiver_main(
     _conf: &PageServerConf,
-    tenantid: ZTenantId,
     timelineid: ZTimelineId,
     wal_producer_connstr: &str,
-    mut interrupt_receiver: oneshot::Receiver<()>,
+    tenantid: ZTenantId,
 ) -> Result<(), Error> {
     // Connect to the database in replication mode.
     info!("connecting to {:?}", wal_producer_connstr);
@@ -205,19 +148,7 @@ fn walreceiver_main(
         wal_producer_connstr
     );
 
-    let runtime = tokio::runtime::Builder::new_current_thread()
-        .enable_all()
-        .build()?;
-
-    let (mut replication_client, connection) =
-        runtime.block_on(tokio_postgres::connect(&connect_cfg, NoTls))?;
-    // This is from tokio-postgres docs, but it is a bit weird in our case because we extensively use block_on
-    runtime.spawn(async move {
-        if let Err(e) = connection.await {
-            eprintln!("connection error: {}", e);
-        }
-    });
-
+    let mut rclient = Client::connect(&connect_cfg, NoTls)?;
     info!("connected!");
 
     // Immediately increment the gauge, then create a job to decrement it on thread exit.
@@ -229,18 +160,12 @@ fn walreceiver_main(
         gauge.dec();
     }
 
-    let identify = runtime.block_on(identify_system(&mut replication_client))?;
+    let identify = identify_system(&mut rclient)?;
     info!("{:?}", identify);
     let end_of_wal = Lsn::from(u64::from(identify.xlogpos));
     let mut caught_up = false;
 
-    let timeline =
-        tenant_mgr::get_timeline_for_tenant(tenantid, timelineid).with_context(|| {
-            format!(
-                "Can not start the walrecever for a remote tenant {}, timeline {}",
-                tenantid, timelineid,
-            )
-        })?;
+    let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)?;
 
     //
     // Start streaming the WAL, from where we left off previously.
@@ -264,24 +189,16 @@ fn walreceiver_main(
 
     let query = format!("START_REPLICATION PHYSICAL {}", startpoint);
 
-    let copy_stream = runtime.block_on(replication_client.copy_both_simple(&query))?;
-    let physical_stream = ReplicationStream::new(copy_stream);
-    pin!(physical_stream);
+    let copy_stream = rclient.copy_both_simple(&query)?;
+    let mut physical_stream = ReplicationIter::new(copy_stream);
 
     let mut waldecoder = WalStreamDecoder::new(startpoint);
 
-    let mut walingest = WalIngest::new(&*timeline, startpoint)?;
+    let checkpoint_bytes = timeline.get_page_at_lsn(RelishTag::Checkpoint, 0, startpoint)?;
+    let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
+    trace!("CheckPoint.nextXid = {}", checkpoint.nextXid.value);
 
-    while let Some(replication_message) = runtime.block_on(async {
-        tokio::select! {
-            replication_message = physical_stream.next() => replication_message,
-            _ = &mut interrupt_receiver => {
-                info!("walreceiver interrupted");
-                None
-            }
-        }
-    }) {
-        let replication_message = replication_message?;
+    while let Some(replication_message) = physical_stream.next()? {
         let status_update = match replication_message {
             ReplicationMessage::XLogData(xlog_data) => {
                 // Pass the WAL data to the decoder, and see if we can decode
@@ -303,8 +220,34 @@ fn walreceiver_main(
                     assert!(lsn.is_aligned());
 
                     let writer = timeline.writer();
-                    walingest.ingest_record(writer.as_ref(), recdata, lsn)?;
 
+                    let mut checkpoint_modified = false;
+
+                    let decoded = decode_wal_record(recdata.clone());
+                    restore_local_repo::save_decoded_record(
+                        &mut checkpoint,
+                        &mut checkpoint_modified,
+                        writer.as_ref(),
+                        &decoded,
+                        recdata,
+                        lsn,
+                    )?;
+
+                    // Check if checkpoint data was updated by save_decoded_record
+                    if checkpoint_modified {
+                        let new_checkpoint_bytes = checkpoint.encode();
+
+                        writer.put_page_image(
+                            RelishTag::Checkpoint,
+                            0,
+                            lsn,
+                            new_checkpoint_bytes,
+                        )?;
+                    }
+
+                    // Now that this record has been fully handled, including updating the
+                    // checkpoint data, let the repository know that it is up-to-date to this LSN
+                    writer.advance_last_record_lsn(lsn);
                     last_rec_lsn = lsn;
                 }
 
@@ -339,30 +282,24 @@ fn walreceiver_main(
         };
 
         if let Some(last_lsn) = status_update {
+            // TODO: More thought should go into what values are sent here.
             let last_lsn = PgLsn::from(u64::from(last_lsn));
-            let timeline_synced_disk_consistent_lsn =
-                tenant_mgr::get_repository_for_tenant(tenantid)?
-                    .get_timeline_state(timelineid)
-                    .and_then(|state| state.remote_disk_consistent_lsn())
-                    .unwrap_or(Lsn(0));
-
-            // The last LSN we processed. It is not guaranteed to survive pageserver crash.
-            let write_lsn = last_lsn;
-            // `disk_consistent_lsn` is the LSN at which page server guarantees local persistence of all received data
-            let flush_lsn = PgLsn::from(u64::from(timeline.get_disk_consistent_lsn()));
-            // The last LSN that is synced to remote storage and is guaranteed to survive pageserver crash
-            // Used by safekeepers to remove WAL preceding `remote_consistent_lsn`.
-            let apply_lsn = PgLsn::from(u64::from(timeline_synced_disk_consistent_lsn));
+            // We are using disk consistent LSN as `write_lsn`, i.e. LSN at which page server
+            // may guarantee persistence of all received data. Safekeeper is not free to remove
+            // WAL preceding `write_lsn`: it should not be requested by this page server.
+            let write_lsn = PgLsn::from(u64::from(timeline.get_disk_consistent_lsn()));
+            let flush_lsn = last_lsn;
+            let apply_lsn = PgLsn::from(0);
             let ts = SystemTime::now();
             const NO_REPLY: u8 = 0;
-            runtime.block_on(
-                physical_stream
-                    .as_mut()
-                    .standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY),
-            )?;
+            physical_stream.standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY)?;
+        }
+
+        if tenant_mgr::shutdown_requested() {
+            debug!("stop walreceiver because pageserver shutdown is requested");
+            break;
         }
     }
-
     Ok(())
 }
 
@@ -389,9 +326,9 @@ pub struct IdentifySystem {
 pub struct IdentifyError;
 
 /// Run the postgres `IDENTIFY_SYSTEM` command
-pub async fn identify_system(client: &mut Client) -> Result<IdentifySystem, Error> {
+pub fn identify_system(client: &mut Client) -> Result<IdentifySystem, Error> {
     let query_str = "IDENTIFY_SYSTEM";
-    let response = client.simple_query(query_str).await?;
+    let response = client.simple_query(query_str)?;
 
     // get(N) from row, then parse it as some destination type.
     fn get_parse<T>(row: &SimpleQueryRow, idx: usize) -> Result<T, IdentifyError>

pageserver/src/walredo.rs

@@ -19,36 +19,39 @@
 //! process, he cannot escape out of it.
 //!
 use byteorder::{ByteOrder, LittleEndian};
-use bytes::{BufMut, Bytes, BytesMut};
+use bytes::{Buf, BufMut, Bytes, BytesMut};
 use lazy_static::lazy_static;
 use log::*;
-use nix::poll::*;
 use serde::Serialize;
 use std::fs;
 use std::fs::OpenOptions;
 use std::io::prelude::*;
-use std::io::{Error, ErrorKind};
-use std::os::unix::io::AsRawFd;
+use std::io::Error;
 use std::path::PathBuf;
 use std::process::Stdio;
-use std::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command};
 use std::sync::Mutex;
 use std::time::Duration;
 use std::time::Instant;
+use tokio::io::AsyncBufReadExt;
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+use tokio::process::{ChildStdin, ChildStdout, Command};
+use tokio::time::timeout;
 use zenith_metrics::{register_histogram, register_int_counter, Histogram, IntCounter};
 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;
-use zenith_utils::nonblock::set_nonblock;
 use zenith_utils::zid::ZTenantId;
 
-use crate::config::PageServerConf;
 use crate::relish::*;
-use crate::repository::ZenithWalRecord;
+use crate::repository::WALRecord;
+use crate::waldecoder::XlMultiXactCreate;
+use crate::waldecoder::XlXactParsedRecord;
+use crate::PageServerConf;
 use postgres_ffi::nonrelfile_utils::mx_offset_to_flags_bitshift;
 use postgres_ffi::nonrelfile_utils::mx_offset_to_flags_offset;
 use postgres_ffi::nonrelfile_utils::mx_offset_to_member_offset;
 use postgres_ffi::nonrelfile_utils::transaction_id_set_status;
 use postgres_ffi::pg_constants;
+use postgres_ffi::XLogRecord;
 
 ///
 /// `RelTag` + block number (`blknum`) gives us a unique id of the page in the cluster.
@@ -79,7 +82,7 @@ pub trait WalRedoManager: Send + Sync {
         blknum: u32,
         lsn: Lsn,
         base_img: Option<Bytes>,
-        records: Vec<(Lsn, ZenithWalRecord)>,
+        records: Vec<(Lsn, WALRecord)>,
     ) -> Result<Bytes, WalRedoError>;
 }
 
@@ -96,7 +99,7 @@ impl crate::walredo::WalRedoManager for DummyRedoManager {
         _blknum: u32,
         _lsn: Lsn,
         _base_img: Option<Bytes>,
-        _records: Vec<(Lsn, ZenithWalRecord)>,
+        _records: Vec<(Lsn, WALRecord)>,
     ) -> Result<Bytes, WalRedoError> {
         Err(WalRedoError::InvalidState)
     }
@@ -136,46 +139,27 @@ pub struct PostgresRedoManager {
     tenantid: ZTenantId,
     conf: &'static PageServerConf,
 
+    runtime: tokio::runtime::Runtime,
     process: Mutex<Option<PostgresRedoProcess>>,
 }
 
-/// Can this request be served by zenith redo funcitons
-/// or we need to pass it to wal-redo postgres process?
-fn can_apply_in_zenith(rec: &ZenithWalRecord) -> bool {
-    // Currently, we don't have bespoken Rust code to replay any
-    // Postgres WAL records. But everything else is handled in zenith.
-    #[allow(clippy::match_like_matches_macro)]
-    match rec {
-        ZenithWalRecord::Postgres {
-            will_init: _,
-            rec: _,
-        } => false,
-        _ => true,
-    }
+#[derive(Debug)]
+struct WalRedoRequest {
+    rel: RelishTag,
+    blknum: u32,
+    lsn: Lsn,
+
+    base_img: Option<Bytes>,
+    records: Vec<(Lsn, WALRecord)>,
 }
 
-fn check_forknum(rel: &RelishTag, expected_forknum: u8) -> bool {
-    if let RelishTag::Relation(RelTag {
-        forknum,
-        spcnode: _,
-        dbnode: _,
-        relnode: _,
-    }) = rel
-    {
-        *forknum == expected_forknum
-    } else {
-        false
+impl WalRedoRequest {
+    // Can this request be served by zenith redo funcitons
+    // or we need to pass it to wal-redo postgres process?
+    fn can_apply_in_zenith(&self) -> bool {
+        !matches!(self.rel, RelishTag::Relation(_))
     }
 }
-
-fn check_slru_segno(rel: &RelishTag, expected_slru: SlruKind, expected_segno: u32) -> bool {
-    if let RelishTag::Slru { slru, segno } = rel {
-        *slru == expected_slru && *segno == expected_segno
-    } else {
-        false
-    }
-}
-
 /// An error happened in WAL redo
 #[derive(Debug, thiserror::Error)]
 pub enum WalRedoError {
@@ -204,37 +188,50 @@ impl WalRedoManager for PostgresRedoManager {
         blknum: u32,
         lsn: Lsn,
         base_img: Option<Bytes>,
-        records: Vec<(Lsn, ZenithWalRecord)>,
+        records: Vec<(Lsn, WALRecord)>,
     ) -> Result<Bytes, WalRedoError> {
-        if records.is_empty() {
-            error!("invalid WAL redo request with no records");
-            return Err(WalRedoError::InvalidRequest);
-        }
+        let start_time;
+        let end_time;
 
-        let mut img: Option<Bytes> = base_img;
-        let mut batch_zenith = can_apply_in_zenith(&records[0].1);
-        let mut batch_start = 0;
-        for i in 1..records.len() {
-            let rec_zenith = can_apply_in_zenith(&records[i].1);
+        let request = WalRedoRequest {
+            rel,
+            blknum,
+            lsn,
+            base_img,
+            records,
+        };
 
-            if rec_zenith != batch_zenith {
-                let result = if batch_zenith {
-                    self.apply_batch_zenith(rel, blknum, lsn, img, &records[batch_start..i])
-                } else {
-                    self.apply_batch_postgres(rel, blknum, lsn, img, &records[batch_start..i])
-                };
-                img = Some(result?);
+        start_time = Instant::now();
+        let result;
 
-                batch_zenith = rec_zenith;
-                batch_start = i;
-            }
-        }
-        // last batch
-        if batch_zenith {
-            self.apply_batch_zenith(rel, blknum, lsn, img, &records[batch_start..])
+        if request.can_apply_in_zenith() {
+            result = self.handle_apply_request_zenith(&request);
+
+            end_time = Instant::now();
+            WAL_REDO_TIME.observe(end_time.duration_since(start_time).as_secs_f64());
         } else {
-            self.apply_batch_postgres(rel, blknum, lsn, img, &records[batch_start..])
+            let mut process_guard = self.process.lock().unwrap();
+            let lock_time = Instant::now();
+
+            // launch the WAL redo process on first use
+            if process_guard.is_none() {
+                let p = self
+                    .runtime
+                    .block_on(PostgresRedoProcess::launch(self.conf, &self.tenantid))?;
+                *process_guard = Some(p);
+            }
+            let process = process_guard.as_mut().unwrap();
+
+            result = self
+                .runtime
+                .block_on(self.handle_apply_request_postgres(process, &request));
+
+            WAL_REDO_WAIT_TIME.observe(lock_time.duration_since(start_time).as_secs_f64());
+            end_time = Instant::now();
+            WAL_REDO_TIME.observe(end_time.duration_since(lock_time).as_secs_f64());
         }
+
+        result
     }
 }
 
@@ -243,8 +240,17 @@ impl PostgresRedoManager {
     /// Create a new PostgresRedoManager.
     ///
     pub fn new(conf: &'static PageServerConf, tenantid: ZTenantId) -> PostgresRedoManager {
+        // We block on waiting for requests on the walredo request channel, but
+        // use async I/O to communicate with the child process. Initialize the
+        // runtime for the async part.
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+
         // The actual process is launched lazily, on first request.
         PostgresRedoManager {
+            runtime,
             tenantid,
             conf,
             process: Mutex::new(None),
@@ -254,229 +260,208 @@ impl PostgresRedoManager {
     ///
     /// Process one request for WAL redo using wal-redo postgres
     ///
-    fn apply_batch_postgres(
+    async fn handle_apply_request_postgres(
         &self,
-        rel: RelishTag,
-        blknum: u32,
-        lsn: Lsn,
-        base_img: Option<Bytes>,
-        records: &[(Lsn, ZenithWalRecord)],
+        process: &mut PostgresRedoProcess,
+        request: &WalRedoRequest,
     ) -> Result<Bytes, WalRedoError> {
-        let start_time = Instant::now();
+        let blknum = request.blknum;
+        let lsn = request.lsn;
+        let base_img = request.base_img.clone();
+        let records = &request.records;
+        let nrecords = records.len();
+
+        let start = Instant::now();
 
         let apply_result: Result<Bytes, Error>;
 
-        let mut process_guard = self.process.lock().unwrap();
-        let lock_time = Instant::now();
-
-        // launch the WAL redo process on first use
-        if process_guard.is_none() {
-            let p = PostgresRedoProcess::launch(self.conf, &self.tenantid)?;
-            *process_guard = Some(p);
-        }
-        let process = process_guard.as_mut().unwrap();
-
-        WAL_REDO_WAIT_TIME.observe(lock_time.duration_since(start_time).as_secs_f64());
-
-        let result = if let RelishTag::Relation(rel) = rel {
+        if let RelishTag::Relation(rel) = request.rel {
             // Relational WAL records are applied using wal-redo-postgres
             let buf_tag = BufferTag { rel, blknum };
-            apply_result = process.apply_wal_records(buf_tag, base_img, records);
+            apply_result = process.apply_wal_records(buf_tag, base_img, records).await;
+
+            let duration = start.elapsed();
+
+            debug!(
+                "postgres applied {} WAL records in {} ms to reconstruct page image at LSN {}",
+                nrecords,
+                duration.as_millis(),
+                lsn
+            );
 
             apply_result.map_err(WalRedoError::IoError)
         } else {
-            error!("unexpected non-relation relish: {:?}", rel);
             Err(WalRedoError::InvalidRequest)
-        };
-
-        let end_time = Instant::now();
-        let duration = end_time.duration_since(lock_time);
-        WAL_REDO_TIME.observe(duration.as_secs_f64());
-        debug!(
-            "postgres applied {} WAL records in {} us to reconstruct page image at LSN {}",
-            records.len(),
-            duration.as_micros(),
-            lsn
-        );
-
-        // If something went wrong, don't try to reuse the process. Kill it, and
-        // next request will launch a new one.
-        if result.is_err() {
-            let process = process_guard.take().unwrap();
-            process.kill();
         }
-        result
     }
 
     ///
-    /// Process a batch of WAL records using bespoken Zenith code.
+    /// Process one request for WAL redo using custom zenith code
     ///
-    fn apply_batch_zenith(
-        &self,
-        rel: RelishTag,
-        blknum: u32,
-        lsn: Lsn,
-        base_img: Option<Bytes>,
-        records: &[(Lsn, ZenithWalRecord)],
-    ) -> Result<Bytes, WalRedoError> {
-        let start_time = Instant::now();
+    fn handle_apply_request_zenith(&self, request: &WalRedoRequest) -> Result<Bytes, WalRedoError> {
+        let rel = request.rel;
+        let blknum = request.blknum;
+        let lsn = request.lsn;
+        let base_img = request.base_img.clone();
+        let records = &request.records;
 
+        let nrecords = records.len();
+
+        let start = Instant::now();
+
+        let apply_result: Result<Bytes, Error>;
+
+        // Non-relational WAL records are handled here, with custom code that has the
+        // same effects as the corresponding Postgres WAL redo function.
+        const ZERO_PAGE: [u8; 8192] = [0u8; 8192];
         let mut page = BytesMut::new();
         if let Some(fpi) = base_img {
             // If full-page image is provided, then use it...
             page.extend_from_slice(&fpi[..]);
         } else {
-            // All the current WAL record types that we can handle require a base image.
-            error!("invalid zenith WAL redo request with no base image");
-            return Err(WalRedoError::InvalidRequest);
+            // otherwise initialize page with zeros
+            page.extend_from_slice(&ZERO_PAGE);
+        }
+        // Apply all collected WAL records
+        for (_lsn, record) in records {
+            let mut buf = record.rec.clone();
+
+            WAL_REDO_RECORD_COUNTER.inc();
+
+            // 1. Parse XLogRecord struct
+            // FIXME: refactor to avoid code duplication.
+            let xlogrec = XLogRecord::from_bytes(&mut buf);
+
+            //move to main data
+            // TODO probably, we should store some records in our special format
+            // to avoid this weird parsing on replay
+            let skip = (record.main_data_offset - pg_constants::SIZEOF_XLOGRECORD) as usize;
+            if buf.remaining() > skip {
+                buf.advance(skip);
+            }
+
+            if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
+                // Transaction manager stuff
+                let rec_segno = match rel {
+                    RelishTag::Slru { slru, segno } => {
+                        assert!(
+                            slru == SlruKind::Clog,
+                            "Not valid XACT relish tag {:?}",
+                            rel
+                        );
+                        segno
+                    }
+                    _ => panic!("Not valid XACT relish tag {:?}", rel),
+                };
+                let parsed_xact =
+                    XlXactParsedRecord::decode(&mut buf, xlogrec.xl_xid, xlogrec.xl_info);
+                if parsed_xact.info == pg_constants::XLOG_XACT_COMMIT
+                    || parsed_xact.info == pg_constants::XLOG_XACT_COMMIT_PREPARED
+                {
+                    transaction_id_set_status(
+                        parsed_xact.xid,
+                        pg_constants::TRANSACTION_STATUS_COMMITTED,
+                        &mut page,
+                    );
+                    for subxact in &parsed_xact.subxacts {
+                        let pageno = *subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
+                        let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+                        let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+                        // only update xids on the requested page
+                        if rec_segno == segno && blknum == rpageno {
+                            transaction_id_set_status(
+                                *subxact,
+                                pg_constants::TRANSACTION_STATUS_COMMITTED,
+                                &mut page,
+                            );
+                        }
+                    }
+                } else if parsed_xact.info == pg_constants::XLOG_XACT_ABORT
+                    || parsed_xact.info == pg_constants::XLOG_XACT_ABORT_PREPARED
+                {
+                    transaction_id_set_status(
+                        parsed_xact.xid,
+                        pg_constants::TRANSACTION_STATUS_ABORTED,
+                        &mut page,
+                    );
+                    for subxact in &parsed_xact.subxacts {
+                        let pageno = *subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
+                        let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+                        let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+                        // only update xids on the requested page
+                        if rec_segno == segno && blknum == rpageno {
+                            transaction_id_set_status(
+                                *subxact,
+                                pg_constants::TRANSACTION_STATUS_ABORTED,
+                                &mut page,
+                            );
+                        }
+                    }
+                }
+            } else if xlogrec.xl_rmid == pg_constants::RM_MULTIXACT_ID {
+                // Multixact operations
+                let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
+                if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
+                    let xlrec = XlMultiXactCreate::decode(&mut buf);
+                    if let RelishTag::Slru {
+                        slru,
+                        segno: rec_segno,
+                    } = rel
+                    {
+                        if slru == SlruKind::MultiXactMembers {
+                            for i in 0..xlrec.nmembers {
+                                let pageno = i / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
+                                let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
+                                let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
+                                if segno == rec_segno && rpageno == blknum {
+                                    // update only target block
+                                    let offset = xlrec.moff + i;
+                                    let memberoff = mx_offset_to_member_offset(offset);
+                                    let flagsoff = mx_offset_to_flags_offset(offset);
+                                    let bshift = mx_offset_to_flags_bitshift(offset);
+                                    let mut flagsval =
+                                        LittleEndian::read_u32(&page[flagsoff..flagsoff + 4]);
+                                    flagsval &=
+                                        !(((1 << pg_constants::MXACT_MEMBER_BITS_PER_XACT) - 1)
+                                            << bshift);
+                                    flagsval |= xlrec.members[i as usize].status << bshift;
+                                    LittleEndian::write_u32(
+                                        &mut page[flagsoff..flagsoff + 4],
+                                        flagsval,
+                                    );
+                                    LittleEndian::write_u32(
+                                        &mut page[memberoff..memberoff + 4],
+                                        xlrec.members[i as usize].xid,
+                                    );
+                                }
+                            }
+                        } else {
+                            // Multixact offsets SLRU
+                            let offs = (xlrec.mid % pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32
+                                * 4) as usize;
+                            LittleEndian::write_u32(&mut page[offs..offs + 4], xlrec.moff);
+                        }
+                    } else {
+                        panic!();
+                    }
+                } else {
+                    panic!();
+                }
+            }
         }
 
-        // Apply all the WAL records in the batch
-        for (record_lsn, record) in records.iter() {
-            self.apply_record_zenith(rel, blknum, &mut page, *record_lsn, record)?;
-        }
-        // Success!
-        let end_time = Instant::now();
-        let duration = end_time.duration_since(start_time);
-        WAL_REDO_TIME.observe(duration.as_secs_f64());
+        apply_result = Ok::<Bytes, Error>(page.freeze());
+
+        let duration = start.elapsed();
 
         debug!(
             "zenith applied {} WAL records in {} ms to reconstruct page image at LSN {}",
-            records.len(),
-            duration.as_micros(),
+            nrecords,
+            duration.as_millis(),
             lsn
         );
 
-        Ok(page.freeze())
-    }
-
-    fn apply_record_zenith(
-        &self,
-        rel: RelishTag,
-        blknum: u32,
-        page: &mut BytesMut,
-        _record_lsn: Lsn,
-        record: &ZenithWalRecord,
-    ) -> Result<(), WalRedoError> {
-        match record {
-            ZenithWalRecord::Postgres {
-                will_init: _,
-                rec: _,
-            } => panic!("tried to pass postgres wal record to zenith WAL redo"),
-            ZenithWalRecord::ClearVisibilityMapFlags { heap_blkno, flags } => {
-                // Calculate the VM block and offset that corresponds to the heap block.
-                let map_block = pg_constants::HEAPBLK_TO_MAPBLOCK(*heap_blkno);
-                let map_byte = pg_constants::HEAPBLK_TO_MAPBYTE(*heap_blkno);
-                let map_offset = pg_constants::HEAPBLK_TO_OFFSET(*heap_blkno);
-
-                // Check that we're modifying the correct VM block.
-                assert!(
-                    check_forknum(&rel, pg_constants::VISIBILITYMAP_FORKNUM),
-                    "ClearVisibilityMapFlags record on unexpected rel {:?}",
-                    rel
-                );
-                assert!(map_block == blknum);
-
-                // equivalent to PageGetContents(page)
-                let map = &mut page[pg_constants::MAXALIGN_SIZE_OF_PAGE_HEADER_DATA..];
-
-                let mask: u8 = flags << map_offset;
-                map[map_byte as usize] &= !mask;
-            }
-            // Non-relational WAL records are handled here, with custom code that has the
-            // same effects as the corresponding Postgres WAL redo function.
-            ZenithWalRecord::ClogSetCommitted { xids } => {
-                for &xid in xids {
-                    let pageno = xid as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
-                    let expected_segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-                    let expected_blknum = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-
-                    // Check that we're modifying the correct CLOG block.
-                    assert!(
-                        check_slru_segno(&rel, SlruKind::Clog, expected_segno),
-                        "ClogSetCommitted record for XID {} with unexpected rel {:?}",
-                        xid,
-                        rel
-                    );
-                    assert!(blknum == expected_blknum);
-
-                    transaction_id_set_status(
-                        xid,
-                        pg_constants::TRANSACTION_STATUS_COMMITTED,
-                        page,
-                    );
-                }
-            }
-            ZenithWalRecord::ClogSetAborted { xids } => {
-                for &xid in xids {
-                    let pageno = xid as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
-                    let expected_segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-                    let expected_blknum = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-
-                    // Check that we're modifying the correct CLOG block.
-                    assert!(
-                        check_slru_segno(&rel, SlruKind::Clog, expected_segno),
-                        "ClogSetCommitted record for XID {} with unexpected rel {:?}",
-                        xid,
-                        rel
-                    );
-                    assert!(blknum == expected_blknum);
-
-                    transaction_id_set_status(xid, pg_constants::TRANSACTION_STATUS_ABORTED, page);
-                }
-            }
-            ZenithWalRecord::MultixactOffsetCreate { mid, moff } => {
-                // Compute the block and offset to modify.
-                // See RecordNewMultiXact in PostgreSQL sources.
-                let pageno = mid / pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32;
-                let entryno = mid % pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32;
-                let offset = (entryno * 4) as usize;
-
-                // Check that we're modifying the correct multixact-offsets block.
-                let expected_segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-                let expected_blknum = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-                assert!(
-                    check_slru_segno(&rel, SlruKind::MultiXactOffsets, expected_segno),
-                    "MultiXactOffsetsCreate record for multi-xid {} with unexpected rel {:?}",
-                    mid,
-                    rel
-                );
-                assert!(blknum == expected_blknum);
-
-                LittleEndian::write_u32(&mut page[offset..offset + 4], *moff);
-            }
-            ZenithWalRecord::MultixactMembersCreate { moff, members } => {
-                for (i, member) in members.iter().enumerate() {
-                    let offset = moff + i as u32;
-
-                    // Compute the block and offset to modify.
-                    // See RecordNewMultiXact in PostgreSQL sources.
-                    let pageno = offset / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-                    let memberoff = mx_offset_to_member_offset(offset);
-                    let flagsoff = mx_offset_to_flags_offset(offset);
-                    let bshift = mx_offset_to_flags_bitshift(offset);
-
-                    // Check that we're modifying the correct multixact-members block.
-                    let expected_segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
-                    let expected_blknum = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
-                    assert!(
-                        check_slru_segno(&rel, SlruKind::MultiXactMembers, expected_segno),
-                        "MultiXactMembersCreate record at offset {} with unexpected rel {:?}",
-                        moff,
-                        rel
-                    );
-                    assert!(blknum == expected_blknum);
-
-                    let mut flagsval = LittleEndian::read_u32(&page[flagsoff..flagsoff + 4]);
-                    flagsval &= !(((1 << pg_constants::MXACT_MEMBER_BITS_PER_XACT) - 1) << bshift);
-                    flagsval |= member.status << bshift;
-                    LittleEndian::write_u32(&mut page[flagsoff..flagsoff + 4], flagsval);
-                    LittleEndian::write_u32(&mut page[memberoff..memberoff + 4], member.xid);
-                }
-            }
-        }
-
-        Ok(())
+        apply_result.map_err(WalRedoError::IoError)
     }
 }
 
@@ -484,17 +469,18 @@ impl PostgresRedoManager {
 /// Handle to the Postgres WAL redo process
 ///
 struct PostgresRedoProcess {
-    child: Child,
     stdin: ChildStdin,
     stdout: ChildStdout,
-    stderr: ChildStderr,
 }
 
 impl PostgresRedoProcess {
     //
     // Start postgres binary in special WAL redo mode.
     //
-    fn launch(conf: &PageServerConf, tenantid: &ZTenantId) -> Result<PostgresRedoProcess, Error> {
+    async fn launch(
+        conf: &PageServerConf,
+        tenantid: &ZTenantId,
+    ) -> Result<PostgresRedoProcess, Error> {
         // FIXME: We need a dummy Postgres cluster to run the process in. Currently, we
         // just create one with constant name. That fails if you try to launch more than
         // one WAL redo manager concurrently.
@@ -515,6 +501,7 @@ impl PostgresRedoProcess {
             .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
             .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap())
             .output()
+            .await
             .expect("failed to execute initdb");
 
         if !initdb.status.success() {
@@ -551,147 +538,102 @@ impl PostgresRedoProcess {
             datadir.display()
         );
 
-        let stdin = child.stdin.take().unwrap();
-        let stdout = child.stdout.take().unwrap();
-        let stderr = child.stderr.take().unwrap();
+        let stdin = child.stdin.take().expect("failed to open child's stdin");
+        let stderr = child.stderr.take().expect("failed to open child's stderr");
+        let stdout = child.stdout.take().expect("failed to open child's stdout");
 
-        set_nonblock(stdin.as_raw_fd())?;
-        set_nonblock(stdout.as_raw_fd())?;
-        set_nonblock(stderr.as_raw_fd())?;
+        // This async block reads the child's stderr, and forwards it to the logger
+        let f_stderr = async {
+            let mut stderr_buffered = tokio::io::BufReader::new(stderr);
 
-        Ok(PostgresRedoProcess {
-            child,
-            stdin,
-            stdout,
-            stderr,
-        })
-    }
+            let mut line = String::new();
+            loop {
+                let res = stderr_buffered.read_line(&mut line).await;
+                if res.is_err() {
+                    debug!("could not convert line to utf-8");
+                    continue;
+                }
+                if res.unwrap() == 0 {
+                    break;
+                }
+                error!("wal-redo-postgres: {}", line.trim());
+                line.clear();
+            }
+            Ok::<(), Error>(())
+        };
+        tokio::spawn(f_stderr);
 
-    fn kill(mut self) {
-        let _ = self.child.kill();
-        if let Ok(exit_status) = self.child.wait() {
-            error!("wal-redo-postgres exited with code {}", exit_status);
-        }
-        drop(self);
+        Ok(PostgresRedoProcess { stdin, stdout })
     }
 
     //
     // Apply given WAL records ('records') over an old page image. Returns
     // new page image.
     //
-    fn apply_wal_records(
+    async fn apply_wal_records(
         &mut self,
         tag: BufferTag,
         base_img: Option<Bytes>,
-        records: &[(Lsn, ZenithWalRecord)],
+        records: &[(Lsn, WALRecord)],
     ) -> Result<Bytes, std::io::Error> {
-        // Serialize all the messages to send the WAL redo process first.
-        //
-        // This could be problematic if there are millions of records to replay,
-        // but in practice the number of records is usually so small that it doesn't
-        // matter, and it's better to keep this code simple.
-        let mut writebuf: Vec<u8> = Vec::new();
-        build_begin_redo_for_block_msg(tag, &mut writebuf);
-        if let Some(img) = base_img {
-            build_push_page_msg(tag, &img, &mut writebuf);
-        }
-        for (lsn, rec) in records.iter() {
-            if let ZenithWalRecord::Postgres {
-                will_init: _,
-                rec: postgres_rec,
-            } = rec
-            {
-                build_apply_record_msg(*lsn, postgres_rec, &mut writebuf);
-            } else {
-                panic!("tried to pass zenith wal record to postgres WAL redo");
-            }
-        }
-        build_get_page_msg(tag, &mut writebuf);
-        WAL_REDO_RECORD_COUNTER.inc_by(records.len() as u64);
-
-        // The input is now in 'writebuf'. Do a blind write first, writing as much as
-        // we can, before calling poll(). That skips one call to poll() if the stdin is
-        // already available for writing, which it almost certainly is because the
-        // process is idle.
-        let mut nwrite = self.stdin.write(&writebuf)?;
-
-        // We expect the WAL redo process to respond with an 8k page image. We read it
-        // into this buffer.
-        let mut resultbuf = vec![0; pg_constants::BLCKSZ.into()];
-        let mut nresult: usize = 0; // # of bytes read into 'resultbuf' so far
-
-        // Prepare for calling poll()
-        let mut pollfds = [
-            PollFd::new(self.stdout.as_raw_fd(), PollFlags::POLLIN),
-            PollFd::new(self.stderr.as_raw_fd(), PollFlags::POLLIN),
-            PollFd::new(self.stdin.as_raw_fd(), PollFlags::POLLOUT),
-        ];
+        let stdout = &mut self.stdout;
+        // Buffer the writes to avoid a lot of small syscalls.
+        let mut stdin = tokio::io::BufWriter::new(&mut self.stdin);
 
         // We do three things simultaneously: send the old base image and WAL records to
         // the child process's stdin, read the result from child's stdout, and forward any logging
         // information that the child writes to its stderr to the page server's log.
-        while nresult < pg_constants::BLCKSZ.into() {
-            // If we have more data to write, wake up if 'stdin' becomes writeable or
-            // we have data to read. Otherwise only wake up if there's data to read.
-            let nfds = if nwrite < writebuf.len() { 3 } else { 2 };
-            let n = nix::poll::poll(&mut pollfds[0..nfds], TIMEOUT.as_millis() as i32)?;
-
-            if n == 0 {
-                return Err(Error::new(ErrorKind::Other, "WAL redo timed out"));
+        //
+        // 'f_stdin' handles writing the base image and WAL records to the child process.
+        // 'f_stdout' below reads the result back. And 'f_stderr', which was spawned into the
+        // tokio runtime in the 'launch' function already, forwards the logging.
+        let f_stdin = async {
+            // Send base image, if any. (If the record initializes the page, previous page
+            // version is not needed.)
+            timeout(
+                TIMEOUT,
+                stdin.write_all(&build_begin_redo_for_block_msg(tag)),
+            )
+            .await??;
+            if let Some(img) = base_img {
+                timeout(TIMEOUT, stdin.write_all(&build_push_page_msg(tag, &img))).await??;
             }
 
-            // If we have some messages in stderr, forward them to the log.
-            let err_revents = pollfds[1].revents().unwrap();
-            if err_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() {
-                let mut errbuf: [u8; 16384] = [0; 16384];
-                let n = self.stderr.read(&mut errbuf)?;
+            // Send WAL records.
+            for (lsn, rec) in records.iter() {
+                WAL_REDO_RECORD_COUNTER.inc();
 
-                // The message might not be split correctly into lines here. But this is
-                // good enough, the important thing is to get the message to the log.
-                if n > 0 {
-                    error!(
-                        "wal-redo-postgres: {}",
-                        String::from_utf8_lossy(&errbuf[0..n])
-                    );
+                stdin
+                    .write_all(&build_apply_record_msg(*lsn, &rec.rec))
+                    .await?;
 
-                    // To make sure we capture all log from the process if it fails, keep
-                    // reading from the stderr, before checking the stdout.
-                    continue;
-                }
-            } else if err_revents.contains(PollFlags::POLLHUP) {
-                return Err(Error::new(
-                    ErrorKind::BrokenPipe,
-                    "WAL redo process closed its stderr unexpectedly",
-                ));
+                //debug!("sent WAL record to wal redo postgres process ({:X}/{:X}",
+                //       r.lsn >> 32, r.lsn & 0xffff_ffff);
             }
+            //debug!("sent {} WAL records to wal redo postgres process ({:X}/{:X}",
+            //       records.len(), lsn >> 32, lsn & 0xffff_ffff);
 
-            // If we have more data to write and 'stdin' is writeable, do write.
-            if nwrite < writebuf.len() {
-                let in_revents = pollfds[2].revents().unwrap();
-                if in_revents & (PollFlags::POLLERR | PollFlags::POLLOUT) != PollFlags::empty() {
-                    nwrite += self.stdin.write(&writebuf[nwrite..])?;
-                } else if in_revents.contains(PollFlags::POLLHUP) {
-                    // We still have more data to write, but the process closed the pipe.
-                    return Err(Error::new(
-                        ErrorKind::BrokenPipe,
-                        "WAL redo process closed its stdin unexpectedly",
-                    ));
-                }
-            }
+            // Send GetPage command to get the result back
+            timeout(TIMEOUT, stdin.write_all(&build_get_page_msg(tag))).await??;
+            timeout(TIMEOUT, stdin.flush()).await??;
+            //debug!("sent GetPage for {}", tag.blknum);
+            Ok::<(), Error>(())
+        };
 
-            // If we have some data in stdout, read it to the result buffer.
-            let out_revents = pollfds[0].revents().unwrap();
-            if out_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() {
-                nresult += self.stdout.read(&mut resultbuf[nresult..])?;
-            } else if out_revents.contains(PollFlags::POLLHUP) {
-                return Err(Error::new(
-                    ErrorKind::BrokenPipe,
-                    "WAL redo process closed its stdout unexpectedly",
-                ));
-            }
-        }
+        // Read back new page image
+        let f_stdout = async {
+            let mut buf = [0u8; 8192];
 
-        Ok(Bytes::from(resultbuf))
+            timeout(TIMEOUT, stdout.read_exact(&mut buf)).await??;
+            //debug!("got response for {}", tag.blknum);
+            Ok::<[u8; 8192], Error>(buf)
+        };
+
+        let res = tokio::try_join!(f_stdout, f_stdin)?;
+
+        let buf = res.0;
+
+        Ok::<Bytes, Error>(Bytes::from(std::vec::Vec::from(buf)))
     }
 }
 
@@ -699,42 +641,62 @@ impl PostgresRedoProcess {
 // process. See vendor/postgres/src/backend/tcop/zenith_wal_redo.c for
 // explanation of the protocol.
 
-fn build_begin_redo_for_block_msg(tag: BufferTag, buf: &mut Vec<u8>) {
+fn build_begin_redo_for_block_msg(tag: BufferTag) -> Vec<u8> {
     let len = 4 + 1 + 4 * 4;
+    let mut buf = Vec::with_capacity(1 + len);
 
     buf.put_u8(b'B');
     buf.put_u32(len as u32);
 
-    tag.ser_into(buf)
+    tag.ser_into(&mut buf)
         .expect("serialize BufferTag should always succeed");
+
+    debug_assert!(buf.len() == 1 + len);
+
+    buf
 }
 
-fn build_push_page_msg(tag: BufferTag, base_img: &[u8], buf: &mut Vec<u8>) {
+fn build_push_page_msg(tag: BufferTag, base_img: &[u8]) -> Vec<u8> {
     assert!(base_img.len() == 8192);
 
     let len = 4 + 1 + 4 * 4 + base_img.len();
+    let mut buf = Vec::with_capacity(1 + len);
 
     buf.put_u8(b'P');
     buf.put_u32(len as u32);
-    tag.ser_into(buf)
+    tag.ser_into(&mut buf)
         .expect("serialize BufferTag should always succeed");
     buf.put(base_img);
+
+    debug_assert!(buf.len() == 1 + len);
+
+    buf
 }
 
-fn build_apply_record_msg(endlsn: Lsn, rec: &[u8], buf: &mut Vec<u8>) {
+fn build_apply_record_msg(endlsn: Lsn, rec: &[u8]) -> Vec<u8> {
     let len = 4 + 8 + rec.len();
+    let mut buf: Vec<u8> = Vec::with_capacity(1 + len);
 
     buf.put_u8(b'A');
     buf.put_u32(len as u32);
     buf.put_u64(endlsn.0);
     buf.put(rec);
+
+    debug_assert!(buf.len() == 1 + len);
+
+    buf
 }
 
-fn build_get_page_msg(tag: BufferTag, buf: &mut Vec<u8>) {
+fn build_get_page_msg(tag: BufferTag) -> Vec<u8> {
     let len = 4 + 1 + 4 * 4;
+    let mut buf = Vec::with_capacity(1 + len);
 
     buf.put_u8(b'G');
     buf.put_u32(len as u32);
-    tag.ser_into(buf)
+    tag.ser_into(&mut buf)
         .expect("serialize BufferTag should always succeed");
+
+    debug_assert!(buf.len() == 1 + len);
+
+    buf
 }

postgres_ffi/build.rs

@@ -75,7 +75,6 @@ fn main() {
         .allowlist_var("XLOG_PAGE_MAGIC")
         .allowlist_var("PG_CONTROL_FILE_SIZE")
         .allowlist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
-        .allowlist_type("PageHeaderData")
         .allowlist_type("DBState")
         // Because structs are used for serialization, tell bindgen to emit
         // explicit padding fields.

postgres_ffi/pg_control_ffi.h

@@ -9,6 +9,5 @@
 #include "access/xlog_internal.h"
 
 #include "storage/block.h"
-#include "storage/bufpage.h"
 #include "storage/off.h"
 #include "access/multixact.h"

postgres_ffi/src/lib.rs

@@ -1,8 +1,6 @@
 #![allow(non_upper_case_globals)]
 #![allow(non_camel_case_types)]
 #![allow(non_snake_case)]
-// bindgen creates some unsafe code with no doc comments.
-#![allow(clippy::missing_safety_doc)]
 // suppress warnings on rust 1.53 due to bindgen unit tests.
 // https://github.com/rust-lang/rust-bindgen/issues/1651
 #![allow(deref_nullptr)]
@@ -15,7 +13,6 @@ pub mod controlfile_utils;
 pub mod nonrelfile_utils;
 pub mod pg_constants;
 pub mod relfile_utils;
-pub mod waldecoder;
 pub mod xlog_utils;
 
 //  See TransactionIdIsNormal in transam.h

postgres_ffi/src/pg_constants.rs

@@ -7,8 +7,6 @@
 //! comments on them.
 //!
 
-use crate::PageHeaderData;
-
 //
 // From pg_tablespace_d.h
 //
@@ -33,14 +31,6 @@ pub const SMGR_TRUNCATE_FSM: u32 = 0x0004;
 pub const BLCKSZ: u16 = 8192;
 pub const RELSEG_SIZE: u32 = 1024 * 1024 * 1024 / (BLCKSZ as u32);
 
-//
-// From bufpage.h
-//
-
-// Assumes 8 byte alignment
-const SIZEOF_PAGE_HEADER_DATA: usize = std::mem::size_of::<PageHeaderData>();
-pub const MAXALIGN_SIZE_OF_PAGE_HEADER_DATA: usize = (SIZEOF_PAGE_HEADER_DATA + 7) & !7;
-
 //
 // constants from clog.h
 //
@@ -49,6 +39,13 @@ pub const CLOG_XACTS_PER_PAGE: u32 = BLCKSZ as u32 * CLOG_XACTS_PER_BYTE;
 pub const CLOG_BITS_PER_XACT: u8 = 2;
 pub const CLOG_XACT_BITMASK: u8 = (1 << CLOG_BITS_PER_XACT) - 1;
 
+//
+// Constants from visbilitymap.h
+//
+pub const SIZE_OF_PAGE_HEADER: u16 = 24;
+pub const BITS_PER_HEAPBLOCK: u16 = 2;
+pub const HEAPBLOCKS_PER_PAGE: u16 = (BLCKSZ - SIZE_OF_PAGE_HEADER) * 8 / BITS_PER_HEAPBLOCK;
+
 pub const TRANSACTION_STATUS_COMMITTED: u8 = 0x01;
 pub const TRANSACTION_STATUS_ABORTED: u8 = 0x02;
 pub const TRANSACTION_STATUS_SUB_COMMITTED: u8 = 0x03;
@@ -56,30 +53,6 @@ pub const TRANSACTION_STATUS_SUB_COMMITTED: u8 = 0x03;
 pub const CLOG_ZEROPAGE: u8 = 0x00;
 pub const CLOG_TRUNCATE: u8 = 0x10;
 
-//
-// Constants from visibilitymap.h, visibilitymapdefs.h and visibilitymap.c
-//
-pub const SIZE_OF_PAGE_HEADER: u16 = 24;
-pub const BITS_PER_BYTE: u16 = 8;
-pub const HEAPBLOCKS_PER_PAGE: u32 =
-    (BLCKSZ - SIZE_OF_PAGE_HEADER) as u32 * 8 / BITS_PER_HEAPBLOCK as u32;
-pub const HEAPBLOCKS_PER_BYTE: u16 = BITS_PER_BYTE / BITS_PER_HEAPBLOCK;
-
-pub const fn HEAPBLK_TO_MAPBLOCK(x: u32) -> u32 {
-    x / HEAPBLOCKS_PER_PAGE
-}
-pub const fn HEAPBLK_TO_MAPBYTE(x: u32) -> u32 {
-    (x % HEAPBLOCKS_PER_PAGE) / HEAPBLOCKS_PER_BYTE as u32
-}
-pub const fn HEAPBLK_TO_OFFSET(x: u32) -> u32 {
-    (x % HEAPBLOCKS_PER_BYTE as u32) * BITS_PER_HEAPBLOCK as u32
-}
-
-pub const BITS_PER_HEAPBLOCK: u16 = 2;
-pub const VISIBILITYMAP_ALL_VISIBLE: u8 = 0x01;
-pub const VISIBILITYMAP_ALL_FROZEN: u8 = 0x02;
-pub const VISIBILITYMAP_VALID_BITS: u8 = 0x03;
-
 // From xact.h
 pub const XLOG_XACT_COMMIT: u8 = 0x00;
 pub const XLOG_XACT_PREPARE: u8 = 0x10;