Fix cherry-pick

.circleci/ansible/.gitignore

@@ -1,4 +0,0 @@
-zenith_install.tar.gz
-.zenith_current_version
-neon_install.tar.gz
-.neon_current_version

.circleci/ansible/deploy.yaml

@@ -1,29 +1,67 @@
-- name: Upload Neon binaries
-  hosts: storage
+- name: Upload Zenith binaries
+  hosts: pageservers:safekeepers
   gather_facts: False
   remote_user: admin
+  vars:
+    force_deploy: false
 
   tasks:
 
-    - name: get latest version of Neon binaries
+    - name: get latest version of Zenith binaries
+      ignore_errors: true
       register: current_version_file
       set_fact:
-        current_version: "{{ lookup('file', '.neon_current_version') | trim }}"
+        current_version: "{{ lookup('file', '.zenith_current_version') | trim }}"
+      tags:
+      - pageserver
+      - safekeeper
+
+    - name: set zero value for current_version
+      when: current_version_file is failed
+      set_fact:
+        current_version: "0"
+      tags:
+      - pageserver
+      - safekeeper
+
+    - name: get deployed version from content of remote file
+      ignore_errors: true
+      ansible.builtin.slurp:
+        src: /usr/local/.zenith_current_version
+      register: remote_version_file
+      tags:
+      - pageserver
+      - safekeeper
+
+    - name: decode remote file content
+      when: remote_version_file is succeeded
+      set_fact:
+        remote_version: "{{ remote_version_file['content'] | b64decode | trim }}"
+      tags:
+      - pageserver
+      - safekeeper
+
+    - name: set zero value for remote_version
+      when: remote_version_file is failed
+      set_fact:
+        remote_version: "0"
       tags:
       - pageserver
       - safekeeper
 
     - name: inform about versions
-      debug: msg="Version to deploy - {{ current_version }}"
+      debug: msg="Version to deploy - {{ current_version }}, version on storage node - {{ remote_version }}"
       tags:
       - pageserver
       - safekeeper
 
-    - name: upload and extract Neon binaries to /usr/local
+
+    - name: upload and extract Zenith binaries to /usr/local
+      when: current_version > remote_version or force_deploy
       ansible.builtin.unarchive:
         owner: root
         group: root
-        src: neon_install.tar.gz
+        src: zenith_install.tar.gz
         dest: /usr/local
       become: true
       tags:
@@ -36,24 +74,14 @@
   hosts: pageservers
   gather_facts: False
   remote_user: admin
+  vars:
+    force_deploy: false
 
   tasks:
-
-    - name: upload init script
-      when: console_mgmt_base_url is defined
-      ansible.builtin.template:
-        src: scripts/init_pageserver.sh
-        dest: /tmp/init_pageserver.sh
-        owner: root
-        group: root
-        mode: '0755'
-      become: true
-      tags:
-      - pageserver
-
     - name: init pageserver
+      when: current_version > remote_version or force_deploy
       shell:
-        cmd: /tmp/init_pageserver.sh
+        cmd: sudo -u pageserver /usr/local/bin/pageserver -c "pg_distrib_dir='/usr/local'" --init -D /storage/pageserver/data
       args:
         creates: "/storage/pageserver/data/tenants"
       environment:
@@ -63,20 +91,8 @@
       tags:
       - pageserver
 
-    - name: update remote storage (s3) config
-      lineinfile:
-        path: /storage/pageserver/data/pageserver.toml
-        line: "{{ item }}"
-      loop:
-        - "[remote_storage]"
-        - "bucket_name = '{{ bucket_name }}'"
-        - "bucket_region = '{{ bucket_region }}'"
-        - "prefix_in_bucket = '{{ inventory_hostname }}'"
-      become: true
-      tags:
-      - pageserver
-
     - name: upload systemd service definition
+      when: current_version > remote_version or force_deploy
       ansible.builtin.template:
         src: systemd/pageserver.service
         dest: /etc/systemd/system/pageserver.service
@@ -88,6 +104,7 @@
       - pageserver
 
     - name: start systemd service
+      when: current_version > remote_version or force_deploy
       ansible.builtin.systemd:
         daemon_reload: yes
         name: pageserver
@@ -98,7 +115,7 @@
       - pageserver
 
     - name: post version to console
-      when: console_mgmt_base_url is defined
+      when: (current_version > remote_version or force_deploy) and console_mgmt_base_url is defined
       shell:
         cmd: |
           INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
@@ -110,42 +127,22 @@
   hosts: safekeepers
   gather_facts: False
   remote_user: admin
+  vars:
+    force_deploy: false
 
   tasks:
 
-    - name: upload init script
-      when: console_mgmt_base_url is defined
-      ansible.builtin.template:
-        src: scripts/init_safekeeper.sh
-        dest: /tmp/init_safekeeper.sh
-        owner: root
-        group: root
-        mode: '0755'
-      become: true
-      tags:
-      - safekeeper
-
-    - name: init safekeeper
-      shell:
-        cmd: /tmp/init_safekeeper.sh
-      args:
-        creates: "/storage/safekeeper/data/safekeeper.id"
-      environment:
-        ZENITH_REPO_DIR: "/storage/safekeeper/data"
-        LD_LIBRARY_PATH: "/usr/local/lib"
-      become: true
-      tags:
-      - safekeeper
-
     # in the future safekeepers should discover pageservers byself
     # but currently use first pageserver that was discovered
     - name: set first pageserver var for safekeepers
+      when: current_version > remote_version or force_deploy
       set_fact:
         first_pageserver: "{{ hostvars[groups['pageservers'][0]]['inventory_hostname'] }}"
       tags:
       - safekeeper
 
     - name: upload systemd service definition
+      when: current_version > remote_version or force_deploy
       ansible.builtin.template:
         src: systemd/safekeeper.service
         dest: /etc/systemd/system/safekeeper.service
@@ -157,6 +154,7 @@
       - safekeeper
 
     - name: start systemd service
+      when: current_version > remote_version or force_deploy
       ansible.builtin.systemd:
         daemon_reload: yes
         name: safekeeper
@@ -167,7 +165,7 @@
       - safekeeper
 
     - name: post version to console
-      when: console_mgmt_base_url is defined
+      when: (current_version > remote_version or force_deploy) and console_mgmt_base_url is defined
       shell:
         cmd: |
           INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)

.circleci/ansible/get_binaries.sh

@@ -4,10 +4,10 @@ set -e
 
 RELEASE=${RELEASE:-false}
 
-# look at docker hub for latest tag for neon docker image
+# look at docker hub for latest tag fo zenith docker image
 if [ "${RELEASE}" = "true" ]; then
     echo "search latest relase tag"
-    VERSION=$(curl -s https://registry.hub.docker.com/v1/repositories/neondatabase/neon/tags |jq -r -S '.[].name' | grep release | sed 's/release-//g' | grep -E '^[0-9]+$' | sort -n | tail -1)
+    VERSION=$(curl -s https://registry.hub.docker.com/v1/repositories/zenithdb/zenith/tags |jq -r -S '.[].name' | grep release | sed 's/release-//g' | tail -1)
     if [ -z "${VERSION}" ]; then
         echo "no any docker tags found, exiting..."
         exit 1
@@ -16,7 +16,7 @@ if [ "${RELEASE}" = "true" ]; then
     fi
 else
     echo "search latest dev tag"
-    VERSION=$(curl -s https://registry.hub.docker.com/v1/repositories/neondatabase/neon/tags |jq -r -S '.[].name' | grep -E '^[0-9]+$' | sort -n | tail -1)
+    VERSION=$(curl -s https://registry.hub.docker.com/v1/repositories/zenithdb/zenith/tags |jq -r -S '.[].name' | grep -v release | tail -1)
     if [ -z "${VERSION}" ]; then
         echo "no any docker tags found, exiting..."
         exit 1
@@ -28,25 +28,25 @@ fi
 echo "found ${VERSION}"
 
 # do initial cleanup
-rm -rf neon_install postgres_install.tar.gz neon_install.tar.gz .neon_current_version
-mkdir neon_install
+rm -rf zenith_install postgres_install.tar.gz zenith_install.tar.gz .zenith_current_version
+mkdir zenith_install
 
 # retrive binaries from docker image
 echo "getting binaries from docker image"
-docker pull --quiet neondatabase/neon:${TAG}
-ID=$(docker create neondatabase/neon:${TAG})
+docker pull --quiet zenithdb/zenith:${TAG}
+ID=$(docker create zenithdb/zenith:${TAG})
 docker cp ${ID}:/data/postgres_install.tar.gz .
-tar -xzf postgres_install.tar.gz -C neon_install
-docker cp ${ID}:/usr/local/bin/pageserver neon_install/bin/
-docker cp ${ID}:/usr/local/bin/safekeeper neon_install/bin/
-docker cp ${ID}:/usr/local/bin/proxy neon_install/bin/
-docker cp ${ID}:/usr/local/bin/postgres neon_install/bin/
+tar -xzf postgres_install.tar.gz -C zenith_install
+docker cp ${ID}:/usr/local/bin/pageserver zenith_install/bin/
+docker cp ${ID}:/usr/local/bin/safekeeper zenith_install/bin/
+docker cp ${ID}:/usr/local/bin/proxy zenith_install/bin/
+docker cp ${ID}:/usr/local/bin/postgres zenith_install/bin/
 docker rm -vf ${ID}
 
 # store version to file (for ansible playbooks) and create binaries tarball
-echo ${VERSION} > neon_install/.neon_current_version
-echo ${VERSION} > .neon_current_version
-tar -czf neon_install.tar.gz -C neon_install .
+echo ${VERSION} > zenith_install/.zenith_current_version
+echo ${VERSION} > .zenith_current_version
+tar -czf zenith_install.tar.gz -C zenith_install .
 
 # do final cleaup
-rm -rf neon_install postgres_install.tar.gz
+rm -rf zenith_install postgres_install.tar.gz

.circleci/ansible/production.hosts

@@ -1,18 +1,7 @@
 [pageservers]
-zenith-1-ps-1 console_region_id=1
+zenith-1-ps-1
 
 [safekeepers]
-zenith-1-sk-1 console_region_id=1
-zenith-1-sk-2 console_region_id=1
-zenith-1-sk-3 console_region_id=1
-
-[storage:children]
-pageservers
-safekeepers
-
-[storage:vars]
-console_mgmt_base_url = http://console-release.local
-bucket_name           = zenith-storage-oregon
-bucket_region         = us-west-2
-etcd_endpoints        = etcd-release.local:2379
-safekeeper_enable_s3_offload = true
+zenith-1-sk-1
+zenith-1-sk-2
+zenith-1-sk-3

.circleci/ansible/scripts/init_pageserver.sh

@@ -1,30 +0,0 @@
-#!/bin/sh
-
-# get instance id from meta-data service
-INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
-
-# store fqdn hostname in var
-HOST=$(hostname -f)
-
-
-cat <<EOF | tee /tmp/payload
-{
-  "version": 1,
-  "host": "${HOST}",
-  "port": 6400,
-  "region_id": {{ console_region_id }},
-  "instance_id": "${INSTANCE_ID}",
-  "http_host": "${HOST}",
-  "http_port": 9898
-}
-EOF
-
-# check if pageserver already registered or not
-if ! curl -sf -X PATCH -d '{}' {{ console_mgmt_base_url }}/api/v1/pageservers/${INSTANCE_ID} -o /dev/null; then
-
-    # not registered, so register it now
-    ID=$(curl -sf -X POST {{ console_mgmt_base_url }}/api/v1/pageservers -d@/tmp/payload | jq -r '.ID')
-
-    # init pageserver
-    sudo -u pageserver /usr/local/bin/pageserver -c "id=${ID}" -c "pg_distrib_dir='/usr/local'" --init -D /storage/pageserver/data
-fi

.circleci/ansible/scripts/init_safekeeper.sh

@@ -1,30 +0,0 @@
-#!/bin/sh
-
-# get instance id from meta-data service
-INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
-
-# store fqdn hostname in var
-HOST=$(hostname -f)
-
-
-cat <<EOF | tee /tmp/payload
-{
-  "version": 1,
-  "host": "${HOST}",
-  "port": 6500,
-  "region_id": {{ console_region_id }},
-  "instance_id": "${INSTANCE_ID}",
-  "http_host": "${HOST}",
-  "http_port": 7676
-}
-EOF
-
-# check if safekeeper already registered or not
-if ! curl -sf -X PATCH -d '{}' {{ console_mgmt_base_url }}/api/v1/safekeepers/${INSTANCE_ID} -o /dev/null; then
-
-    # not registered, so register it now
-    ID=$(curl -sf -X POST {{ console_mgmt_base_url }}/api/v1/safekeepers -d@/tmp/payload | jq -r '.ID')
-
-    # init safekeeper
-    sudo -u safekeeper /usr/local/bin/safekeeper --id ${ID} --init -D /storage/safekeeper/data
-fi

.circleci/ansible/staging.hosts

@@ -1,19 +1,7 @@
 [pageservers]
-#zenith-us-stage-ps-1 console_region_id=27
-zenith-us-stage-ps-2 console_region_id=27
+zenith-us-stage-ps-1
 
 [safekeepers]
-zenith-us-stage-sk-1 console_region_id=27
-zenith-us-stage-sk-4 console_region_id=27
-zenith-us-stage-sk-5 console_region_id=27
-
-[storage:children]
-pageservers
-safekeepers
-
-[storage:vars]
-console_mgmt_base_url = http://console-staging.local
-bucket_name           = zenith-staging-storage-us-east-1
-bucket_region         = us-east-1
-etcd_endpoints        = etcd-staging.local:2379
-safekeeper_enable_s3_offload = false
+zenith-us-stage-sk-1
+zenith-us-stage-sk-2
+zenith-us-stage-sk-3

.circleci/ansible/systemd/safekeeper.service

@@ -6,7 +6,7 @@ After=network.target auditd.service
 Type=simple
 User=safekeeper
 Environment=RUST_BACKTRACE=1 ZENITH_REPO_DIR=/storage/safekeeper/data LD_LIBRARY_PATH=/usr/local/lib
-ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}.local:6500 --listen-http {{ inventory_hostname }}.local:7676 -p {{ first_pageserver }}:6400 -D /storage/safekeeper/data --broker-endpoints={{ etcd_endpoints }} --enable-s3-offload={{ safekeeper_enable_s3_offload }}
+ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}.local:6500 --listen-http {{ inventory_hostname }}.local:7676 -p {{ first_pageserver }}:6400 -D /storage/safekeeper/data
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=mixed
 KillSignal=SIGINT

.circleci/config.yml

@@ -1,18 +1,18 @@
 version: 2.1
 
 executors:
-  neon-xlarge-executor:
+  zenith-xlarge-executor:
     resource_class: xlarge
     docker:
       # NB: when changed, do not forget to update rust image tag in all Dockerfiles
-      - image: zimg/rust:1.58
-  neon-executor:
+      - image: zimg/rust:1.56
+  zenith-executor:
     docker:
-      - image: zimg/rust:1.58
+      - image: zimg/rust:1.56
 
 jobs:
   check-codestyle-rust:
-    executor: neon-xlarge-executor
+    executor: zenith-xlarge-executor
     steps:
       - checkout
       - run:
@@ -22,7 +22,7 @@ jobs:
 
   # A job to build postgres
   build-postgres:
-    executor: neon-xlarge-executor
+    executor: zenith-xlarge-executor
     parameters:
       build_type:
         type: enum
@@ -34,13 +34,10 @@ jobs:
       - checkout
 
         # Grab the postgres git revision to build a cache key.
-        # Append makefile as it could change the way postgres is built.
         # Note this works even though the submodule hasn't been checkout out yet.
       - run:
           name: Get postgres cache key
-          command: |
-              git rev-parse HEAD:vendor/postgres > /tmp/cache-key-postgres
-              cat Makefile >> /tmp/cache-key-postgres
+          command: git rev-parse HEAD:vendor/postgres > /tmp/cache-key-postgres
 
       - restore_cache:
           name: Restore postgres cache
@@ -67,9 +64,9 @@ jobs:
           paths:
             - tmp_install
 
-  # A job to build Neon rust code
-  build-neon:
-    executor: neon-xlarge-executor
+  # A job to build zenith rust code
+  build-zenith:
+    executor: zenith-xlarge-executor
     parameters:
       build_type:
         type: enum
@@ -81,14 +78,11 @@ jobs:
       - checkout
 
         # Grab the postgres git revision to build a cache key.
-        # Append makefile as it could change the way postgres is built.
         # Note this works even though the submodule hasn't been checkout out yet.
       - run:
           name: Get postgres cache key
           command: |
             git rev-parse HEAD:vendor/postgres > /tmp/cache-key-postgres
-            cat Makefile >> /tmp/cache-key-postgres
-
 
       - restore_cache:
           name: Restore postgres cache
@@ -113,16 +107,11 @@ jobs:
               CARGO_FLAGS=
             elif [[ $BUILD_TYPE == "release" ]]; then
               cov_prefix=()
-              CARGO_FLAGS="--release --features profiling"
+              CARGO_FLAGS=--release
             fi
 
             export CARGO_INCREMENTAL=0
-            export CACHEPOT_BUCKET=zenith-rust-cachepot
-            export RUSTC_WRAPPER=cachepot
-            export AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}"
-            export AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}"
-            "${cov_prefix[@]}" mold -run cargo build $CARGO_FLAGS --features failpoints --bins --tests
-            cachepot -s
+            "${cov_prefix[@]}" mold -run cargo build $CARGO_FLAGS --bins --tests
 
       - save_cache:
           name: Save rust cache
@@ -132,19 +121,31 @@ jobs:
             - ~/.cargo/git
             - target
 
+        # Run style checks
+        # has to run separately from cargo fmt section
+        # since needs to run with dependencies
+      - run:
+          name: cargo clippy
+          command: |
+            if [[ $BUILD_TYPE == "debug" ]]; then
+              cov_prefix=(scripts/coverage "--profraw-prefix=$CIRCLE_JOB" --dir=/tmp/zenith/coverage run)
+            elif [[ $BUILD_TYPE == "release" ]]; then
+              cov_prefix=()
+            fi
+
+            "${cov_prefix[@]}" ./run_clippy.sh
+
         # Run rust unit tests
       - run:
           name: cargo test
           command: |
             if [[ $BUILD_TYPE == "debug" ]]; then
               cov_prefix=(scripts/coverage "--profraw-prefix=$CIRCLE_JOB" --dir=/tmp/zenith/coverage run)
-              CARGO_FLAGS=
             elif [[ $BUILD_TYPE == "release" ]]; then
               cov_prefix=()
-              CARGO_FLAGS=--release
             fi
 
-            "${cov_prefix[@]}" cargo test $CARGO_FLAGS
+            "${cov_prefix[@]}" cargo test
 
         # Install the rust binaries, for use by test jobs
       - run:
@@ -209,17 +210,17 @@ jobs:
             - "*"
 
   check-codestyle-python:
-    executor: neon-executor
+    executor: zenith-executor
     steps:
       - checkout
       - restore_cache:
           keys:
-            - v2-python-deps-{{ checksum "poetry.lock" }}
+            - v1-python-deps-{{ checksum "poetry.lock" }}
       - run:
           name: Install deps
           command: ./scripts/pysync
       - save_cache:
-          key: v2-python-deps-{{ checksum "poetry.lock" }}
+          key: v1-python-deps-{{ checksum "poetry.lock" }}
           paths:
             - /home/circleci/.cache/pypoetry/virtualenvs
       - run:
@@ -232,7 +233,7 @@ jobs:
           command: poetry run mypy .
 
   run-pytest:
-    executor: neon-executor
+    executor: zenith-executor
     parameters:
       # pytest args to specify the tests to run.
       #
@@ -273,12 +274,12 @@ jobs:
             - run: git submodule update --init --depth 1
       - restore_cache:
           keys:
-            - v2-python-deps-{{ checksum "poetry.lock" }}
+            - v1-python-deps-{{ checksum "poetry.lock" }}
       - run:
           name: Install deps
           command: ./scripts/pysync
       - save_cache:
-          key: v2-python-deps-{{ checksum "poetry.lock" }}
+          key: v1-python-deps-{{ checksum "poetry.lock" }}
           paths:
             - /home/circleci/.cache/pypoetry/virtualenvs
       - run:
@@ -355,7 +356,7 @@ jobs:
           when: always
           command: |
             du -sh /tmp/test_output/*
-            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "safekeeper.log" ! -name "regression.diffs" ! -name "junit.xml" ! -name "*.filediff" ! -name "*.stdout" ! -name "*.stderr" ! -name "flamegraph.svg" -delete
+            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "safekeeper.log" ! -name "regression.diffs" ! -name "junit.xml" ! -name "*.filediff" ! -name "*.stdout" ! -name "*.stderr" -delete
             du -sh /tmp/test_output/*
       - store_artifacts:
           path: /tmp/test_output
@@ -376,7 +377,7 @@ jobs:
             - "*"
 
   coverage-report:
-    executor: neon-xlarge-executor
+    executor: zenith-xlarge-executor
     steps:
       - attach_workspace:
           at: /tmp/zenith
@@ -391,7 +392,7 @@ jobs:
       - run:
           name: Build coverage report
           command: |
-            COMMIT_URL=https://github.com/neondatabase/neon/commit/$CIRCLE_SHA1
+            COMMIT_URL=https://github.com/zenithdb/zenith/commit/$CIRCLE_SHA1
 
             scripts/coverage \
               --dir=/tmp/zenith/coverage report \
@@ -402,11 +403,11 @@ jobs:
           name: Upload coverage report
           command: |
             LOCAL_REPO=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
-            REPORT_URL=https://neondatabase.github.io/zenith-coverage-data/$CIRCLE_SHA1
-            COMMIT_URL=https://github.com/neondatabase/neon/commit/$CIRCLE_SHA1
+            REPORT_URL=https://zenithdb.github.io/zenith-coverage-data/$CIRCLE_SHA1
+            COMMIT_URL=https://github.com/zenithdb/zenith/commit/$CIRCLE_SHA1
 
             scripts/git-upload \
-              --repo=https://$VIP_VAP_ACCESS_TOKEN@github.com/neondatabase/zenith-coverage-data.git \
+              --repo=https://$VIP_VAP_ACCESS_TOKEN@github.com/zenithdb/zenith-coverage-data.git \
               --message="Add code coverage for $COMMIT_URL" \
               copy /tmp/zenith/coverage/report $CIRCLE_SHA1 # COPY FROM TO_RELATIVE
 
@@ -423,7 +424,7 @@ jobs:
                 \"target_url\": \"$REPORT_URL\"
               }"
 
-  # Build neondatabase/neon:latest image and push it to Docker hub
+  # Build zenithdb/zenith:latest image and push it to Docker hub
   docker-image:
     docker:
       - image: cimg/base:2021.04
@@ -437,18 +438,18 @@ jobs:
       - run:
           name: Build and push Docker image
           command: |
-            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
             DOCKER_TAG=$(git log --oneline|wc -l)
             docker build \
               --pull \
               --build-arg GIT_VERSION=${CIRCLE_SHA1} \
               --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
               --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
-              --tag neondatabase/neon:${DOCKER_TAG} --tag neondatabase/neon:latest .
-            docker push neondatabase/neon:${DOCKER_TAG}
-            docker push neondatabase/neon:latest
+              --tag zenithdb/zenith:${DOCKER_TAG} --tag zenithdb/zenith:latest .
+            docker push zenithdb/zenith:${DOCKER_TAG}
+            docker push zenithdb/zenith:latest
 
-  # Build neondatabase/compute-node:latest image and push it to Docker hub
+  # Build zenithdb/compute-node:latest image and push it to Docker hub
   docker-image-compute:
     docker:
       - image: cimg/base:2021.04
@@ -456,31 +457,28 @@ jobs:
       - checkout
       - setup_remote_docker:
           docker_layer_caching: true
-      # Build neondatabase/compute-tools:latest image and push it to Docker hub
+      # Build zenithdb/compute-tools:latest image and push it to Docker hub
       # TODO: this should probably also use versioned tag, not just :latest.
       # XXX: but should it? We build and use it only locally now.
       - run:
           name: Build and push compute-tools Docker image
           command: |
-            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
-            docker build \
-              --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
-              --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
-              --tag neondatabase/compute-tools:latest -f Dockerfile.compute-tools .
-            docker push neondatabase/compute-tools:latest
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
+            docker build -t zenithdb/compute-tools:latest -f Dockerfile.compute-tools .
+            docker push zenithdb/compute-tools:latest
       - run:
           name: Init postgres submodule
           command: git submodule update --init --depth 1
       - run:
           name: Build and push compute-node Docker image
           command: |
-            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
             DOCKER_TAG=$(git log --oneline|wc -l)
-            docker build --tag neondatabase/compute-node:${DOCKER_TAG} --tag neondatabase/compute-node:latest vendor/postgres
-            docker push neondatabase/compute-node:${DOCKER_TAG}
-            docker push neondatabase/compute-node:latest
+            docker build --tag zenithdb/compute-node:${DOCKER_TAG} --tag zenithdb/compute-node:latest vendor/postgres
+            docker push zenithdb/compute-node:${DOCKER_TAG}
+            docker push zenithdb/compute-node:latest
 
-  # Build production neondatabase/neon:release image and push it to Docker hub
+  # Build production zenithdb/zenith:release image and push it to Docker hub
   docker-image-release:
     docker:
       - image: cimg/base:2021.04
@@ -494,18 +492,18 @@ jobs:
       - run:
           name: Build and push Docker image
           command: |
-            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
             DOCKER_TAG="release-$(git log --oneline|wc -l)"
             docker build \
               --pull \
               --build-arg GIT_VERSION=${CIRCLE_SHA1} \
               --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
               --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
-              --tag neondatabase/neon:${DOCKER_TAG} --tag neondatabase/neon:release .
-            docker push neondatabase/neon:${DOCKER_TAG}
-            docker push neondatabase/neon:release
+              --tag zenithdb/zenith:${DOCKER_TAG} --tag zenithdb/zenith:release .
+            docker push zenithdb/zenith:${DOCKER_TAG}
+            docker push zenithdb/zenith:release
 
-  # Build production neondatabase/compute-node:release image and push it to Docker hub
+  # Build production zenithdb/compute-node:release image and push it to Docker hub
   docker-image-compute-release:
     docker:
       - image: cimg/base:2021.04
@@ -513,29 +511,26 @@ jobs:
       - checkout
       - setup_remote_docker:
           docker_layer_caching: true
-      # Build neondatabase/compute-tools:release image and push it to Docker hub
+      # Build zenithdb/compute-tools:release image and push it to Docker hub
       # TODO: this should probably also use versioned tag, not just :latest.
       # XXX: but should it? We build and use it only locally now.
       - run:
           name: Build and push compute-tools Docker image
           command: |
-            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
-            docker build \
-              --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
-              --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
-              --tag neondatabase/compute-tools:release -f Dockerfile.compute-tools .
-            docker push neondatabase/compute-tools:release
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
+            docker build -t zenithdb/compute-tools:release -f Dockerfile.compute-tools .
+            docker push zenithdb/compute-tools:release
       - run:
           name: Init postgres submodule
           command: git submodule update --init --depth 1
       - run:
           name: Build and push compute-node Docker image
           command: |
-            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
             DOCKER_TAG="release-$(git log --oneline|wc -l)"
-            docker build --tag neondatabase/compute-node:${DOCKER_TAG} --tag neondatabase/compute-node:release vendor/postgres
-            docker push neondatabase/compute-node:${DOCKER_TAG}
-            docker push neondatabase/compute-node:release
+            docker build --tag zenithdb/compute-node:${DOCKER_TAG} --tag zenithdb/compute-node:release vendor/postgres
+            docker push zenithdb/compute-node:${DOCKER_TAG}
+            docker push zenithdb/compute-node:release
 
   deploy-staging:
     docker:
@@ -561,7 +556,7 @@ jobs:
             rm -f ssh-key ssh-key-cert.pub
 
             ansible-playbook deploy.yaml -i staging.hosts
-            rm -f neon_install.tar.gz .neon_current_version
+            rm -f zenith_install.tar.gz .zenith_current_version
 
   deploy-staging-proxy:
     docker:
@@ -579,13 +574,13 @@ jobs:
           name: Setup helm v3
           command: |
             curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
-            helm repo add neondatabase https://neondatabase.github.io/helm-charts
+            helm repo add zenithdb https://zenithdb.github.io/helm-charts
       - run:
           name: Re-deploy proxy
           command: |
             DOCKER_TAG=$(git log --oneline|wc -l)
-            helm upgrade neon-proxy       neondatabase/neon-proxy --install -f .circleci/helm-values/staging.proxy.yaml --set image.tag=${DOCKER_TAG} --wait
-            helm upgrade neon-proxy-scram neondatabase/neon-proxy --install -f .circleci/helm-values/staging.proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait
+            helm upgrade zenith-proxy zenithdb/zenith-proxy --install -f .circleci/helm-values/staging.proxy.yaml --set image.tag=${DOCKER_TAG} --wait
+
 
   deploy-release:
     docker:
@@ -610,8 +605,8 @@ jobs:
             ssh-add ssh-key
             rm -f ssh-key ssh-key-cert.pub
 
-            ansible-playbook deploy.yaml -i production.hosts
-            rm -f neon_install.tar.gz .neon_current_version
+            ansible-playbook deploy.yaml -i production.hosts -e console_mgmt_base_url=http://console-release.local
+            rm -f zenith_install.tar.gz .zenith_current_version
 
   deploy-release-proxy:
     docker:
@@ -629,7 +624,7 @@ jobs:
           name: Setup helm v3
           command: |
             curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
-            helm repo add zenithdb https://neondatabase.github.io/helm-charts
+            helm repo add zenithdb https://zenithdb.github.io/helm-charts
       - run:
           name: Re-deploy proxy
           command: |
@@ -658,7 +653,7 @@ jobs:
             --data \
               "{
                 \"state\": \"pending\",
-                \"context\": \"neon-cloud-e2e\",
+                \"context\": \"zenith-remote-ci\",
                 \"description\": \"[$REMOTE_REPO] Remote CI job is about to start\"
               }"
       - run:
@@ -674,7 +669,7 @@ jobs:
               "{
                 \"ref\": \"main\",
                 \"inputs\": {
-                  \"ci_job_name\": \"neon-cloud-e2e\",
+                  \"ci_job_name\": \"zenith-remote-ci\",
                   \"commit_hash\": \"$CIRCLE_SHA1\",
                   \"remote_repo\": \"$LOCAL_REPO\"
                 }
@@ -690,8 +685,8 @@ workflows:
           matrix:
             parameters:
               build_type: ["debug", "release"]
-      - build-neon:
-          name: build-neon-<< matrix.build_type >>
+      - build-zenith:
+          name: build-zenith-<< matrix.build_type >>
           matrix:
             parameters:
               build_type: ["debug", "release"]
@@ -706,7 +701,7 @@ workflows:
           test_selection: batch_pg_regress
           needs_postgres_source: true
           requires:
-            - build-neon-<< matrix.build_type >>
+            - build-zenith-<< matrix.build_type >>
       - run-pytest:
           name: other-tests-<< matrix.build_type >>
           matrix:
@@ -714,7 +709,7 @@ workflows:
               build_type: ["debug", "release"]
           test_selection: batch_others
           requires:
-            - build-neon-<< matrix.build_type >>
+            - build-zenith-<< matrix.build_type >>
       - run-pytest:
           name: benchmarks
           context: PERF_TEST_RESULT_CONNSTR
@@ -723,7 +718,7 @@ workflows:
           run_in_parallel: false
           save_perf_report: true
           requires:
-            - build-neon-release
+            - build-zenith-release
       - coverage-report:
           # Context passes credentials for gh api
           context: CI_ACCESS_TOKEN
@@ -814,11 +809,11 @@ workflows:
       - remote-ci-trigger:
           # Context passes credentials for gh api
           context: CI_ACCESS_TOKEN
-          remote_repo: "neondatabase/cloud"
+          remote_repo: "zenithdb/console"
           requires:
             # XXX: Successful build doesn't mean everything is OK, but
             # the job to be triggered takes so much time to complete (~22 min)
             # that it's better not to wait for the commented-out steps
-            - build-neon-release
+            - build-zenith-debug
             # - pg_regress-tests-release
             # - other-tests-release

.circleci/helm-values/production.proxy.yaml

@@ -1,12 +1,9 @@
 # Helm chart values for zenith-proxy.
 # This is a YAML-formatted file.
 
-image:
-  repository: neondatabase/neon
-
 settings:
-  authEndpoint: "https://console.neon.tech/authenticate_proxy_request/"
-  uri: "https://console.neon.tech/psql_session/"
+  authEndpoint: "https://console.zenith.tech/authenticate_proxy_request/"
+  uri: "https://console.zenith.tech/psql_session/"
 
 # -- Additional labels for zenith-proxy pods
 podLabels:
@@ -28,7 +25,7 @@ exposedService:
     service.beta.kubernetes.io/aws-load-balancer-type: external
     service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
     service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: start.zenith.tech,connect.neon.tech,pg.neon.tech
+    external-dns.alpha.kubernetes.io/hostname: start.zenith.tech
 
 metrics:
   enabled: true

.circleci/helm-values/staging.proxy-scram.yaml

@@ -1,31 +0,0 @@
-# Helm chart values for zenith-proxy.
-# This is a YAML-formatted file.
-
-image:
-  repository: neondatabase/neon
-
-settings:
-  authBackend: "console"
-  authEndpoint: "http://console-staging.local/management/api/v2"
-  domain: "*.cloud.stage.neon.tech"
-
-# -- Additional labels for zenith-proxy pods
-podLabels:
-  zenith_service: proxy-scram
-  zenith_env: staging
-  zenith_region: us-east-1
-  zenith_region_slug: virginia
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: cloud.stage.neon.tech
-
-metrics:
-  enabled: true
-  serviceMonitor:
-    enabled: true
-    selector:
-      release: kube-prometheus-stack

.circleci/helm-values/staging.proxy.yaml

@@ -1,12 +1,9 @@
 # Helm chart values for zenith-proxy.
 # This is a YAML-formatted file.
 
-image:
-  repository: neondatabase/neon
-
 settings:
-  authEndpoint: "https://console.stage.neon.tech/authenticate_proxy_request/"
-  uri: "https://console.stage.neon.tech/psql_session/"
+  authEndpoint: "https://console.stage.zenith.tech/authenticate_proxy_request/"
+  uri: "https://console.stage.zenith.tech/psql_session/"
 
 # -- Additional labels for zenith-proxy pods
 podLabels:
@@ -20,7 +17,7 @@ exposedService:
     service.beta.kubernetes.io/aws-load-balancer-type: external
     service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
     service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: connect.stage.neon.tech
+    external-dns.alpha.kubernetes.io/hostname: start.stage.zenith.tech
 
 metrics:
   enabled: true

.config/hakari.toml

@@ -1,26 +0,0 @@
-# This file contains settings for `cargo hakari`.
-# See https://docs.rs/cargo-hakari/latest/cargo_hakari/config for a full list of options.
-
-hakari-package = "workspace_hack"
-
-# Format for `workspace-hack = ...` lines in other Cargo.tomls. Requires cargo-hakari 0.9.8 or above.
-dep-format-version = "2"
-
-# Setting workspace.resolver = "2" in the root Cargo.toml is HIGHLY recommended.
-# Hakari works much better with the new feature resolver.
-# For more about the new feature resolver, see:
-# https://blog.rust-lang.org/2021/03/25/Rust-1.51.0.html#cargos-new-feature-resolver
-# Have to keep the resolver still here since hakari requires this field,
-# despite it's now the default for 2021 edition & cargo.
-resolver = "2"
-
-# Add triples corresponding to platforms commonly used by developers here.
-# https://doc.rust-lang.org/rustc/platform-support.html
-platforms = [
-    # "x86_64-unknown-linux-gnu",
-    # "x86_64-apple-darwin",
-    # "x86_64-pc-windows-msvc",
-]
-
-# Write out exact versions rather than a semver range. (Defaults to false.)
-# exact-versions = true

.github/workflows/benchmarking.yml

@@ -26,7 +26,7 @@ jobs:
     runs-on: [self-hosted, zenith-benchmarker]
 
     env:
-      POSTGRES_DISTRIB_DIR: "/usr/pgsql-13"
+      PG_BIN: "/usr/pgsql-13/bin"
 
     steps:
     - name: Checkout zenith repo
@@ -51,7 +51,7 @@ jobs:
         echo Poetry
         poetry --version
         echo Pgbench
-        $POSTGRES_DISTRIB_DIR/bin/pgbench --version
+        $PG_BIN/pgbench --version
 
     # FIXME cluster setup is skipped due to various changes in console API
     # for now pre created cluster is used. When API gain some stability
@@ -66,7 +66,7 @@ jobs:
 
         echo "Starting cluster"
         # wake up the cluster
-        $POSTGRES_DISTRIB_DIR/bin/psql $BENCHMARK_CONNSTR -c "SELECT 1"
+        $PG_BIN/psql $BENCHMARK_CONNSTR -c "SELECT 1"
 
     - name: Run benchmark
       # pgbench is installed system wide from official repo
@@ -83,11 +83,8 @@ jobs:
       # sudo yum install postgresql13-contrib
       # actual binaries are located in /usr/pgsql-13/bin/
       env:
-        # The pgbench test runs two tests of given duration against each scale.
-        # So the total runtime with these parameters is 2 * 2 * 300 = 1200, or 20 minutes.
-        # Plus time needed to initialize the test databases.
-        TEST_PG_BENCH_DURATIONS_MATRIX: "300"
-        TEST_PG_BENCH_SCALES_MATRIX: "10,100"
+        TEST_PG_BENCH_TRANSACTIONS_MATRIX: "5000,10000,20000"
+        TEST_PG_BENCH_SCALES_MATRIX: "10,15"
         PLATFORM: "zenith-staging"
         BENCHMARK_CONNSTR: "${{ secrets.BENCHMARK_STAGING_CONNSTR }}"
         REMOTE_ENV: "1" # indicate to test harness that we do not have zenith binaries locally

.github/workflows/testing.yml

@@ -1,6 +1,10 @@
 name: Build and Test
 
-on: push
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
 
 jobs:
   regression-check:
@@ -9,7 +13,7 @@ jobs:
         # If we want to duplicate this job for different
         # Rust toolchains (e.g. nightly or 1.37.0), add them here.
         rust_toolchain: [stable]
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest]
     timeout-minutes: 30
     name: run regression test suite
     runs-on: ${{ matrix.os }}
@@ -28,16 +32,11 @@ jobs:
           toolchain: ${{ matrix.rust_toolchain }}
           override: true
 
-      - name: Install Ubuntu postgres dependencies
-        if: matrix.os == 'ubuntu-latest'
+      - name: Install postgres dependencies
         run: |
           sudo apt update
           sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev
 
-      - name: Install macOs postgres dependencies
-        if: matrix.os == 'macos-latest'
-        run: brew install flex bison
-
       - name: Set pg revision for caching
         id: pg_ver
         run: echo ::set-output name=pg_rev::$(git rev-parse HEAD:vendor/postgres)
@@ -52,7 +51,8 @@ jobs:
 
       - name: Build postgres
         if: steps.cache_pg.outputs.cache-hit != 'true'
-        run: make postgres
+        run: |
+          make postgres
 
       - name: Cache cargo deps
         id: cache_cargo
@@ -62,10 +62,13 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('./Cargo.lock') }}
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
 
-      - name: Run cargo clippy
-        run: ./run_clippy.sh
+      # Use `env CARGO_INCREMENTAL=0` to mitigate https://github.com/rust-lang/rust/issues/91696 for rustc 1.57.0
+      - name: Run cargo build
+        run: |
+          env CARGO_INCREMENTAL=0 cargo build --workspace --bins --examples --tests
 
       - name: Run cargo test
-        run: cargo test --all --all-targets
+        run: |
+          env CARGO_INCREMENTAL=0 cargo test -- --nocapture --test-threads=1

.gitignore

@@ -11,6 +11,3 @@ test_output/
 # Coverage
 *.profraw
 *.profdata
-
-*.key
-*.crt

Cargo.toml

@@ -3,11 +3,13 @@ members = [
     "compute_tools",
     "control_plane",
     "pageserver",
+    "postgres_ffi",
     "proxy",
-    "safekeeper",
+    "walkeeper",
     "workspace_hack",
-    "neon_local",
-    "libs/*",
+    "zenith",
+    "zenith_metrics",
+    "zenith_utils",
 ]
 
 [profile.release]
@@ -15,7 +17,7 @@ members = [
 # Besides, debug info should not affect the performance.
 debug = true
 
-# This is only needed for proxy's tests.
-# TODO: we should probably fork `tokio-postgres-rustls` instead.
+# This is only needed for proxy's tests
+# TODO: we should probably fork tokio-postgres-rustls instead
 [patch.crates-io]
-tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }

Dockerfile

@@ -1,5 +1,7 @@
 # Build Postgres
-FROM zimg/rust:1.58 AS pg-build
+#
+#FROM zimg/rust:1.56 AS pg-build
+FROM zenithdb/build:buster-20220309 AS pg-build
 WORKDIR /pg
 
 USER root
@@ -9,26 +11,26 @@ COPY Makefile Makefile
 
 ENV BUILD_TYPE release
 RUN set -e \
-    && mold -run make -j $(nproc) -s postgres \
+    && make -j $(nproc) -s postgres \
     && rm -rf tmp_install/build \
     && tar -C tmp_install -czf /postgres_install.tar.gz .
 
 # Build zenith binaries
-FROM zimg/rust:1.58 AS build
+#
+#FROM zimg/rust:1.56 AS build
+FROM zenithdb/build:buster-20220309 AS build
 ARG GIT_VERSION=local
 
 ARG CACHEPOT_BUCKET=zenith-rust-cachepot
 ARG AWS_ACCESS_KEY_ID
 ARG AWS_SECRET_ACCESS_KEY
+#ENV RUSTC_WRAPPER cachepot
+ENV RUSTC_WRAPPER /usr/local/cargo/bin/cachepot
 
 COPY --from=pg-build /pg/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
 COPY . .
 
-# Show build caching stats to check if it was used in the end.
-# Has to be the part of the same RUN since cachepot daemon is killed in the end of this RUN, loosing the compilation stats.
-RUN set -e \
-    && sudo -E "PATH=$PATH" mold -run cargo build --release \
-    && cachepot -s
+RUN cargo build --release
 
 # Build final image
 #

Dockerfile.build

@@ -0,0 +1,23 @@
+FROM rust:1.56.1-slim-buster
+WORKDIR /home/circleci/project
+
+RUN set -e \
+    && apt-get update \
+    && apt-get -yq install \
+        automake \
+        libtool \
+        build-essential \
+        bison \
+        flex \
+        libreadline-dev \
+        zlib1g-dev \
+        libxml2-dev \
+        libseccomp-dev \
+        pkg-config \
+        libssl-dev \
+        clang
+
+RUN set -e \
+    && rustup component add clippy \
+    && cargo install cargo-audit \
+    && cargo install --git https://github.com/paritytech/cachepot

Dockerfile.compute-tools

@@ -1,18 +1,14 @@
 # First transient image to build compute_tools binaries
 # NB: keep in sync with rust image version in .circle/config.yml
-FROM zimg/rust:1.58 AS rust-build
+FROM rust:1.56.1-slim-buster AS rust-build
 
-ARG CACHEPOT_BUCKET=zenith-rust-cachepot
-ARG AWS_ACCESS_KEY_ID
-ARG AWS_SECRET_ACCESS_KEY
+WORKDIR /zenith
 
 COPY . .
 
-RUN set -e \
-    && sudo -E "PATH=$PATH" mold -run cargo build -p compute_tools --release \
-    && cachepot -s
+RUN cargo build -p compute_tools --release
 
 # Final image that only has one binary
 FROM debian:buster-slim
 
-COPY --from=rust-build /home/circleci/project/target/release/zenith_ctl /usr/local/bin/zenith_ctl
+COPY --from=rust-build /zenith/target/release/zenith_ctl /usr/local/bin/zenith_ctl

Makefile

@@ -78,11 +78,6 @@ postgres: postgres-configure \
 	$(MAKE) -C tmp_install/build/contrib/zenith install
 	+@echo "Compiling contrib/zenith_test_utils"
 	$(MAKE) -C tmp_install/build/contrib/zenith_test_utils install
-	+@echo "Compiling pg_buffercache"
-	$(MAKE) -C tmp_install/build/contrib/pg_buffercache install
-	+@echo "Compiling pageinspect"
-	$(MAKE) -C tmp_install/build/contrib/pageinspect install
-
 
 .PHONY: postgres-clean
 postgres-clean:

README.md

@@ -1,22 +1,19 @@
-# Neon
+# Zenith
 
-Neon is a serverless open source alternative to AWS Aurora Postgres. It separates storage and compute and substitutes PostgreSQL storage layer by redistributing data across a cluster of nodes.
-
-The project used to be called "Zenith". Many of the commands and code comments
-still refer to "zenith", but we are in the process of renaming things.
+Zenith is a serverless open source alternative to AWS Aurora Postgres. It separates storage and compute and substitutes PostgreSQL storage layer by redistributing data across a cluster of nodes.
 
 ## Architecture overview
 
-A Neon installation consists of compute nodes and Neon storage engine.
+A Zenith installation consists of compute nodes and Zenith storage engine.
 
-Compute nodes are stateless PostgreSQL nodes, backed by Neon storage engine.
+Compute nodes are stateless PostgreSQL nodes, backed by Zenith storage engine.
 
-Neon storage engine consists of two major components:
+Zenith storage engine consists of two major components:
 - Pageserver. Scalable storage backend for compute nodes.
 - WAL service. The service that receives WAL from compute node and ensures that it is stored durably.
 
 Pageserver consists of:
-- Repository - Neon storage implementation.
+- Repository - Zenith storage implementation.
 - WAL receiver - service that receives WAL from WAL service and stores it in the repository.
 - Page service - service that communicates with compute nodes and responds with pages from the repository.
 - WAL redo - service that builds pages from base images and WAL records on Page service request.
@@ -31,17 +28,17 @@ apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libsec
 libssl-dev clang pkg-config libpq-dev
 ```
 
-[Rust] 1.58 or later is also required.
+[Rust] 1.56.1 or later is also required.
 
 To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively.
 
 To run the integration tests or Python scripts (not required to use the code), install
 Python (3.7 or higher), and install python3 packages using `./scripts/pysync` (requires poetry) in the project directory.
 
-2. Build neon and patched postgres
+2. Build zenith and patched postgres
 ```sh
-git clone --recursive https://github.com/neondatabase/neon.git
-cd neon
+git clone --recursive https://github.com/zenithdb/zenith.git
+cd zenith
 make -j5
 ```
 
@@ -49,30 +46,32 @@ make -j5
 ```sh
 # Create repository in .zenith with proper paths to binaries and data
 # Later that would be responsibility of a package install script
-> ./target/debug/neon_local init
-initializing tenantid 9ef87a5bf0d92544f6fafeeb3239695c
-created initial timeline de200bd42b49cc1814412c7e592dd6e9 timeline.lsn 0/16B5A50
-initial timeline de200bd42b49cc1814412c7e592dd6e9 created
+> ./target/debug/zenith init
+initializing tenantid c03ba6b7ad4c5e9cf556f059ade44229
+created initial timeline 5b014a9e41b4b63ce1a1febc04503636 timeline.lsn 0/169C3C8
+created main branch
 pageserver init succeeded
 
 # start pageserver and safekeeper
-> ./target/debug/neon_local start
-Starting pageserver at '127.0.0.1:64000' in '.zenith'
+> ./target/debug/zenith start
+Starting pageserver at 'localhost:64000' in '.zenith'
 Pageserver started
-initializing for sk 1 for 7676
-Starting safekeeper at '127.0.0.1:5454' in '.zenith/safekeepers/sk1'
+initializing for single for 7676
+Starting safekeeper at '127.0.0.1:5454' in '.zenith/safekeepers/single'
 Safekeeper started
 
 # start postgres compute node
-> ./target/debug/neon_local pg start main
-Starting new postgres main on timeline de200bd42b49cc1814412c7e592dd6e9 ...
-Extracting base backup to create postgres instance: path=.zenith/pgdatadirs/tenants/9ef87a5bf0d92544f6fafeeb3239695c/main port=55432
+> ./target/debug/zenith pg start main
+Starting new postgres main on timeline 5b014a9e41b4b63ce1a1febc04503636 ...
+Extracting base backup to create postgres instance: path=.zenith/pgdatadirs/tenants/c03ba6b7ad4c5e9cf556f059ade44229/main port=55432
 Starting postgres node at 'host=127.0.0.1 port=55432 user=zenith_admin dbname=postgres'
+waiting for server to start.... done
+server started
 
 # check list of running postgres instances
-> ./target/debug/neon_local pg list
- NODE  ADDRESS          TIMELINE                          BRANCH NAME  LSN        STATUS
- main  127.0.0.1:55432  de200bd42b49cc1814412c7e592dd6e9  main         0/16B5BA8  running
+> ./target/debug/zenith pg list
+NODE	ADDRESS	TIMELINES	BRANCH NAME	LSN		STATUS
+main	127.0.0.1:55432	5b014a9e41b4b63ce1a1febc04503636	main	0/1609610	running
 ```
 
 4. Now it is possible to connect to postgres and run some queries:
@@ -92,25 +91,18 @@ postgres=# select * from t;
 5. And create branches and run postgres on them:
 ```sh
 # create branch named migration_check
-> ./target/debug/neon_local timeline branch --branch-name migration_check
-Created timeline 'b3b863fa45fa9e57e615f9f2d944e601' at Lsn 0/16F9A00 for tenant: 9ef87a5bf0d92544f6fafeeb3239695c. Ancestor timeline: 'main'
+> ./target/debug/zenith timeline branch --branch-name migration_check
+Created timeline '0e9331cad6efbafe6a88dd73ae21a5c9' at Lsn 0/16F5830 for tenant: c03ba6b7ad4c5e9cf556f059ade44229. Ancestor timeline: 'main'
 
 # check branches tree
-> ./target/debug/neon_local timeline list
-(L) main [de200bd42b49cc1814412c7e592dd6e9]
-(L) ┗━ @0/16F9A00: migration_check [b3b863fa45fa9e57e615f9f2d944e601]
+> ./target/debug/zenith timeline list
+ main [5b014a9e41b4b63ce1a1febc04503636]
+ ┗━ @0/1609610: migration_check [0e9331cad6efbafe6a88dd73ae21a5c9]
 
 # start postgres on that branch
-> ./target/debug/neon_local pg start migration_check --branch-name migration_check
-Starting new postgres migration_check on timeline b3b863fa45fa9e57e615f9f2d944e601 ...
-Extracting base backup to create postgres instance: path=.zenith/pgdatadirs/tenants/9ef87a5bf0d92544f6fafeeb3239695c/migration_check port=55433
-Starting postgres node at 'host=127.0.0.1 port=55433 user=zenith_admin dbname=postgres'
-
-# check the new list of running postgres instances
-> ./target/debug/neon_local pg list
- NODE             ADDRESS          TIMELINE                          BRANCH NAME      LSN        STATUS
- main             127.0.0.1:55432  de200bd42b49cc1814412c7e592dd6e9  main             0/16F9A38  running
- migration_check  127.0.0.1:55433  b3b863fa45fa9e57e615f9f2d944e601  migration_check  0/16F9A70  running
+> ./target/debug/zenith pg start migration_check
+Starting postgres node at 'host=127.0.0.1 port=55433 user=stas'
+waiting for server to start.... done
 
 # this new postgres instance will have all the data from 'main' postgres,
 # but all modifications would not affect data in original postgres
@@ -123,26 +115,18 @@ postgres=# select * from t;
 
 postgres=# insert into t values(2,2);
 INSERT 0 1
-
-# check that the new change doesn't affect the 'main' postgres
-> psql -p55432 -h 127.0.0.1 -U zenith_admin postgres
-postgres=# select * from t;
- key | value
------+-------
-   1 | 1
-(1 row)
 ```
 
 6. If you want to run tests afterwards (see below), you have to stop all the running the pageserver, safekeeper and postgres instances
    you have just started. You can stop them all with one command:
 ```sh
-> ./target/debug/neon_local stop
+> ./target/debug/zenith stop
 ```
 
 ## Running tests
 
 ```sh
-git clone --recursive https://github.com/neondatabase/neon.git
+git clone --recursive https://github.com/zenithdb/zenith.git
 make # builds also postgres and installs it to ./tmp_install
 ./scripts/pytest
 ```
@@ -157,14 +141,14 @@ To view your `rustdoc` documentation in a browser, try running `cargo doc --no-d
 
 ### Postgres-specific terms
 
-Due to Neon's very close relation with PostgreSQL internals, there are numerous specific terms used.
+Due to Zenith's very close relation with PostgreSQL internals, there are numerous specific terms used.
 Same applies to certain spelling: i.e. we use MB to denote 1024 * 1024 bytes, while MiB would be technically more correct, it's inconsistent with what PostgreSQL code and its documentation use.
 
 To get more familiar with this aspect, refer to:
 
-- [Neon glossary](/docs/glossary.md)
+- [Zenith glossary](/docs/glossary.md)
 - [PostgreSQL glossary](https://www.postgresql.org/docs/13/glossary.html)
-- Other PostgreSQL documentation and sources (Neon fork sources can be found [here](https://github.com/neondatabase/postgres))
+- Other PostgreSQL documentation and sources (Zenith fork sources can be found [here](https://github.com/zenithdb/postgres))
 
 ## Join the development
 

compute_tools/Cargo.toml

@@ -11,11 +11,9 @@ clap = "3.0"
 env_logger = "0.9"
 hyper = { version = "0.14", features = ["full"] }
 log = { version = "0.4", features = ["std", "serde"] }
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 regex = "1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 tar = "0.4"
-tokio = { version = "1.17", features = ["macros", "rt", "rt-multi-thread"] }
-tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-workspace_hack = { version = "0.1", path = "../workspace_hack" }
+tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] }

compute_tools/src/bin/zenith_ctl.rs

@@ -38,7 +38,6 @@ use clap::Arg;
 use log::info;
 use postgres::{Client, NoTls};
 
-use compute_tools::checker::create_writablity_check_data;
 use compute_tools::config;
 use compute_tools::http_api::launch_http_server;
 use compute_tools::logger::*;
@@ -129,8 +128,6 @@ fn run_compute(state: &Arc<RwLock<ComputeState>>) -> Result<ExitStatus> {
 
     handle_roles(&read_state.spec, &mut client)?;
     handle_databases(&read_state.spec, &mut client)?;
-    handle_grants(&read_state.spec, &mut client)?;
-    create_writablity_check_data(&mut client)?;
 
     // 'Close' connection
     drop(client);
@@ -158,7 +155,7 @@ fn run_compute(state: &Arc<RwLock<ComputeState>>) -> Result<ExitStatus> {
 }
 
 fn main() -> Result<()> {
-    // TODO: re-use `utils::logging` later
+    // TODO: re-use `zenith_utils::logging` later
     init_logger(DEFAULT_LOG_LEVEL)?;
 
     // Env variable is set by `cargo`

compute_tools/src/checker.rs

@@ -1,46 +0,0 @@
-use std::sync::{Arc, RwLock};
-
-use anyhow::{anyhow, Result};
-use log::error;
-use postgres::Client;
-use tokio_postgres::NoTls;
-
-use crate::zenith::ComputeState;
-
-pub fn create_writablity_check_data(client: &mut Client) -> Result<()> {
-    let query = "
-    CREATE TABLE IF NOT EXISTS health_check (
-        id serial primary key,
-        updated_at timestamptz default now()
-    );
-    INSERT INTO health_check VALUES (1, now())
-        ON CONFLICT (id) DO UPDATE
-         SET updated_at = now();";
-    let result = client.simple_query(query)?;
-    if result.len() < 2 {
-        return Err(anyhow::format_err!("executed  {} queries", result.len()));
-    }
-    Ok(())
-}
-
-pub async fn check_writability(state: &Arc<RwLock<ComputeState>>) -> Result<()> {
-    let connstr = state.read().unwrap().connstr.clone();
-    let (client, connection) = tokio_postgres::connect(&connstr, NoTls).await?;
-    if client.is_closed() {
-        return Err(anyhow!("connection to postgres closed"));
-    }
-    tokio::spawn(async move {
-        if let Err(e) = connection.await {
-            error!("connection error: {}", e);
-        }
-    });
-
-    let result = client
-        .simple_query("UPDATE health_check SET updated_at = now() WHERE id = 1;")
-        .await?;
-
-    if result.len() != 1 {
-        return Err(anyhow!("statement can't be executed"));
-    }
-    Ok(())
-}

compute_tools/src/http_api.rs

@@ -11,7 +11,7 @@ use log::{error, info};
 use crate::zenith::*;
 
 // Service function to handle all available routes.
-async fn routes(req: Request<Body>, state: Arc<RwLock<ComputeState>>) -> Response<Body> {
+fn routes(req: Request<Body>, state: Arc<RwLock<ComputeState>>) -> Response<Body> {
     match (req.method(), req.uri().path()) {
         // Timestamp of the last Postgres activity in the plain text.
         (&Method::GET, "/last_activity") => {
@@ -29,15 +29,6 @@ async fn routes(req: Request<Body>, state: Arc<RwLock<ComputeState>>) -> Respons
             Response::new(Body::from(format!("{}", state.ready)))
         }
 
-        (&Method::GET, "/check_writability") => {
-            info!("serving /check_writability GET request");
-            let res = crate::checker::check_writability(&state).await;
-            match res {
-                Ok(_) => Response::new(Body::from("true")),
-                Err(e) => Response::new(Body::from(e.to_string())),
-            }
-        }
-
         // Return the `404 Not Found` for any other routes.
         _ => {
             let mut not_found = Response::new(Body::from("404 Not Found"));
@@ -57,7 +48,7 @@ async fn serve(state: Arc<RwLock<ComputeState>>) {
         async move {
             Ok::<_, Infallible>(service_fn(move |req: Request<Body>| {
                 let state = state.clone();
-                async move { Ok::<_, Infallible>(routes(req, state).await) }
+                async move { Ok::<_, Infallible>(routes(req, state)) }
             }))
         }
     });

compute_tools/src/lib.rs

@@ -2,7 +2,6 @@
 //! Various tools and helpers to handle cluster / compute node (Postgres)
 //! configuration.
 //!
-pub mod checker;
 pub mod config;
 pub mod http_api;
 #[macro_use]

compute_tools/src/pg_helpers.rs

@@ -132,14 +132,7 @@ impl Role {
         let mut params: String = "LOGIN".to_string();
 
         if let Some(pass) = &self.encrypted_password {
-            // Some time ago we supported only md5 and treated all encrypted_password as md5.
-            // Now we also support SCRAM-SHA-256 and to preserve compatibility
-            // we treat all encrypted_password as md5 unless they starts with SCRAM-SHA-256.
-            if pass.starts_with("SCRAM-SHA-256") {
-                params.push_str(&format!(" PASSWORD '{}'", pass));
-            } else {
-                params.push_str(&format!(" PASSWORD 'md5{}'", pass));
-            }
+            params.push_str(&format!(" PASSWORD 'md5{}'", pass));
         } else {
             params.push_str(" PASSWORD NULL");
         }

compute_tools/src/spec.rs

@@ -136,20 +136,13 @@ pub fn handle_roles(spec: &ClusterSpec, client: &mut Client) -> Result<()> {
                 xact.execute(query.as_str(), &[])?;
             }
         } else {
-            info!("role name: '{}'", &name);
+            info!("role name {}", &name);
             let mut query: String = format!("CREATE ROLE {} ", name.quote());
-            info!("role create query: '{}'", &query);
+            info!("role create query {}", &query);
             info_print!(" -> create");
 
             query.push_str(&role.to_pg_options());
             xact.execute(query.as_str(), &[])?;
-
-            let grant_query = format!(
-                "grant pg_read_all_data, pg_write_all_data to {}",
-                name.quote()
-            );
-            xact.execute(grant_query.as_str(), &[])?;
-            info!("role grant query: '{}'", &grant_query);
         }
 
         info_print!("\n");
@@ -251,24 +244,3 @@ pub fn handle_databases(spec: &ClusterSpec, client: &mut Client) -> Result<()> {
 
     Ok(())
 }
-
-// Grant CREATE ON DATABASE to the database owner
-// to allow clients create trusted extensions.
-pub fn handle_grants(spec: &ClusterSpec, client: &mut Client) -> Result<()> {
-    info!("cluster spec grants:");
-
-    for db in &spec.cluster.databases {
-        let dbname = &db.name;
-
-        let query: String = format!(
-            "GRANT CREATE ON DATABASE {} TO {}",
-            dbname.quote(),
-            db.owner.quote()
-        );
-        info!("grant query {}", &query);
-
-        client.execute(query.as_str(), &[])?;
-    }
-
-    Ok(())
-}

control_plane/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 
 [dependencies]
 tar = "0.4.33"
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
 serde = { version = "1.0", features = ["derive"] }
 serde_with = "1.12.0"
 toml = "0.5"
@@ -18,6 +18,6 @@ url = "2.2.2"
 reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] }
 
 pageserver = { path = "../pageserver" }
-safekeeper = { path = "../safekeeper" }
-utils = { path = "../libs/utils" }
-workspace_hack = { version = "0.1", path = "../workspace_hack" }
+walkeeper = { path = "../walkeeper" }
+zenith_utils = { path = "../zenith_utils" }
+workspace_hack = { path = "../workspace_hack" }

control_plane/src/compute.rs

@@ -11,12 +11,11 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use anyhow::{Context, Result};
-use utils::{
-    connstring::connection_host_port,
-    lsn::Lsn,
-    postgres_backend::AuthType,
-    zid::{ZTenantId, ZTimelineId},
-};
+use zenith_utils::connstring::connection_host_port;
+use zenith_utils::lsn::Lsn;
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::ZTenantId;
+use zenith_utils::zid::ZTimelineId;
 
 use crate::local_env::LocalEnv;
 use crate::postgresql_conf::PostgresConf;
@@ -273,7 +272,12 @@ impl PostgresNode {
         conf.append("wal_sender_timeout", "5s");
         conf.append("listen_addresses", &self.address.ip().to_string());
         conf.append("port", &self.address.port().to_string());
-        conf.append("wal_keep_size", "0");
+
+        // Never clean up old WAL. TODO: We should use a replication
+        // slot or something proper, to prevent the compute node
+        // from removing WAL that hasn't been streamed to the safekeeper or
+        // page server yet. (gh issue #349)
+        conf.append("wal_keep_size", "10TB");
 
         // Configure the node to fetch pages from pageserver
         let pageserver_connstr = {
@@ -327,14 +331,14 @@ impl PostgresNode {
             // Configure the node to connect to the safekeepers
             conf.append("synchronous_standby_names", "walproposer");
 
-            let safekeepers = self
+            let wal_acceptors = self
                 .env
                 .safekeepers
                 .iter()
                 .map(|sk| format!("localhost:{}", sk.pg_port))
                 .collect::<Vec<String>>()
                 .join(",");
-            conf.append("wal_acceptors", &safekeepers);
+            conf.append("wal_acceptors", &wal_acceptors);
         } else {
             // We only use setup without safekeepers for tests,
             // and don't care about data durability on pageserver,
@@ -416,15 +420,10 @@ impl PostgresNode {
         if let Some(token) = auth_token {
             cmd.env("ZENITH_AUTH_TOKEN", token);
         }
+        let pg_ctl = cmd.status().context("pg_ctl failed")?;
 
-        let pg_ctl = cmd.output().context("pg_ctl failed")?;
-        if !pg_ctl.status.success() {
-            anyhow::bail!(
-                "pg_ctl failed, exit code: {}, stdout: {}, stderr: {}",
-                pg_ctl.status,
-                String::from_utf8_lossy(&pg_ctl.stdout),
-                String::from_utf8_lossy(&pg_ctl.stderr),
-            );
+        if !pg_ctl.success() {
+            anyhow::bail!("pg_ctl failed");
         }
         Ok(())
     }

control_plane/src/local_env.rs

@@ -11,11 +11,9 @@ use std::env;
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
-use utils::{
-    auth::{encode_from_key_file, Claims, Scope},
-    postgres_backend::AuthType,
-    zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId},
-};
+use zenith_utils::auth::{encode_from_key_file, Claims, Scope};
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId};
 
 use crate::safekeeper::SafekeeperNode;
 
@@ -59,14 +57,6 @@ pub struct LocalEnv {
     #[serde(default)]
     pub private_key_path: PathBuf,
 
-    // A comma separated broker (etcd) endpoints for storage nodes coordination, e.g. 'http://127.0.0.1:2379'.
-    #[serde(default)]
-    pub broker_endpoints: Option<String>,
-
-    /// A prefix to all to any key when pushing/polling etcd from a node.
-    #[serde(default)]
-    pub broker_etcd_prefix: Option<String>,
-
     pub pageserver: PageServerConf,
 
     #[serde(default)]

control_plane/src/safekeeper.rs

@@ -13,17 +13,15 @@ use nix::unistd::Pid;
 use postgres::Config;
 use reqwest::blocking::{Client, RequestBuilder, Response};
 use reqwest::{IntoUrl, Method};
-use safekeeper::http::models::TimelineCreateRequest;
 use thiserror::Error;
-use utils::{
-    connstring::connection_address,
-    http::error::HttpErrorBody,
-    zid::{ZNodeId, ZTenantId, ZTimelineId},
-};
+use walkeeper::http::models::TimelineCreateRequest;
+use zenith_utils::http::error::HttpErrorBody;
+use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId};
 
 use crate::local_env::{LocalEnv, SafekeeperConf};
 use crate::storage::PageServerNode;
 use crate::{fill_rust_env_vars, read_pidfile};
+use zenith_utils::connstring::connection_address;
 
 #[derive(Error, Debug)]
 pub enum SafekeeperHttpError {
@@ -75,9 +73,6 @@ pub struct SafekeeperNode {
     pub http_base_url: String,
 
     pub pageserver: Arc<PageServerNode>,
-
-    broker_endpoints: Option<String>,
-    broker_etcd_prefix: Option<String>,
 }
 
 impl SafekeeperNode {
@@ -94,8 +89,6 @@ impl SafekeeperNode {
             http_client: Client::new(),
             http_base_url: format!("http://127.0.0.1:{}/v1", conf.http_port),
             pageserver,
-            broker_endpoints: env.broker_endpoints.clone(),
-            broker_etcd_prefix: env.broker_etcd_prefix.clone(),
         }
     }
 
@@ -142,12 +135,6 @@ impl SafekeeperNode {
         if !self.conf.sync {
             cmd.arg("--no-sync");
         }
-        if let Some(ref ep) = self.broker_endpoints {
-            cmd.args(&["--broker-endpoints", ep]);
-        }
-        if let Some(prefix) = self.broker_etcd_prefix.as_deref() {
-            cmd.args(&["--broker-etcd-prefix", prefix]);
-        }
 
         if !cmd.status()?.success() {
             bail!(

control_plane/src/storage.rs

@@ -1,4 +1,3 @@
-use std::collections::HashMap;
 use std::io::Write;
 use std::net::TcpStream;
 use std::path::PathBuf;
@@ -10,23 +9,21 @@ use anyhow::{bail, Context};
 use nix::errno::Errno;
 use nix::sys::signal::{kill, Signal};
 use nix::unistd::Pid;
-use pageserver::http::models::{TenantConfigRequest, TenantCreateRequest, TimelineCreateRequest};
+use pageserver::http::models::{TenantCreateRequest, TimelineCreateRequest};
 use pageserver::timelines::TimelineInfo;
 use postgres::{Config, NoTls};
 use reqwest::blocking::{Client, RequestBuilder, Response};
 use reqwest::{IntoUrl, Method};
 use thiserror::Error;
-use utils::{
-    connstring::connection_address,
-    http::error::HttpErrorBody,
-    lsn::Lsn,
-    postgres_backend::AuthType,
-    zid::{ZTenantId, ZTimelineId},
-};
+use zenith_utils::http::error::HttpErrorBody;
+use zenith_utils::lsn::Lsn;
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
 use crate::local_env::LocalEnv;
 use crate::{fill_rust_env_vars, read_pidfile};
 use pageserver::tenant_mgr::TenantInfo;
+use zenith_utils::connstring::connection_address;
 
 #[derive(Error, Debug)]
 pub enum PageserverHttpError {
@@ -151,25 +148,14 @@ impl PageServerNode {
         let initial_timeline_id_string = initial_timeline_id.to_string();
         args.extend(["--initial-timeline-id", &initial_timeline_id_string]);
 
-        let cmd_with_args = cmd.args(args);
-        let init_output = fill_rust_env_vars(cmd_with_args)
+        let init_output = fill_rust_env_vars(cmd.args(args))
             .output()
-            .with_context(|| {
-                format!("failed to init pageserver with command {:?}", cmd_with_args)
-            })?;
+            .context("pageserver init failed")?;
 
         if !init_output.status.success() {
-            bail!(
-                "init invocation failed, {}\nStdout: {}\nStderr: {}",
-                init_output.status,
-                String::from_utf8_lossy(&init_output.stdout),
-                String::from_utf8_lossy(&init_output.stderr)
-            );
+            bail!("pageserver init failed");
         }
 
-        // echo the captured output of the init command
-        println!("{}", String::from_utf8_lossy(&init_output.stdout));
-
         Ok(initial_timeline_id)
     }
 
@@ -189,6 +175,8 @@ impl PageServerNode {
         );
         io::stdout().flush().unwrap();
 
+        let mut cmd = Command::new(self.env.pageserver_bin()?);
+
         let repo_path = self.repo_path();
         let mut args = vec!["-D", repo_path.to_str().unwrap()];
 
@@ -196,11 +184,9 @@ impl PageServerNode {
             args.extend(["-c", config_override]);
         }
 
-        let mut cmd = Command::new(self.env.pageserver_bin()?);
-        let mut filled_cmd = fill_rust_env_vars(cmd.args(&args).arg("--daemonize"));
-        filled_cmd = fill_aws_secrets_vars(filled_cmd);
+        fill_rust_env_vars(cmd.args(&args).arg("--daemonize"));
 
-        if !filled_cmd.status()?.success() {
+        if !cmd.status()?.success() {
             bail!(
                 "Pageserver failed to start. See '{}' for details.",
                 self.repo_path().join("pageserver.log").display()
@@ -348,36 +334,10 @@ impl PageServerNode {
     pub fn tenant_create(
         &self,
         new_tenant_id: Option<ZTenantId>,
-        settings: HashMap<&str, &str>,
     ) -> anyhow::Result<Option<ZTenantId>> {
         let tenant_id_string = self
             .http_request(Method::POST, format!("{}/tenant", self.http_base_url))
-            .json(&TenantCreateRequest {
-                new_tenant_id,
-                checkpoint_distance: settings
-                    .get("checkpoint_distance")
-                    .map(|x| x.parse::<u64>())
-                    .transpose()?,
-                compaction_target_size: settings
-                    .get("compaction_target_size")
-                    .map(|x| x.parse::<u64>())
-                    .transpose()?,
-                compaction_period: settings.get("compaction_period").map(|x| x.to_string()),
-                compaction_threshold: settings
-                    .get("compaction_threshold")
-                    .map(|x| x.parse::<usize>())
-                    .transpose()?,
-                gc_horizon: settings
-                    .get("gc_horizon")
-                    .map(|x| x.parse::<u64>())
-                    .transpose()?,
-                gc_period: settings.get("gc_period").map(|x| x.to_string()),
-                image_creation_threshold: settings
-                    .get("image_creation_threshold")
-                    .map(|x| x.parse::<usize>())
-                    .transpose()?,
-                pitr_interval: settings.get("pitr_interval").map(|x| x.to_string()),
-            })
+            .json(&TenantCreateRequest { new_tenant_id })
             .send()?
             .error_from_body()?
             .json::<Option<String>>()?;
@@ -394,35 +354,6 @@ impl PageServerNode {
             .transpose()
     }
 
-    pub fn tenant_config(&self, tenant_id: ZTenantId, settings: HashMap<&str, &str>) -> Result<()> {
-        self.http_request(Method::PUT, format!("{}/tenant/config", self.http_base_url))
-            .json(&TenantConfigRequest {
-                tenant_id,
-                checkpoint_distance: settings
-                    .get("checkpoint_distance")
-                    .map(|x| x.parse::<u64>().unwrap()),
-                compaction_target_size: settings
-                    .get("compaction_target_size")
-                    .map(|x| x.parse::<u64>().unwrap()),
-                compaction_period: settings.get("compaction_period").map(|x| x.to_string()),
-                compaction_threshold: settings
-                    .get("compaction_threshold")
-                    .map(|x| x.parse::<usize>().unwrap()),
-                gc_horizon: settings
-                    .get("gc_horizon")
-                    .map(|x| x.parse::<u64>().unwrap()),
-                gc_period: settings.get("gc_period").map(|x| x.to_string()),
-                image_creation_threshold: settings
-                    .get("image_creation_threshold")
-                    .map(|x| x.parse::<usize>().unwrap()),
-                pitr_interval: settings.get("pitr_interval").map(|x| x.to_string()),
-            })
-            .send()?
-            .error_from_body()?;
-
-        Ok(())
-    }
-
     pub fn timeline_list(&self, tenant_id: &ZTenantId) -> anyhow::Result<Vec<TimelineInfo>> {
         let timeline_infos: Vec<TimelineInfo> = self
             .http_request(
@@ -460,12 +391,3 @@ impl PageServerNode {
         Ok(timeline_info_response)
     }
 }
-
-fn fill_aws_secrets_vars(mut cmd: &mut Command) -> &mut Command {
-    for env_key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] {
-        if let Ok(value) = std::env::var(env_key) {
-            cmd = cmd.env(env_key, value);
-        }
-    }
-    cmd
-}

docs/README.md

@@ -7,8 +7,8 @@
 - [glossary.md](glossary.md) — Glossary of all the terms used in codebase.
 - [multitenancy.md](multitenancy.md) — how multitenancy is organized in the pageserver and Zenith CLI.
 - [sourcetree.md](sourcetree.md) — Overview of the source tree layeout.
-- [pageserver/README.md](/pageserver/README.md) — pageserver overview.
-- [postgres_ffi/README.md](/libs/postgres_ffi/README.md) — Postgres FFI overview.
+- [pageserver/README](/pageserver/README) — pageserver overview.
+- [postgres_ffi/README](/postgres_ffi/README) — Postgres FFI overview.
 - [test_runner/README.md](/test_runner/README.md) — tests infrastructure overview.
-- [safekeeper/README.md](/safekeeper/README.md) — WAL service overview.
+- [walkeeper/README](/walkeeper/README) — WAL service overview.
 - [core_changes.md](core_changes.md) - Description of Zenith changes in Postgres core

docs/authentication.md

@@ -27,4 +27,4 @@ management_token = jwt.encode({"scope": "pageserverapi"}, auth_keys.priv, algori
 tenant_token = jwt.encode({"scope": "tenant", "tenant_id": ps.initial_tenant}, auth_keys.priv, algorithm="RS256")
 ```
 
-Utility functions to work with jwts in rust are located in libs/utils/src/auth.rs
+Utility functions to work with jwts in rust are located in zenith_utils/src/auth.rs

docs/glossary.md

@@ -21,7 +21,7 @@ NOTE:It has nothing to do with PostgreSQL pg_basebackup.
 
 ### Branch
 
-We can create branch at certain LSN using `zenith timeline branch` command.
+We can create branch at certain LSN using `zenith branch` command.
 Each Branch lives in a corresponding timeline[] and has an ancestor[].
 
 
@@ -29,32 +29,24 @@ Each Branch lives in a corresponding timeline[] and has an ancestor[].
 
 NOTE: This is an overloaded term.
 
-A checkpoint record in the WAL marks a point in the WAL sequence at which it is guaranteed that all data files have been updated with all information from shared memory modified before that checkpoint;
+A checkpoint record in the WAL marks a point in the WAL sequence at which it is guaranteed that all data files have been updated with all information from shared memory modified before that checkpoint; 
 
 ### Checkpoint (Layered repository)
 
 NOTE: This is an overloaded term.
 
 Whenever enough WAL has been accumulated in memory, the page server []
-writes out the changes from the in-memory layer into a new delta layer file. This process
-is called "checkpointing".
+writes out the changes from in-memory layers into new layer files[]. This process
+is called "checkpointing". The page server only creates layer files for
+relations that have been modified since the last checkpoint. 
 
 Configuration parameter `checkpoint_distance` defines the distance
 from current LSN to perform checkpoint of in-memory layers.
 Default is `DEFAULT_CHECKPOINT_DISTANCE`.
+Set this parameter to `0` to force checkpoint of every layer.
 
-### Compaction
-
-A background operation on layer files. Compaction takes a number of L0
-layer files, each of which covers the whole key space and a range of
-LSN, and reshuffles the data in them into L1 files so that each file
-covers the whole LSN range, but only part of the key space.
-
-Compaction should also opportunistically leave obsolete page versions
-from the L1 files, and materialize other page versions for faster
-access. That hasn't been implemented as of this writing, though.
-
-
+Configuration parameter `checkpoint_period` defines the interval between checkpoint iterations.
+Default is `DEFAULT_CHECKPOINT_PERIOD`.
 ### Compute node
 
 Stateless Postgres node that stores data in pageserver.
@@ -62,10 +54,10 @@ Stateless Postgres node that stores data in pageserver.
 ### Garbage collection
 
 The process of removing old on-disk layers that are not needed by any timeline anymore.
-
 ### Fork
 
 Each of the separate segmented file sets in which a relation is stored. The main fork is where the actual data resides. There also exist two secondary forks for metadata: the free space map and the visibility map.
+Each PostgreSQL fork is considered a separate relish.
 
 ### Layer
 
@@ -80,15 +72,15 @@ are immutable. See pageserver/src/layered_repository/README.md for more.
 ### Layer file (on-disk layer)
 
 Layered repository on-disk format is based on immutable files.  The
-files are called "layer files". There are two kinds of layer files:
-image files and delta files. An image file contains a "snapshot" of a
-range of keys at a particular LSN, and a delta file contains WAL
-records applicable to a range of keys, in a range of LSNs.
+files are called "layer files". Each file corresponds to one RELISH_SEG_SIZE
+segment of a PostgreSQL relation fork. There are two kinds of layer
+files: image files and delta files. An image file contains a
+"snapshot" of the segment at a particular LSN, and a delta file
+contains WAL records applicable to the segment, in a range of LSNs.
 
 ### Layer map
 
-The layer map tracks what layers exist in a timeline.
-
+The layer map tracks what layers exist for all the relishes in a timeline.
 ### Layered repository
 
 Zenith repository implementation that keeps data in layers.
@@ -108,10 +100,10 @@ PostgreSQL LSNs and functions to monitor them:
 * `pg_current_wal_lsn()` - Returns the current write-ahead log write location.
 * `pg_current_wal_flush_lsn()` - Returns the current write-ahead log flush location.
 * `pg_last_wal_receive_lsn()` - Returns the last write-ahead log location that has been received and synced to disk by streaming replication. While streaming replication is in progress this will increase monotonically.
-* `pg_last_wal_replay_lsn ()` - Returns the last write-ahead log location that has been replayed during recovery. If recovery is still in progress this will increase monotonically.
+* `pg_last_wal_replay_lsn ()` - Returns the last write-ahead log location that has been replayed during recovery. If recovery is still in progress this will increase monotonically. 
 [source PostgreSQL documentation](https://www.postgresql.org/docs/devel/functions-admin.html):
 
-Zenith safekeeper LSNs. For more check [safekeeper/README_PROTO.md](/safekeeper/README_PROTO.md)
+Zenith safekeeper LSNs. For more check [walkeeper/README_PROTO.md](/walkeeper/README_PROTO.md)
 * `CommitLSN`: position in WAL confirmed by quorum safekeepers.
 * `RestartLSN`: position in WAL confirmed by all safekeepers.
 * `FlushLSN`: part of WAL persisted to the disk by safekeeper.
@@ -157,6 +149,14 @@ and create new databases and accounts (control plane API in our case).
 
 The generic term in PostgreSQL for all objects in a database that have a name and a list of attributes defined in a specific order.
 
+### Relish
+
+We call each relation and other file that is stored in the
+repository a "relish". It comes from "rel"-ish, as in "kind of a
+rel", because it covers relations as well as other things that are
+not relations, but are treated similarly for the purposes of the
+storage layer.
+
 ### Replication slot
 
 
@@ -173,24 +173,33 @@ One repository corresponds to one Tenant.
 
 How much history do we need to keep around for PITR and read-only nodes?
 
-### Segment
+### Segment (PostgreSQL)
+
+NOTE: This is an overloaded term.
 
 A physical file that stores data for a given relation. File segments are
 limited in size by a compile-time setting (1 gigabyte by default), so if a
 relation exceeds that size, it is split into multiple segments.
 
+### Segment (Layered Repository)
+
+NOTE: This is an overloaded term.
+
+Segment is a RELISH_SEG_SIZE slice of relish (identified by a SegmentTag).
+
 ### SLRU
 
 SLRUs include pg_clog, pg_multixact/members, and
 pg_multixact/offsets. There are other SLRUs in PostgreSQL, but
 they don't need to be stored permanently (e.g. pg_subtrans),
 or we do not support them in zenith yet (pg_commit_ts).
+Each SLRU segment is considered a separate relish[].
 
 ### Tenant (Multitenancy)
 Tenant represents a single customer, interacting with Zenith.
 Wal redo[] activity, timelines[], layers[] are managed for each tenant independently.
 One pageserver[] can serve multiple tenants at once.
-One safekeeper
+One safekeeper 
 
 See `docs/multitenancy.md` for more.
 

docs/rfcs/004-durability.md

@@ -22,7 +22,7 @@ In addition to the WAL safekeeper nodes, the WAL is archived in
 S3. WAL that has been archived to S3 can be removed from the
 safekeepers, so the safekeepers don't need a lot of disk space.
 
-```
+
                                 +----------------+
                         +-----> | WAL safekeeper |
                         |       +----------------+
@@ -42,23 +42,23 @@ safekeepers, so the safekeepers don't need a lot of disk space.
                   \
                    \
                     \
-                     \          +--------+
-                      \         |        |
-                       +------> |   S3   |
-                                |        |
-                                +--------+
+                     \      +--------+
+					  \		|        |
+					   +-->	|   S3   |
+							|        |
+                            +--------+
+
 
-```
 Every WAL safekeeper holds a section of WAL, and a VCL value.
 The WAL can be divided into three portions:
 
-```
+
                                     VCL                   LSN
                                      |                     |
                                      V                     V
 .................ccccccccccccccccccccXXXXXXXXXXXXXXXXXXXXXXX
 Archived WAL       Completed WAL          In-flight WAL
-```
+
 
 Note that all this WAL kept in a safekeeper is a contiguous section.
 This is different from Aurora: In Aurora, there can be holes in the

docs/rfcs/009-snapshot-first-storage-cli.md

@@ -12,7 +12,7 @@ Init empty pageserver using `initdb` in temporary directory.
 
 `--storage_dest=FILE_PREFIX | S3_PREFIX |...` option defines object storage type, all other parameters are passed via env variables. Inspired by WAL-G style naming : https://wal-g.readthedocs.io/STORAGES/.
 
-Save`storage_dest` and other parameters in config.
+Save`storage_dest` and other parameters in config. 
 Push snapshots to `storage_dest` in background.
 
 ```
@@ -21,7 +21,7 @@ zenith start
 ```
 
 #### 2. Restart pageserver (manually or crash-recovery).
-Take `storage_dest` from pageserver config, start pageserver from latest snapshot in `storage_dest`.
+Take `storage_dest` from pageserver config, start pageserver from latest snapshot in `storage_dest`. 
 Push snapshots to `storage_dest` in background.
 
 ```
@@ -32,7 +32,7 @@ zenith start
 Start pageserver from existing snapshot.
 Path to snapshot provided via `--snapshot_path=FILE_PREFIX | S3_PREFIX | ...`
 Do not save `snapshot_path` and `snapshot_format` in config, as it is a one-time operation.
-Save`storage_dest` parameters in config.
+Save`storage_dest` parameters in config. 
 Push snapshots to `storage_dest` in background.
 ```
 //I.e. we want to start zenith on top of existing $PGDATA and use s3 as a persistent storage.
@@ -42,15 +42,15 @@ zenith start
 How to pass credentials needed for `snapshot_path`?
 
 #### 4. Export.
-Manually push snapshot to `snapshot_path` which differs from `storage_dest`
+Manually push snapshot to `snapshot_path` which differs from `storage_dest` 
 Optionally set `snapshot_format`, which can be plain pgdata format or zenith format.
 ```
 zenith export --snapshot_path=FILE_PREFIX --snapshot_format=pgdata
 ```
 
 #### Notes and questions
-- safekeeper s3_offload should use same (similar) syntax for storage. How to set it in UI?
+- walkeeper s3_offload should use same (similar) syntax for storage. How to set it in UI?
 - Why do we need `zenith init` as a separate command? Can't we init everything at first start?
 - We can think of better names for all options.
 - Export to plain postgres format will be useless, if we are not 100% compatible on page level.
-I can recall at least one such difference - PD_WAL_LOGGED flag in pages.
+I can recall at least one such difference - PD_WAL_LOGGED flag in pages.

docs/rfcs/014-safekeepers-gossip.md

@@ -1,69 +0,0 @@
-# Safekeeper gossip
-
-Extracted from this [PR](https://github.com/zenithdb/rfcs/pull/13)
-
-## Motivation
-
-In some situations, safekeeper (SK) needs coordination with other SK's that serve the same tenant:
-
-1. WAL deletion. SK needs to know what WAL was already safely replicated to delete it. Now we keep WAL indefinitely.
-2. Deciding on who is sending WAL to the pageserver. Now sending SK crash may lead to a livelock where nobody sends WAL to the pageserver.
-3. To enable SK to SK direct recovery without involving the compute
-
-## Summary
-
-Compute node has connection strings to each safekeeper. During each compute->safekeeper connection establishment, the compute node should pass down all that connection strings to each safekeeper. With that info, safekeepers may establish Postgres connections to each other and periodically send ping messages with LSN payload.
-
-## Components
-
-safekeeper, compute, compute<->safekeeper protocol, possibly console (group SK addresses)
-
-## Proposed implementation
-
-Each safekeeper can periodically ping all its peers and share connectivity and liveness info. If the ping was not receiver for, let's say, four ping periods, we may consider sending safekeeper as dead. That would mean some of the alive safekeepers should connect to the pageserver. One way to decide which one exactly: `make_connection = my_node_id == min(alive_nodes)`
-
-Since safekeepers are multi-tenant, we may establish either per-tenant physical connections or per-safekeeper ones. So it makes sense to group "logical" connections between corresponding tenants on different nodes into a single physical connection. That means that we should implement an interconnect thread that maintains physical connections and periodically broadcasts info about all tenants.
-
-Right now console may assign any 3 SK addresses to a given compute node. That may lead to a high number of gossip connections between SK's. Instead, we can assign safekeeper triples to the compute node. But if we want to "break"/" change" group by an ad-hoc action, we can do it.
-
-### Corner cases
-
-- Current safekeeper may be alive but may not have connectivity to the pageserver
-
-  To address that, we need to gossip visibility info. Based on that info, we may define SK as alive only when it can connect to the pageserver.
-
-- Current safekeeper may be alive but may not have connectivity with the compute node.
-
-  We may broadcast last_received_lsn and presence of compute connection and decide who is alive based on that.
-
-- It is tricky to decide when to shut down gossip connections because we need to be sure that pageserver got all the committed (in the distributed sense, so local SK info is not enough) records, and it may never lose them. It is not a strict requirement since `--sync-safekeepers` that happen before the compute start will allow the pageserver to consume missing WAL, but it is better to do that in the background. So the condition may look like that: `majority_max(flush_lsn) == pageserver_s3_lsn` Here we rely on the two facts:
-    - that `--sync-safekeepers` happened after the compute shutdown, and it advanced local commit_lsn's allowing pageserver to consume that WAL.
-
-    - we wait for the `pageserver_s3_lsn` advancement to avoid pageserver's last_received_lsn/disk_consistent_lsn going backward due to the disk/hardware failure and subsequent S3 recovery
-
-    If those conditions are not met, we will have some gossip activity (but that may be okay).
-
-## Pros/cons
-
-Pros:
-
-- distributed, does not introduce new services (like etcd), does not add console as a storage dependency
-- lays the foundation for gossip-based recovery
-
-Cons:
-
-- Only compute knows a set of safekeepers, but they should communicate even without compute node. In case of safekeepers restart, we will lose that info and can't gossip anymore. Hence we can't trim some WAL tail until the compute node start. Also, it is ugly.
-
-- If the console assigns a random set of safekeepers to each Postgres, we may end up in a situation where each safekeeper needs to have a connection with all other safekeepers. We can group safekeepers into isolated triples in the console to avoid that. Then "mixing" would happen only if we do rebalancing.
-
-## Alternative implementation
-
-We can have a selected node (e.g., console) with everybody reporting to it.
-
-## Security implications
-
-We don't increase the attack surface here. Communication can happen in a private network that is not exposed to users.
-
-## Scalability implications
-
-The only thing that may grow as we grow the number of computes is the number of gossip connections. But if we group safekeepers and assign a compute node to the random SK triple, the number of connections would be constant.

docs/rfcs/014-storage-lsm.md

@@ -1,145 +0,0 @@
-# Why LSM trees?
-
-In general, an LSM tree has the nice property that random updates are
-fast, but the disk writes are sequential. When a new file is created,
-it is immutable. New files are created and old ones are deleted, but
-existing files are never modified. That fits well with storing the
-files on S3.
-
-Currently, we create a lot of small files. That is mostly a problem
-with S3, because each GET/PUT operation is expensive, and LIST
-operation only returns 1000 objects at a time, and isn't free
-either. Currently, the files are "archived" together into larger
-checkpoint files before they're uploaded to S3 to alleviate that
-problem, but garbage collecting data from the archive files would be
-difficult and we have not implemented it. This proposal addresses that
-problem.
-
-
-# Overview
-
-
-```
-^ LSN
-|
-|      Memtable:     +-----------------------------+
-|                    |                             |
-|                    +-----------------------------+
-|
-|
-|            L0:     +-----------------------------+
-|                    |                             |
-|                    +-----------------------------+
-|
-|                    +-----------------------------+
-|                    |                             |
-|                    +-----------------------------+
-|
-|                    +-----------------------------+
-|                    |                             |
-|                    +-----------------------------+
-|
-|                    +-----------------------------+
-|                    |                             |
-|                    +-----------------------------+
-|
-|
-|           L1:      +-------+ +-----+ +--+  +-+
-|                    |       | |     | |  |  | |
-|                    |       | |     | |  |  | |
-|                    +-------+ +-----+ +--+  +-+
-|
-|                       +----+ +-----+ +--+  +----+
-|                       |    | |     | |  |  |    |
-|                       |    | |     | |  |  |    |
-|                       +----+ +-----+ +--+  +----+
-|
-+--------------------------------------------------------------> Page ID
-
-
-+---+
-|   |   Layer file
-+---+
-```
-
-
-# Memtable
-
-When new WAL arrives, it is first put into the Memtable. Despite the
-name, the Memtable is not a purely in-memory data structure. It can
-spill to a temporary file on disk if the system is low on memory, and
-is accessed through a buffer cache.
-
-If the page server crashes, the Memtable is lost. It is rebuilt by
-processing again the WAL that's newer than the latest layer in L0.
-
-The size of the Memtable is configured by the "checkpoint distance"
-setting. Because anything that hasn't been flushed to disk and
-uploaded to S3 yet needs to be kept in the safekeeper, the "checkpoint
-distance" also determines the amount of WAL that needs to kept in the
-safekeeper.
-
-# L0
-
-When the Memtable fills up, it is written out to a new file in L0. The
-files are immutable; when a file is created, it is never
-modified. Each file in L0 is roughly 1 GB in size (*). Like the
-Memtable, each file in L0 covers the whole key range.
-
-When enough files have been accumulated in L0, compaction
-starts. Compaction processes all the files in L0 and reshuffles the
-data to create a new set of files in L1.
-
-
-(*) except in corner cases like if we want to shut down the page
-server and want to flush out the memtable to disk even though it's not
-full yet.
-
-
-# L1
-
-L1 consists of ~ 1 GB files like L0. But each file covers only part of
-the overall key space, and a larger range of LSNs. This speeds up
-searches. When you're looking for a given page, you need to check all
-the files in L0, to see if they contain a page version for the requested
-page. But in L1, you only need to check the files whose key range covers
-the requested page. This is particularly important at cold start, when
-checking a file means downloading it from S3.
-
-Partitioning by key range also helps with garbage collection. If only a
-part of the database is updated, we will accumulate more files for
-the hot part in L1, and old files can be removed without affecting the
-cold part.
-
-
-# Image layers
-
-So far, we've only talked about delta layers. In addition to the delta
-layers, we create image layers, when "enough" WAL has been accumulated
-for some part of the database. Each image layer covers a 1 GB range of
-key space. It contains images of the pages at a single LSN, a snapshot
-if you will.
-
-The exact heuristic for what "enough" means is not clear yet. Maybe
-create a new image layer when 10 GB of WAL has been accumulated for a
-1 GB segment.
-
-The image layers limit the number of layers that a search needs to
-check. That put a cap on read latency, and it also allows garbage
-collecting layers that are older than the GC horizon.
-
-
-# Partitioning scheme
-
-When compaction happens and creates a new set of files in L1, how do
-we partition the data into the files?
-
-- Goal is that each file is ~ 1 GB in size
-- Try to match partition boundaries at relation boundaries. (See [1]
-  for how PebblesDB does this, and for why that's important)
-- Greedy algorithm
-
-# Additional Reading
-
-[1] Paper on PebblesDB and how it does partitioning.
-https://www.cs.utexas.edu/~rak/papers/sosp17-pebblesdb.pdf

docs/rfcs/015-storage-messaging.md

@@ -1,295 +0,0 @@
-# Storage messaging
-
-Created on 19.01.22
-
-Initially created [here](https://github.com/zenithdb/rfcs/pull/16) by @kelvich.
-
-That it is an alternative to (014-safekeeper-gossip)[]
-
-## Motivation
-
-As in 014-safekeeper-gossip we need to solve the following problems:
-
-* Trim WAL on safekeepers
-* Decide on which SK should push WAL to the S3
-* Decide on which SK should forward WAL to the pageserver
-* Decide on when to shut down SK<->pageserver connection
-
-This RFC suggests a more generic and hopefully more manageable way to address those problems. However, unlike 014-safekeeper-gossip, it does not bring us any closer to safekeeper-to-safekeeper recovery but rather unties two sets of different issues we previously wanted to solve with gossip.
-
-Also, with this approach, we would not need "call me maybe" anymore, and the pageserver will have all the data required to understand that it needs to reconnect to another safekeeper.
-
-## Summary
-
-Instead of p2p gossip, let's have a centralized broker where all the storage nodes report per-timeline state. Each storage node should have a `--broker-url=1.2.3.4` CLI param.
-
-Here I propose two ways to do that. After a lot of arguing with myself, I'm leaning towards the etcd approach. My arguments for it are in the pros/cons section. Both options require adding a Grpc client in our codebase either directly or as an etcd dependency.
-
-## Non-goals
-
-That RFC does *not* suggest moving the compute to pageserver and compute to safekeeper mappings out of the console. The console is still the only place in the cluster responsible for the persistency of that info. So I'm implying that each pageserver and safekeeper exactly knows what timelines he serves, as it currently is. We need some mechanism for a new pageserver to discover mapping info, but that is out of the scope of this RFC.
-
-## Impacted components
-
-pageserver, safekeeper
-adds either etcd or console as a storage dependency
-
-## Possible implementation: custom message broker in the console
-
-We've decided to go with an etcd approach instead of the message broker.
-
-<details closed>
-<summary>Original suggestion</summary>
-<br>
-We can add a Grpc service in the console that acts as a message broker since the console knows the addresses of all the components. The broker can ignore the payload and only redirect messages. So, for example, each safekeeper may send a message to the peering safekeepers or to the pageserver responsible for a given timeline.
-
-Message format could be `{sender, destination, payload}`.
-
-The destination is either:
-1. `sk_#{tenant}_#{timeline}` -- to be broadcasted on all safekeepers, responsible for that timeline, or
-2. `pserver_#{tenant}_#{timeline}` -- to be broadcasted on all pageservers, responsible for that timeline
-
-Sender is either:
-1. `sk_#{sk_id}`, or
-2. `pserver_#{pserver_id}`
-
-I can think of the following behavior to address our original problems:
-
-* WAL trimming
-  Each safekeeper periodically broadcasts `(write_lsn, commit_lsn)` to all peering (peering == responsible for that timeline) safekeepers
-
-* Decide on which SK should push WAL to the S3
-
-  Each safekeeper periodically broadcasts `i_am_alive_#{current_timestamp}` message to all peering safekeepers. That way, safekeepers may maintain the vector of alive peers (loose one, with false negatives). Alive safekeeper with the minimal id pushes data to S3.
-
-* Decide on which SK should forward WAL to the pageserver
-
-  Each safekeeper periodically sends (write_lsn, commit_lsn, compute_connected) to the relevant pageservers. With that info, pageserver can maintain a view of the safekeepers state, connect to a random one, and detect the moments (e.g., one the safekeepers is not making progress or down) when it needs to reconnect to another safekeeper. Pageserver should resolve exact IP addresses through the console, e.g., exchange `#sk_#{sk_id}` to `4.5.6.7:6400`.
-
-  Pageserver connection to the safekeeper triggered by the state change `compute_connected: false -> true`. With that, we don't need "call me maybe" anymore.
-
-  Also, we don't have a "peer address amnesia" problem as in the gossip approach (with gossip, after a simultaneous reboot, safekeepers wouldn't know each other addresses until the next compute connection).
-
-* Decide on when to shutdown sk<->pageserver connection
-
-  Again, pageserver would have all the info to understand when to shut down the safekeeper connection.
-
-### Scalability
-
-One node is enough (c) No, seriously, it is enough.
-
-### High Availability
-
-Broker lives in the console, so we can rely on k8s maintaining the console app alive.
-
-If the console is down, we won't trim WAL and reconnect the pageserver to another safekeeper. But, at the same, if the console is down, we already can't accept new compute connections and start stopped computes, so we are making things a bit worse, but not dramatically.
-
-### Interactions
-
-```
-         .________________.
-sk_1 <-> |                | <-> pserver_1
-...      | Console broker |     ...
-sk_n <-> |________________| <-> pserver_m
-```
-</details>
-
-
-## Implementation: etcd state store
-
-Alternatively, we can set up `etcd` and maintain the following data structure in it:
-
-```ruby
-"compute_#{tenant}_#{timeline}" => {
-    safekeepers => {
-        "sk_#{sk_id}" => {
-            write_lsn: "0/AEDF130",
-            commit_lsn: "0/AEDF100",
-            compute_connected: true,
-            last_updated: 1642621138,
-        },
-    }
-}
-```
-
-As etcd doesn't support field updates in the nested objects that translates to the following set of keys:
-
-```ruby
-"compute_#{tenant}_#{timeline}/safekeepers/sk_#{sk_id}/write_lsn",
-"compute_#{tenant}_#{timeline}/safekeepers/sk_#{sk_id}/commit_lsn",
-...
-```
-
-Each storage node can subscribe to the relevant sets of keys and maintain a local view of that structure. So in terms of the data flow, everything is the same as in the previous approach. Still, we can avoid implementing the message broker and prevent runtime storage dependency on a console.
-
-### Safekeeper address discovery
-
-During the startup safekeeper should publish the address he is listening on as the part of `{"sk_#{sk_id}" => ip_address}`. Then the pageserver can resolve `sk_#{sk_id}` to the actual address. This way it would work both locally and in the cloud setup. Safekeeper should have `--advertised-address` CLI option so that we can listen on e.g. 0.0.0.0 but advertize something more useful.
-
-### Safekeeper behavior
-
-For each timeline safekeeper periodically broadcasts `compute_#{tenant}_#{timeline}/safekeepers/sk_#{sk_id}/*` fields. It subscribes to changes of `compute_#{tenant}_#{timeline}` -- that way safekeeper will have an information about peering safekeepers.
-That amount of information is enough to properly trim WAL. To decide on who is pushing the data to S3 safekeeper may use etcd leases or broadcast a timestamp and hence track who is alive.
-
-### Pageserver behavior
-
-Pageserver subscribes to `compute_#{tenant}_#{timeline}` for each tenant it owns. With that info, pageserver can maintain a view of the safekeepers state, connect to a random one, and detect the moments (e.g., one the safekeepers is not making progress or down) when it needs to reconnect to another safekeeper. Pageserver should resolve exact IP addresses through the console, e.g., exchange `#sk_#{sk_id}` to `4.5.6.7:6400`.
-
-Pageserver connection to the safekeeper can be triggered by the state change `compute_connected: false -> true`. With that, we don't need "call me maybe" anymore.
-
-As an alternative to compute_connected, we can track timestamp of the latest message arrived to safekeeper from compute. Usually compute broadcasts KeepAlive to all safekeepers every second, so it'll be updated every second when connection is ok. Then the connection can be considered down when this timestamp isn't updated for a several seconds.
-
-This will help to faster detect issues with safekeeper (and switch to another) in the following cases:
-
-      when compute failed but TCP connection stays alive until timeout (usually about a minute)
-      when safekeeper failed and didn't set compute_connected to false
-
-Another way to deal with [2] is to process (write_lsn, commit_lsn, compute_connected) as a KeepAlive on the pageserver side and detect issues when sk_id don't send anything for some time. This way is fully compliant to this RFC.
-
-Also, we don't have a "peer address amnesia" problem as in the gossip approach (with gossip, after a simultaneous reboot, safekeepers wouldn't know each other addresses until the next compute connection).
-
-### Interactions
-
-```
-         .________________.
-sk_1 <-> |                | <-> pserver_1
-...      |      etcd      |     ...
-sk_n <-> |________________| <-> pserver_m
-```
-
-### Sequence diagrams for different workflows
-
-#### Cluster startup
-
-```mermaid
-sequenceDiagram
-    autonumber
-    participant C as Compute
-    participant SK1
-    participant SK2
-    participant SK3
-    participant PS1
-    participant PS2
-    participant O as Orchestrator
-    participant M as Metadata Service
-
-    PS1->>M: subscribe to updates to state of timeline N
-    C->>+SK1: WAL push
-    loop constantly update current lsns
-        SK1->>-M: I'm at lsn A
-    end
-    C->>+SK2: WAL push
-    loop constantly update current lsns
-        SK2->>-M: I'm at lsn B
-    end
-    C->>+SK3: WAL push
-    loop constantly update current lsns
-        SK3->>-M: I'm at lsn C
-    end
-    loop request pages
-        C->>+PS1: get_page@lsn
-        PS1->>-C: page image
-    end
-    M->>PS1: New compute appeared for timeline N. SK1 at A, SK2 at B, SK3 at C
-    note over PS1: Say SK1 at A=200, SK2 at B=150 SK3 at C=100 <br> so connect to SK1 because it is the most up to date one
-    PS1->>SK1: start replication
-```
-
-#### Behavour of services during typical operations
-
-```mermaid
-sequenceDiagram
-    autonumber
-    participant C as Compute
-    participant SK1
-    participant SK2
-    participant SK3
-    participant PS1
-    participant PS2
-    participant O as Orchestrator
-    participant M as Metadata Service
-
-    note over C,M: Scenario 1: Pageserver checkpoint
-    note over PS1: Upload data to S3
-    PS1->>M: Update remote consistent lsn
-    M->>SK1: propagate remote consistent lsn update
-    note over SK1: truncate WAL up to remote consistent lsn
-    M->>SK2: propagate remote consistent lsn update
-    note over SK2: truncate WAL up to remote consistent lsn
-    M->>SK3: propagate remote consistent lsn update
-    note over SK3: truncate WAL up to remote consistent lsn
-    note over C,M: Scenario 2: SK1 finds itself lagging behind MAX(150 (SK2), 200 (SK2)) - 100 (SK1) > THRESHOLD
-    SK1->>SK2: Fetch WAL delta between 100 (SK1) and 200 (SK2)
-    note over C,M: Scenario 3: PS1 detects that SK1 is lagging behind: Connection from SK1 is broken or there is no messages from it in 30 seconds.
-    note over PS1: e.g. SK2 is at 150, SK3 is at 100, chose SK2 as a new replication source
-    PS1->>SK2: start replication
-```
-
-#### Behaviour during timeline relocation
-
-```mermaid
-sequenceDiagram
-    autonumber
-    participant C as Compute
-    participant SK1
-    participant SK2
-    participant SK3
-    participant PS1
-    participant PS2
-    participant O as Orchestrator
-    participant M as Metadata Service
-
-    note over C,M: Timeline is being relocated from PS1 to PS2
-    O->>+PS2: Attach timeline
-    PS2->>-O: 202 Accepted if timeline exists in S3
-    note over PS2: Download timeline from S3
-     note over O: Poll for timeline download (or subscribe to metadata service)
-    loop wait for attach to complete
-        O->>PS2: timeline detail should answer that timeline is ready
-    end
-    PS2->>M: Register downloaded timeline
-    PS2->>M: Get safekeepers for timeline, subscribe to changes
-    PS2->>SK1: Start replication to catch up
-    note over O: PS2 catched up, time to switch compute
-    O->>C: Restart compute with new pageserver url in config
-    note over C: Wal push is restarted
-    loop request pages
-        C->>+PS2: get_page@lsn
-        PS2->>-C: page image
-    end
-    O->>PS1: detach timeline
-    note over C,M: Scenario 1: Attach call failed
-    O--xPS2: Attach timeline
-    note over O: The operation can be safely retried, <br> if we hit some threshold we can try another pageserver
-    note over C,M: Scenario 2: Attach succeeded but pageserver failed to download the data or start replication
-    loop wait for attach to complete
-        O--xPS2: timeline detail should answer that timeline is ready
-    end
-    note over O: Can wait for a timeout, and then try another pageserver <br> there should be a limit on number of different pageservers to try
-    note over C,M: Scenario 3: Detach fails
-    O--xPS1: Detach timeline
-    note over O: can be retried, if continues to fail might lead to data duplication in s3
-```
-
-# Pros/cons
-
-## Console broker/etcd vs gossip:
-
-Gossip pros:
-* gossip allows running storage without the console or etcd
-
-Console broker/etcd pros:
-* simpler
-* solves "call me maybe" as well
-* avoid possible N-to-N connection issues with gossip without grouping safekeepers in pre-defined triples
-
-## Console broker vs. etcd:
-
-Initially, I wanted to avoid etcd as a dependency mostly because I've seen how painful for Clickhouse was their ZooKeeper dependency: in each chat, at each conference, people were complaining about configuration and maintenance barriers with ZooKeeper. It was that bad that ClickHouse re-implemented ZooKeeper to embed it: https://clickhouse.com/docs/en/operations/clickhouse-keeper/.
-
-But with an etcd we are in a bit different situation:
-
-1. We don't need persistency and strong consistency guarantees for the data we store in the etcd
-2. etcd uses Grpc as a protocol, and messages are pretty simple
-
-So it looks like implementing in-mem store with etcd interface is straightforward thing _if we will want that in future_. At the same time, we can avoid implementing it right now, and we will be able to run local zenith installation with etcd running somewhere in the background (as opposed to building and running console, which in turn requires Postgres).

docs/rfcs/016-connection-routing.md

@@ -1,151 +0,0 @@
-# Dispatching a connection
-
-For each client connection, Neon service needs to authenticate the
-connection, and route it to the right PostgreSQL instance.
-
-## Authentication
-
-There are three different ways to authenticate:
-
-- anonymous; no authentication needed
-- PostgreSQL authentication
-- github single sign-on using browser
-
-In anonymous access, the user doesn't need to perform any
-authentication at all. This can be used e.g. in interactive PostgreSQL
-documentation, allowing you to run the examples very quickly. Similar
-to sqlfiddle.com.
-
-PostgreSQL authentication works the same as always. All the different
-PostgreSQL authentication options like SCRAM, kerberos, etc. are
-available. [1]
-
-The third option is to authenticate with github single sign-on. When
-you open the connection in psql, you get a link that you open with
-your browser. Opening the link redirects you to github authentication,
-and lets the connection to proceed. This is also known as "Link auth" [2].
-
-
-## Routing the connection
-
-When a client starts a connection, it needs to be routed to the
-correct PostgreSQL instance. Routing can be done by the proxy, acting
-as a man-in-the-middle, or the connection can be routed at the network
-level based on the hostname or IP address.
-
-Either way, Neon needs to identify which PostgreSQL instance the
-connection should be routed to. If the instance is not already
-running, it needs to be started. Some connections always require a new
-PostgreSQL instance to be created, e.g. if you want to run a one-off
-query against a particular point-in-time.
-
-The PostgreSQL instance is identified by:
-- Neon account (possibly anonymous)
-- cluster (known as tenant in the storage?)
-- branch or snapshot name
-- timestamp (PITR)
-- primary or read-replica
-- one-off read replica
-- one-off writeable branch
-
-When you are using regular PostgreSQL authentication or anonymous
-access, the connection URL needs to contain all the information needed
-for the routing. With github single sign-on, the browser is involved
-and some details - the Neon account in particular - can be deduced
-from the authentication exchange.
-
-There are three methods for identifying the PostgreSQL instance:
-
-- Browser interaction (link auth)
-- Options in the connection URL and the domain name
-- A pre-defined endpoint, identified by domain name or IP address
-
-### Link Auth
-
-    postgres://<username>@start.neon.tech/<dbname>
-
-This gives you a link that you open in browser. Clicking the link
-performs github authentication, and the Neon account name is
-provided to the proxy behind the scenes. The proxy routes the
-connection to the primary PostgreSQL instance in cluster called
-"main", branch "main".
-
-Further ideas:
-- You could pre-define a different target for link auth
-  connections in the UI.
-- You could have a drop-down in the browser, allowing you to connect
-  to any cluster you want. Link Auth can be like Teleport.
-
-### Connection URL
-
-The connection URL looks like this:
-
-    postgres://<username>@<cluster-id>.db.neon.tech/<dbname>
-
-By default, this connects you to the primary PostgreSQL instance
-running on the "main" branch in the named cluster [3]. However, you can
-change that by specifying options in the connection URL. The following
-options are supported:
-
-| option name  | Description                                                                                       | Examples                                            |
-| ---          | ---                                                                                               | ---                                                 |
-| cluster      | Cluster name                                                                                      | cluster:myproject                                   |
-| branch       | Branch name                                                                                       | branch:main                                         |
-| timestamp    | Connect to an instance at given point-in-time.                                                    | timestamp:2022-04-08 timestamp:2022-04-08T11:42:16Z |
-| lsn          | Connect to an instance at given LSN                                                               | lsn:0/12FF0420                                      |
-| read-replica | Connect to a read-replica. If the parameter is 'new', a new instance is created for this session. | read-replica read-replica:new                       |
-
-For example, to read branch 'testing' as it was on Mar 31, 2022, you could
-specify a timestamp in the connection URL [4]:
-
-    postgres://alice@cluster-1234.db.neon.tech/postgres?options=branch:testing,timestamp:2022-03-31
-
-Connecting with cluster name and options can be disabled in the UI. If
-disabled, you can only connect using a pre-defined endpoint.
-
-### Pre-defined Endpoint
-
-Instead of providing the cluster name, branch, and all those options
-in the connection URL, you can define a named endpoint with the same
-options.
-
-In the UI, click "create endpoint". Fill in the details:
-
-- Cluster name
-- Branch
-- timestamp or LSN
-- is this for the primary or for a read replica
-- etc.
-
-When you click Finish, a named endpoint is created. You can now use the endpoint ID to connect:
-
-    postgres://<username>@<endpoint-id>.endpoint.neon.tech/<dbname>
-
-
-An endpoint can be assigned a static or dynamic IP address, so that
-you can connect to it with clients that don't support TLS SNI. Maybe
-bypass the proxy altogether, but that ought to be invisible to the
-user.
-
-You can limit the range of source IP addresses that are allowed to
-connect to an endpoint. An endpoint can also be exposed in an Amazon
-VPC, allowing direct connections from applications.
-
-
-# Footnotes
-
-[1] I'm not sure how feasible it is to set up configure like Kerberos
-or LDAP in a cloud environment. But in principle I think we should
-allow customers to have the full power of PostgreSQL, including all
-authentication options. However, it's up to the customer to configure
-it correctly.
-
-[2] Link is a way to both authenticate and to route the connection
-
-[3] This assumes that cluster-ids are globally unique, across all
-Neon accounts.
-
-[4] The syntax accepted in the connection URL is limited by libpq. The
-only way to pass arbitrary options to the server (or our proxy) is
-with the "options" keyword, and the options must be percent-encoded. I
-think the above would work but i haven't tested it

docs/rfcs/cluster-size-limits.md

@@ -1,79 +0,0 @@
-Cluster size limits
-==================
-
-## Summary
-
-One of the resource consumption limits for free-tier users is a cluster size limit.
-
-To enforce it, we need to calculate the timeline size and check if the limit is reached before relation create/extend operations.
-If the limit is reached, the query must fail with some meaningful error/warning.
-We may want to exempt some operations from the quota to allow users free space to fit back into the limit.
-
-The stateless compute node that performs validation is separate from the storage that calculates the usage, so we need to exchange cluster size information between those components.
-
-## Motivation
-
-Limit the maximum size of a PostgreSQL instance to limit free tier users (and other tiers in the future).
-First of all, this is needed to control our free tier production costs.
-Another reason to limit resources is risk management — we haven't (fully) tested and optimized zenith for big clusters,
-so we don't want to give users access to the functionality that we don't think is ready.
-
-## Components
-
-* pageserver - calculate the size consumed by a timeline and add it to the feedback message.
-* safekeeper - pass feedback message from pageserver to compute.
-* compute - receive feedback message, enforce size limit based on GUC `zenith.max_cluster_size`.
-* console - set and update `zenith.max_cluster_size` setting
-
-## Proposed implementation
-
-First of all, it's necessary to define timeline size.
-
-The current approach is to count all data, including SLRUs. (not including WAL)
-Here we think of it as a physical disk underneath the Postgres cluster.
-This is how the `LOGICAL_TIMELINE_SIZE` metric is implemented in the pageserver.
-
-Alternatively, we could count only relation data. As in pg_database_size().
-This approach is somewhat more user-friendly because it is the data that is really affected by the user.
-On the other hand, it puts us in a weaker position than other services, i.e., RDS.
-We will need to refactor the timeline_size counter or add another counter to implement it. 
-
-Timeline size is updated during wal digestion. It is not versioned and is valid at the last_received_lsn moment.
-Then this size should be reported to compute node.
-
-`current_timeline_size` value is included in the walreceiver's custom feedback message: `ZenithFeedback.`
-
-(PR about protocol changes https://github.com/zenithdb/zenith/pull/1037).
-
-This message is received by the safekeeper and propagated to compute node as a part of `AppendResponse`.
-
-Finally, when compute node receives the `current_timeline_size` from safekeeper (or from pageserver directly), it updates the global variable.
-
-And then every zenith_extend() operation checks if limit is reached `(current_timeline_size > zenith.max_cluster_size)` and throws `ERRCODE_DISK_FULL` error if so.
-(see Postgres error codes [https://www.postgresql.org/docs/devel/errcodes-appendix.html](https://www.postgresql.org/docs/devel/errcodes-appendix.html))
-
-TODO:
-We can allow autovacuum processes to bypass this check, simply checking `IsAutoVacuumWorkerProcess()`.
-It would be nice to allow manual VACUUM and VACUUM FULL to bypass the check, but it's uneasy to distinguish these operations at the low level.
-See issues https://github.com/neondatabase/neon/issues/1245
-https://github.com/zenithdb/zenith/issues/1445
-
-TODO:
-We should warn users if the limit is soon to be reached.
-
-### **Reliability, failure modes and corner cases**
-
-1. `current_timeline_size` is valid at the last received and digested by pageserver lsn.
-    
-    If pageserver lags behind compute node, `current_timeline_size` will lag too. This lag can be tuned using backpressure, but it is not expected to be 0 all the time.
-    
-    So transactions that happen in this lsn range may cause limit overflow. Especially operations that generate (i.e., CREATE DATABASE) or free (i.e., TRUNCATE) a lot of data pages while generating a small amount of WAL. Are there other operations like this?
-    
-    Currently, CREATE DATABASE operations are restricted in the console. So this is not an issue.
-
-
-### **Security implications**
-
-We treat compute as an untrusted component. That's why we try to isolate it with secure container runtime or a VM.
-Malicious users may change the `zenith.max_cluster_size`, so we need an extra size limit check.
-To cover this case, we also monitor the compute node size in the console.

docs/settings.md

@@ -6,6 +6,7 @@ If there's no such file during `init` phase of the server, it creates the file i
 There's a possibility to pass an arbitrary config value to the pageserver binary as an argument: such values override
 the values in the config file, if any are specified for the same key and get into the final config during init phase.
 
+
 ### Config example
 
 ```toml
@@ -34,9 +35,9 @@ Yet, it validates the config values it can (e.g. postgres install dir) and error
 
 Note the `[remote_storage]` section: it's a [table](https://toml.io/en/v1.0.0#table) in TOML specification and
 
-- either has to be placed in the config after the table-less values such as `initial_superuser_name = 'zenith_admin'`
+* either has to be placed in the config after the table-less values such as `initial_superuser_name = 'zenith_admin'`
 
-- or can be placed anywhere if rewritten in identical form as [inline table](https://toml.io/en/v1.0.0#inline-table): `remote_storage = {foo = 2}`
+* or can be placed anywhere if rewritten in identical form as [inline table](https://toml.io/en/v1.0.0#inline-table): `remote_storage = {foo = 2}`
 
 ### Config values
 
@@ -56,7 +57,7 @@ but it will trigger a checkpoint operation to get it back below the
 limit.
 
 `checkpoint_distance` also determines how much WAL needs to be kept
-durable in the safekeeper. The safekeeper must have capacity to hold
+durable in the safekeeper.  The safekeeper must have capacity to hold
 this much WAL, with some headroom, otherwise you can get stuck in a
 situation where the safekeeper is full and stops accepting new WAL,
 but the pageserver is not flushing out and releasing the space in the
@@ -67,15 +68,11 @@ S3.
 
 The unit is # of bytes.
 
-#### compaction_period
+#### checkpoint_period
 
-Every `compaction_period` seconds, the page server checks if
-maintenance operations, like compaction, are needed on the layer
-files. Default is 1 s, which should be fine.
-
-#### compaction_target_size
-
-File sizes for L0 delta and L1 image layers. Default is 128MB.
+The pageserver checks whether `checkpoint_distance` has been reached
+every `checkpoint_period` seconds. Default is 1 s, which should be
+fine.
 
 #### gc_horizon
 
@@ -88,14 +85,6 @@ away.
 
 Interval at which garbage collection is triggered. Default is 100 s.
 
-#### image_creation_threshold
-
-L0 delta layer threshold for L1 iamge layer creation. Default is 3.
-
-#### pitr_interval
-
-WAL retention duration for PITR branching. Default is 30 days.
-
 #### initial_superuser_name
 
 Name of the initial superuser role, passed to initdb when a new tenant
@@ -162,11 +151,12 @@ bucket_region = 'eu-north-1'
 # Optional, pageserver uses entire bucket if the prefix is not specified.
 prefix_in_bucket = '/some/prefix/'
 
-# S3 API query limit to avoid getting errors/throttling from AWS.
-concurrency_limit = 100
-```
+# Access key to connect to the bucket ("login" part of the credentials)
+access_key_id = 'SOMEKEYAAAAASADSAH*#'
 
-If no IAM bucket access is used during the remote storage usage, use the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables to set the access credentials.
+# Secret access key to connect to the bucket ("password" part of the credentials)
+secret_access_key = 'SOMEsEcReTsd292v'
+```
 
 ###### General remote storage configuration
 
@@ -177,13 +167,14 @@ Besides, there are parameters common for all types of remote storage that can be
 
 ```toml
 [remote_storage]
-# Max number of concurrent timeline synchronized (layers uploaded or downloaded) with the remote storage at the same time.
-max_concurrent_syncs = 50
+# Max number of concurrent connections to open for uploading to or downloading from the remote storage.
+max_concurrent_sync = 100
 
 # Max number of errors a single task can have before it's considered failed and not attempted to run anymore.
 max_sync_errors = 10
 ```
 
+
 ## safekeeper
 
 TODO

docs/sourcetree.md

@@ -28,7 +28,12 @@ The pageserver has a few different duties:
 - Receive WAL from the WAL service and decode it.
 - Replay WAL that's applicable to the chunks that the Page Server maintains
 
-For more detailed info, see [/pageserver/README](/pageserver/README.md)
+For more detailed info, see `/pageserver/README`
+
+`/postgres_ffi`:
+
+Utility functions for interacting with PostgreSQL file formats.
+Misc constants, copied from PostgreSQL headers.
 
 `/proxy`:
 
@@ -52,38 +57,29 @@ PostgreSQL extension that implements storage manager API and network communicati
 
 PostgreSQL extension that contains functions needed for testing and debugging.
 
-`/safekeeper`:
+`/walkeeper`:
 
 The zenith WAL service that receives WAL from a primary compute nodes and streams it to the pageserver.
 It acts as a holding area and redistribution center for recently generated WAL.
 
-For more detailed info, see [/safekeeper/README](/safekeeper/README.md)
+For more detailed info, see `/walkeeper/README`
 
 `/workspace_hack`:
 The workspace_hack crate exists only to pin down some dependencies.
 
-We use [cargo-hakari](https://crates.io/crates/cargo-hakari) for automation.
-
 `/zenith`
 
 Main entry point for the 'zenith' CLI utility.
 TODO: Doesn't it belong to control_plane?
 
-`/libs`:
-Unites granular neon helper crates under the hood.
+`/zenith_metrics`:
 
-`/libs/postgres_ffi`:
-
-Utility functions for interacting with PostgreSQL file formats.
-Misc constants, copied from PostgreSQL headers.
-
-`/libs/utils`:
-Generic helpers that are shared between other crates in this repository.
-A subject for future modularization.
-
-`/libs/metrics`:
 Helpers for exposing Prometheus metrics from the server.
 
+`/zenith_utils`:
+
+Helpers that are shared between other crates in this repository.
+
 ## Using Python
 Note that Debian/Ubuntu Python packages are stale, as it commonly happens,
 so manual installation of dependencies is not recommended.

libs/etcd_broker/Cargo.toml

@@ -1,17 +0,0 @@
-[package]
- name = "etcd_broker"
- version = "0.1.0"
- edition = "2021"
-
- [dependencies]
- etcd-client = "0.9.0"
- regex = "1.4.5"
- serde = { version = "1.0", features = ["derive"] }
- serde_json = "1"
- serde_with = "1.12.0"
-
- utils = { path = "../utils" }
- workspace_hack = { version = "0.1", path = "../../workspace_hack" }
- tokio = "1"
- tracing = "0.1"
- thiserror = "1"

libs/etcd_broker/src/lib.rs

@@ -1,344 +0,0 @@
-//! A set of primitives to access a shared data/updates, propagated via etcd broker (not persistent).
-//! Intended to connect services to each other, not to store their data.
-use std::{
-    collections::{hash_map, HashMap},
-    fmt::Display,
-    str::FromStr,
-};
-
-use regex::{Captures, Regex};
-use serde::{Deserialize, Serialize};
-use serde_with::{serde_as, DisplayFromStr};
-
-pub use etcd_client::*;
-
-use tokio::{sync::mpsc, task::JoinHandle};
-use tracing::*;
-use utils::{
-    lsn::Lsn,
-    zid::{ZNodeId, ZTenantId, ZTenantTimelineId},
-};
-
-#[derive(Debug, Deserialize, Serialize)]
-struct SafekeeperTimeline {
-    safekeeper_id: ZNodeId,
-    info: SkTimelineInfo,
-}
-
-/// Published data about safekeeper's timeline. Fields made optional for easy migrations.
-#[serde_as]
-#[derive(Debug, Deserialize, Serialize)]
-pub struct SkTimelineInfo {
-    /// Term of the last entry.
-    pub last_log_term: Option<u64>,
-    /// LSN of the last record.
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    #[serde(default)]
-    pub flush_lsn: Option<Lsn>,
-    /// Up to which LSN safekeeper regards its WAL as committed.
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    #[serde(default)]
-    pub commit_lsn: Option<Lsn>,
-    /// LSN up to which safekeeper offloaded WAL to s3.
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    #[serde(default)]
-    pub s3_wal_lsn: Option<Lsn>,
-    /// LSN of last checkpoint uploaded by pageserver.
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    #[serde(default)]
-    pub remote_consistent_lsn: Option<Lsn>,
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    #[serde(default)]
-    pub peer_horizon_lsn: Option<Lsn>,
-    #[serde(default)]
-    pub safekeeper_connection_string: Option<String>,
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum BrokerError {
-    #[error("Etcd client error: {0}. Context: {1}")]
-    EtcdClient(etcd_client::Error, String),
-    #[error("Error during parsing etcd data: {0}")]
-    ParsingError(String),
-    #[error("Internal error: {0}")]
-    InternalError(String),
-}
-
-/// A way to control the data retrieval from a certain subscription.
-pub struct SkTimelineSubscription {
-    safekeeper_timeline_updates:
-        mpsc::UnboundedReceiver<HashMap<ZTenantTimelineId, HashMap<ZNodeId, SkTimelineInfo>>>,
-    kind: SkTimelineSubscriptionKind,
-    watcher_handle: JoinHandle<Result<(), BrokerError>>,
-    watcher: Watcher,
-}
-
-impl SkTimelineSubscription {
-    /// Asynchronously polls for more data from the subscription, suspending the current future if there's no data sent yet.
-    pub async fn fetch_data(
-        &mut self,
-    ) -> Option<HashMap<ZTenantTimelineId, HashMap<ZNodeId, SkTimelineInfo>>> {
-        self.safekeeper_timeline_updates.recv().await
-    }
-
-    /// Cancels the subscription, stopping the data poller and waiting for it to shut down.
-    pub async fn cancel(mut self) -> Result<(), BrokerError> {
-        self.watcher.cancel().await.map_err(|e| {
-            BrokerError::EtcdClient(
-                e,
-                format!(
-                    "Failed to cancel timeline subscription, kind: {:?}",
-                    self.kind
-                ),
-            )
-        })?;
-        self.watcher_handle.await.map_err(|e| {
-            BrokerError::InternalError(format!(
-                "Failed to join the timeline updates task, kind: {:?}, error: {e}",
-                self.kind
-            ))
-        })?
-    }
-}
-
-/// The subscription kind to the timeline updates from safekeeper.
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct SkTimelineSubscriptionKind {
-    broker_prefix: String,
-    kind: SubscriptionKind,
-}
-
-impl SkTimelineSubscriptionKind {
-    pub fn all(broker_prefix: String) -> Self {
-        Self {
-            broker_prefix,
-            kind: SubscriptionKind::All,
-        }
-    }
-
-    pub fn tenant(broker_prefix: String, tenant: ZTenantId) -> Self {
-        Self {
-            broker_prefix,
-            kind: SubscriptionKind::Tenant(tenant),
-        }
-    }
-
-    pub fn timeline(broker_prefix: String, timeline: ZTenantTimelineId) -> Self {
-        Self {
-            broker_prefix,
-            kind: SubscriptionKind::Timeline(timeline),
-        }
-    }
-
-    fn watch_regex(&self) -> Regex {
-        match self.kind {
-            SubscriptionKind::All => Regex::new(&format!(
-                r"^{}/([[:xdigit:]]+)/([[:xdigit:]]+)/safekeeper/([[:digit:]])$",
-                self.broker_prefix
-            ))
-            .expect("wrong regex for 'everything' subscription"),
-            SubscriptionKind::Tenant(tenant_id) => Regex::new(&format!(
-                r"^{}/{tenant_id}/([[:xdigit:]]+)/safekeeper/([[:digit:]])$",
-                self.broker_prefix
-            ))
-            .expect("wrong regex for 'tenant' subscription"),
-            SubscriptionKind::Timeline(ZTenantTimelineId {
-                tenant_id,
-                timeline_id,
-            }) => Regex::new(&format!(
-                r"^{}/{tenant_id}/{timeline_id}/safekeeper/([[:digit:]])$",
-                self.broker_prefix
-            ))
-            .expect("wrong regex for 'timeline' subscription"),
-        }
-    }
-
-    /// Etcd key to use for watching a certain timeline updates from safekeepers.
-    pub fn watch_key(&self) -> String {
-        match self.kind {
-            SubscriptionKind::All => self.broker_prefix.to_string(),
-            SubscriptionKind::Tenant(tenant_id) => {
-                format!("{}/{tenant_id}/safekeeper", self.broker_prefix)
-            }
-            SubscriptionKind::Timeline(ZTenantTimelineId {
-                tenant_id,
-                timeline_id,
-            }) => format!(
-                "{}/{tenant_id}/{timeline_id}/safekeeper",
-                self.broker_prefix
-            ),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-enum SubscriptionKind {
-    /// Get every timeline update.
-    All,
-    /// Get certain tenant timelines' updates.
-    Tenant(ZTenantId),
-    /// Get certain timeline updates.
-    Timeline(ZTenantTimelineId),
-}
-
-/// Creates a background task to poll etcd for timeline updates from safekeepers.
-/// Stops and returns `Err` on any error during etcd communication.
-/// Watches the key changes until either the watcher is cancelled via etcd or the subscription cancellation handle,
-/// exiting normally in such cases.
-pub async fn subscribe_to_safekeeper_timeline_updates(
-    client: &mut Client,
-    subscription: SkTimelineSubscriptionKind,
-) -> Result<SkTimelineSubscription, BrokerError> {
-    info!("Subscribing to timeline updates, subscription kind: {subscription:?}");
-
-    let (watcher, mut stream) = client
-        .watch(
-            subscription.watch_key(),
-            Some(WatchOptions::new().with_prefix()),
-        )
-        .await
-        .map_err(|e| {
-            BrokerError::EtcdClient(
-                e,
-                format!("Failed to init the watch for subscription {subscription:?}"),
-            )
-        })?;
-
-    let (timeline_updates_sender, safekeeper_timeline_updates) = mpsc::unbounded_channel();
-
-    let subscription_kind = subscription.kind;
-    let regex = subscription.watch_regex();
-    let watcher_handle = tokio::spawn(async move {
-        while let Some(resp) = stream.message().await.map_err(|e| BrokerError::InternalError(format!(
-            "Failed to get messages from the subscription stream, kind: {subscription_kind:?}, error: {e}"
-        )))? {
-            if resp.canceled() {
-                info!("Watch for timeline updates subscription was canceled, exiting");
-                break;
-            }
-
-            let mut timeline_updates: HashMap<ZTenantTimelineId, HashMap<ZNodeId, SkTimelineInfo>> = HashMap::new();
-            // Keep track that the timeline data updates from etcd arrive in the right order.
-            // https://etcd.io/docs/v3.5/learning/api_guarantees/#isolation-level-and-consistency-of-replicas
-            // > etcd does not ensure linearizability for watch operations. Users are expected to verify the revision of watch responses to ensure correct ordering.
-            let mut timeline_etcd_versions: HashMap<ZTenantTimelineId, i64> = HashMap::new();
-
-
-            let events = resp.events();
-            debug!("Processing {} events", events.len());
-
-            for event in events {
-                if EventType::Put == event.event_type() {
-                    if let Some(new_etcd_kv) = event.kv() {
-                        let new_kv_version = new_etcd_kv.version();
-
-                        match parse_etcd_key_value(subscription_kind, &regex, new_etcd_kv) {
-                            Ok(Some((zttid, timeline))) => {
-                                match timeline_updates
-                                    .entry(zttid)
-                                    .or_default()
-                                    .entry(timeline.safekeeper_id)
-                                {
-                                    hash_map::Entry::Occupied(mut o) => {
-                                        let old_etcd_kv_version = timeline_etcd_versions.get(&zttid).copied().unwrap_or(i64::MIN);
-                                        if old_etcd_kv_version < new_kv_version {
-                                            o.insert(timeline.info);
-                                            timeline_etcd_versions.insert(zttid,new_kv_version);
-                                        }
-                                    }
-                                    hash_map::Entry::Vacant(v) => {
-                                        v.insert(timeline.info);
-                                        timeline_etcd_versions.insert(zttid,new_kv_version);
-                                    }
-                                }
-                            }
-                            Ok(None) => {}
-                            Err(e) => error!("Failed to parse timeline update: {e}"),
-                        };
-                    }
-                }
-            }
-
-            if let Err(e) = timeline_updates_sender.send(timeline_updates) {
-                info!("Timeline updates sender got dropped, exiting: {e}");
-                break;
-            }
-        }
-
-        Ok(())
-    });
-
-    Ok(SkTimelineSubscription {
-        kind: subscription,
-        safekeeper_timeline_updates,
-        watcher_handle,
-        watcher,
-    })
-}
-
-fn parse_etcd_key_value(
-    subscription_kind: SubscriptionKind,
-    regex: &Regex,
-    kv: &KeyValue,
-) -> Result<Option<(ZTenantTimelineId, SafekeeperTimeline)>, BrokerError> {
-    let caps = if let Some(caps) = regex.captures(kv.key_str().map_err(|e| {
-        BrokerError::EtcdClient(e, format!("Failed to represent kv {kv:?} as key str"))
-    })?) {
-        caps
-    } else {
-        return Ok(None);
-    };
-
-    let (zttid, safekeeper_id) = match subscription_kind {
-        SubscriptionKind::All => (
-            ZTenantTimelineId::new(
-                parse_capture(&caps, 1).map_err(BrokerError::ParsingError)?,
-                parse_capture(&caps, 2).map_err(BrokerError::ParsingError)?,
-            ),
-            ZNodeId(parse_capture(&caps, 3).map_err(BrokerError::ParsingError)?),
-        ),
-        SubscriptionKind::Tenant(tenant_id) => (
-            ZTenantTimelineId::new(
-                tenant_id,
-                parse_capture(&caps, 1).map_err(BrokerError::ParsingError)?,
-            ),
-            ZNodeId(parse_capture(&caps, 2).map_err(BrokerError::ParsingError)?),
-        ),
-        SubscriptionKind::Timeline(zttid) => (
-            zttid,
-            ZNodeId(parse_capture(&caps, 1).map_err(BrokerError::ParsingError)?),
-        ),
-    };
-
-    let info_str = kv.value_str().map_err(|e| {
-        BrokerError::EtcdClient(e, format!("Failed to represent kv {kv:?} as value str"))
-    })?;
-    Ok(Some((
-        zttid,
-        SafekeeperTimeline {
-            safekeeper_id,
-            info: serde_json::from_str(info_str).map_err(|e| {
-                BrokerError::ParsingError(format!(
-                    "Failed to parse '{info_str}' as safekeeper timeline info: {e}"
-                ))
-            })?,
-        },
-    )))
-}
-
-fn parse_capture<T>(caps: &Captures, index: usize) -> Result<T, String>
-where
-    T: FromStr,
-    <T as FromStr>::Err: Display,
-{
-    let capture_match = caps
-        .get(index)
-        .ok_or_else(|| format!("Failed to get capture match at index {index}"))?
-        .as_str();
-    capture_match.parse().map_err(|e| {
-        format!(
-            "Failed to parse {} from {capture_match}: {e}",
-            std::any::type_name::<T>()
-        )
-    })
-}

libs/remote_storage/Cargo.toml

@@ -1,20 +0,0 @@
-[package]
-name = "remote_storage"
-version = "0.1.0"
-edition = "2021"
-
-[dependencies]
-anyhow = { version = "1.0", features = ["backtrace"] }
-tokio = { version = "1.17", features = ["sync", "macros", "fs", "io-util"] }
-tokio-util = { version = "0.7", features = ["io"] }
-tracing = "0.1.27"
-rusoto_core = "0.48"
-rusoto_s3 = "0.48"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1"
-async-trait = "0.1"
-
-workspace_hack = { version = "0.1", path = "../../workspace_hack" }
-
-[dev-dependencies]
-tempfile = "3.2"

libs/remote_storage/src/lib.rs

@@ -1,232 +0,0 @@
-//! A set of generic storage abstractions for the page server to use when backing up and restoring its state from the external storage.
-//! No other modules from this tree are supposed to be used directly by the external code.
-//!
-//! [`RemoteStorage`] trait a CRUD-like generic abstraction to use for adapting external storages with a few implementations:
-//!   * [`local_fs`] allows to use local file system as an external storage
-//!   * [`s3_bucket`] uses AWS S3 bucket as an external storage
-//!
-mod local_fs;
-mod s3_bucket;
-
-use std::{
-    borrow::Cow,
-    collections::HashMap,
-    ffi::OsStr,
-    num::{NonZeroU32, NonZeroUsize},
-    path::{Path, PathBuf},
-};
-
-use anyhow::Context;
-use tokio::io;
-use tracing::info;
-
-pub use self::{
-    local_fs::LocalFs,
-    s3_bucket::{S3Bucket, S3ObjectKey},
-};
-
-/// How many different timelines can be processed simultaneously when synchronizing layers with the remote storage.
-/// During regular work, pageserver produces one layer file per timeline checkpoint, with bursts of concurrency
-/// during start (where local and remote timelines are compared and initial sync tasks are scheduled) and timeline attach.
-/// Both cases may trigger timeline download, that might download a lot of layers. This concurrency is limited by the clients internally, if needed.
-pub const DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS: usize = 50;
-pub const DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS: u32 = 10;
-/// Currently, sync happens with AWS S3, that has two limits on requests per second:
-/// ~200 RPS for IAM services
-/// https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.html
-/// ~3500 PUT/COPY/POST/DELETE or 5500 GET/HEAD S3 requests
-/// https://aws.amazon.com/premiumsupport/knowledge-center/s3-request-limit-avoid-throttling/
-pub const DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT: usize = 100;
-
-/// Storage (potentially remote) API to manage its state.
-/// This storage tries to be unaware of any layered repository context,
-/// providing basic CRUD operations for storage files.
-#[async_trait::async_trait]
-pub trait RemoteStorage: Send + Sync {
-    /// A way to uniquely reference a file in the remote storage.
-    type RemoteObjectId;
-
-    /// Attempts to derive the storage path out of the local path, if the latter is correct.
-    fn remote_object_id(&self, local_path: &Path) -> anyhow::Result<Self::RemoteObjectId>;
-
-    /// Gets the download path of the given storage file.
-    fn local_path(&self, remote_object_id: &Self::RemoteObjectId) -> anyhow::Result<PathBuf>;
-
-    /// Lists all items the storage has right now.
-    async fn list(&self) -> anyhow::Result<Vec<Self::RemoteObjectId>>;
-
-    /// Streams the local file contents into remote into the remote storage entry.
-    async fn upload(
-        &self,
-        from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
-        // S3 PUT request requires the content length to be specified,
-        // otherwise it starts to fail with the concurrent connection count increasing.
-        from_size_bytes: usize,
-        to: &Self::RemoteObjectId,
-        metadata: Option<StorageMetadata>,
-    ) -> anyhow::Result<()>;
-
-    /// Streams the remote storage entry contents into the buffered writer given, returns the filled writer.
-    /// Returns the metadata, if any was stored with the file previously.
-    async fn download(
-        &self,
-        from: &Self::RemoteObjectId,
-        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>>;
-
-    /// Streams a given byte range of the remote storage entry contents into the buffered writer given, returns the filled writer.
-    /// Returns the metadata, if any was stored with the file previously.
-    async fn download_byte_range(
-        &self,
-        from: &Self::RemoteObjectId,
-        start_inclusive: u64,
-        end_exclusive: Option<u64>,
-        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>>;
-
-    async fn delete(&self, path: &Self::RemoteObjectId) -> anyhow::Result<()>;
-}
-
-/// TODO kb
-pub enum GenericRemoteStorage {
-    Local(LocalFs),
-    S3(S3Bucket),
-}
-
-impl GenericRemoteStorage {
-    pub fn new(
-        working_directory: PathBuf,
-        storage_config: &RemoteStorageConfig,
-    ) -> anyhow::Result<Self> {
-        match &storage_config.storage {
-            RemoteStorageKind::LocalFs(root) => {
-                info!("Using fs root '{}' as a remote storage", root.display());
-                LocalFs::new(root.clone(), working_directory).map(GenericRemoteStorage::Local)
-            }
-            RemoteStorageKind::AwsS3(s3_config) => {
-                info!("Using s3 bucket '{}' in region '{}' as a remote storage, prefix in bucket: '{:?}', bucket endpoint: '{:?}'",
-                    s3_config.bucket_name, s3_config.bucket_region, s3_config.prefix_in_bucket, s3_config.endpoint);
-                S3Bucket::new(s3_config, working_directory).map(GenericRemoteStorage::S3)
-            }
-        }
-    }
-}
-
-/// Extra set of key-value pairs that contain arbitrary metadata about the storage entry.
-/// Immutable, cannot be changed once the file is created.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct StorageMetadata(HashMap<String, String>);
-
-fn strip_path_prefix<'a>(prefix: &'a Path, path: &'a Path) -> anyhow::Result<&'a Path> {
-    if prefix == path {
-        anyhow::bail!(
-            "Prefix and the path are equal, cannot strip: '{}'",
-            prefix.display()
-        )
-    } else {
-        path.strip_prefix(prefix).with_context(|| {
-            format!(
-                "Path '{}' is not prefixed with '{}'",
-                path.display(),
-                prefix.display(),
-            )
-        })
-    }
-}
-
-/// External backup storage configuration, enough for creating a client for that storage.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct RemoteStorageConfig {
-    /// Max allowed number of concurrent sync operations between the API user and the remote storage.
-    pub max_concurrent_syncs: NonZeroUsize,
-    /// Max allowed errors before the sync task is considered failed and evicted.
-    pub max_sync_errors: NonZeroU32,
-    /// The storage connection configuration.
-    pub storage: RemoteStorageKind,
-}
-
-/// A kind of a remote storage to connect to, with its connection configuration.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum RemoteStorageKind {
-    /// Storage based on local file system.
-    /// Specify a root folder to place all stored files into.
-    LocalFs(PathBuf),
-    /// AWS S3 based storage, storing all files in the S3 bucket
-    /// specified by the config
-    AwsS3(S3Config),
-}
-
-/// AWS S3 bucket coordinates and access credentials to manage the bucket contents (read and write).
-#[derive(Clone, PartialEq, Eq)]
-pub struct S3Config {
-    /// Name of the bucket to connect to.
-    pub bucket_name: String,
-    /// The region where the bucket is located at.
-    pub bucket_region: String,
-    /// A "subfolder" in the bucket, to use the same bucket separately by multiple remote storage users at once.
-    pub prefix_in_bucket: Option<String>,
-    /// A base URL to send S3 requests to.
-    /// By default, the endpoint is derived from a region name, assuming it's
-    /// an AWS S3 region name, erroring on wrong region name.
-    /// Endpoint provides a way to support other S3 flavors and their regions.
-    ///
-    /// Example: `http://127.0.0.1:5000`
-    pub endpoint: Option<String>,
-    /// AWS S3 has various limits on its API calls, we need not to exceed those.
-    /// See [`DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT`] for more details.
-    pub concurrency_limit: NonZeroUsize,
-}
-
-impl std::fmt::Debug for S3Config {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("S3Config")
-            .field("bucket_name", &self.bucket_name)
-            .field("bucket_region", &self.bucket_region)
-            .field("prefix_in_bucket", &self.prefix_in_bucket)
-            .field("concurrency_limit", &self.concurrency_limit)
-            .finish()
-    }
-}
-
-pub fn path_with_suffix_extension(original_path: impl AsRef<Path>, suffix: &str) -> PathBuf {
-    let new_extension = match original_path
-        .as_ref()
-        .extension()
-        .map(OsStr::to_string_lossy)
-    {
-        Some(extension) => Cow::Owned(format!("{extension}.{suffix}")),
-        None => Cow::Borrowed(suffix),
-    };
-    original_path
-        .as_ref()
-        .with_extension(new_extension.as_ref())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_path_with_suffix_extension() {
-        let p = PathBuf::from("/foo/bar");
-        assert_eq!(
-            &path_with_suffix_extension(&p, "temp").to_string_lossy(),
-            "/foo/bar.temp"
-        );
-        let p = PathBuf::from("/foo/bar");
-        assert_eq!(
-            &path_with_suffix_extension(&p, "temp.temp").to_string_lossy(),
-            "/foo/bar.temp.temp"
-        );
-        let p = PathBuf::from("/foo/bar.baz");
-        assert_eq!(
-            &path_with_suffix_extension(&p, "temp.temp").to_string_lossy(),
-            "/foo/bar.baz.temp.temp"
-        );
-        let p = PathBuf::from("/foo/bar.baz");
-        assert_eq!(
-            &path_with_suffix_extension(&p, ".temp").to_string_lossy(),
-            "/foo/bar.baz..temp"
-        );
-    }
-}

libs/remote_storage/src/s3_bucket.rs

@@ -1,465 +0,0 @@
-//! AWS S3 storage wrapper around `rusoto` library.
-//!
-//! Respects `prefix_in_bucket` property from [`S3Config`],
-//! allowing multiple api users to independently work with the same S3 bucket, if
-//! their bucket prefixes are both specified and different.
-
-use std::path::{Path, PathBuf};
-
-use anyhow::Context;
-use rusoto_core::{
-    credential::{InstanceMetadataProvider, StaticProvider},
-    HttpClient, Region,
-};
-use rusoto_s3::{
-    DeleteObjectRequest, GetObjectRequest, ListObjectsV2Request, PutObjectRequest, S3Client,
-    StreamingBody, S3,
-};
-use tokio::{io, sync::Semaphore};
-use tokio_util::io::ReaderStream;
-use tracing::debug;
-
-use crate::{strip_path_prefix, RemoteStorage, S3Config};
-
-use super::StorageMetadata;
-
-const S3_PREFIX_SEPARATOR: char = '/';
-
-#[derive(Debug, Eq, PartialEq, PartialOrd, Ord, Hash)]
-pub struct S3ObjectKey(String);
-
-impl S3ObjectKey {
-    fn key(&self) -> &str {
-        &self.0
-    }
-
-    fn download_destination(&self, workdir: &Path, prefix_to_strip: Option<&str>) -> PathBuf {
-        let path_without_prefix = match prefix_to_strip {
-            Some(prefix) => self.0.strip_prefix(prefix).unwrap_or_else(|| {
-                panic!(
-                    "Could not strip prefix '{}' from S3 object key '{}'",
-                    prefix, self.0
-                )
-            }),
-            None => &self.0,
-        };
-
-        workdir.join(
-            path_without_prefix
-                .split(S3_PREFIX_SEPARATOR)
-                .collect::<PathBuf>(),
-        )
-    }
-}
-
-/// AWS S3 storage.
-pub struct S3Bucket {
-    workdir: PathBuf,
-    client: S3Client,
-    bucket_name: String,
-    prefix_in_bucket: Option<String>,
-    // Every request to S3 can be throttled or cancelled, if a certain number of requests per second is exceeded.
-    // Same goes to IAM, which is queried before every S3 request, if enabled. IAM has even lower RPS threshold.
-    // The helps to ensure we don't exceed the thresholds.
-    concurrency_limiter: Semaphore,
-}
-
-impl S3Bucket {
-    /// Creates the S3 storage, errors if incorrect AWS S3 configuration provided.
-    pub fn new(aws_config: &S3Config, workdir: PathBuf) -> anyhow::Result<Self> {
-        debug!(
-            "Creating s3 remote storage for S3 bucket {}",
-            aws_config.bucket_name
-        );
-        let region = match aws_config.endpoint.clone() {
-            Some(custom_endpoint) => Region::Custom {
-                name: aws_config.bucket_region.clone(),
-                endpoint: custom_endpoint,
-            },
-            None => aws_config
-                .bucket_region
-                .parse::<Region>()
-                .context("Failed to parse the s3 region from config")?,
-        };
-        let request_dispatcher = HttpClient::new().context("Failed to create S3 http client")?;
-
-        let access_key_id = std::env::var("AWS_ACCESS_KEY_ID").ok();
-        let secret_access_key = std::env::var("AWS_SECRET_ACCESS_KEY").ok();
-
-        let client = if access_key_id.is_none() && secret_access_key.is_none() {
-            debug!("Using IAM-based AWS access");
-            S3Client::new_with(request_dispatcher, InstanceMetadataProvider::new(), region)
-        } else {
-            debug!("Using credentials-based AWS access");
-            S3Client::new_with(
-                request_dispatcher,
-                StaticProvider::new_minimal(
-                    access_key_id.unwrap_or_default(),
-                    secret_access_key.unwrap_or_default(),
-                ),
-                region,
-            )
-        };
-
-        let prefix_in_bucket = aws_config.prefix_in_bucket.as_deref().map(|prefix| {
-            let mut prefix = prefix;
-            while prefix.starts_with(S3_PREFIX_SEPARATOR) {
-                prefix = &prefix[1..]
-            }
-
-            let mut prefix = prefix.to_string();
-            while prefix.ends_with(S3_PREFIX_SEPARATOR) {
-                prefix.pop();
-            }
-            prefix
-        });
-
-        Ok(Self {
-            client,
-            workdir,
-            bucket_name: aws_config.bucket_name.clone(),
-            prefix_in_bucket,
-            concurrency_limiter: Semaphore::new(aws_config.concurrency_limit.get()),
-        })
-    }
-}
-
-#[async_trait::async_trait]
-impl RemoteStorage for S3Bucket {
-    type RemoteObjectId = S3ObjectKey;
-
-    fn remote_object_id(&self, local_path: &Path) -> anyhow::Result<Self::RemoteObjectId> {
-        let relative_path = strip_path_prefix(&self.workdir, local_path)?;
-        let mut key = self.prefix_in_bucket.clone().unwrap_or_default();
-        for segment in relative_path {
-            key.push(S3_PREFIX_SEPARATOR);
-            key.push_str(&segment.to_string_lossy());
-        }
-        Ok(S3ObjectKey(key))
-    }
-
-    fn local_path(&self, storage_path: &Self::RemoteObjectId) -> anyhow::Result<PathBuf> {
-        Ok(storage_path.download_destination(&self.workdir, self.prefix_in_bucket.as_deref()))
-    }
-
-    async fn list(&self) -> anyhow::Result<Vec<Self::RemoteObjectId>> {
-        let mut document_keys = Vec::new();
-
-        let mut continuation_token = None;
-        loop {
-            let _guard = self
-                .concurrency_limiter
-                .acquire()
-                .await
-                .context("Concurrency limiter semaphore got closed during S3 list")?;
-            let fetch_response = self
-                .client
-                .list_objects_v2(ListObjectsV2Request {
-                    bucket: self.bucket_name.clone(),
-                    prefix: self.prefix_in_bucket.clone(),
-                    continuation_token,
-                    ..ListObjectsV2Request::default()
-                })
-                .await?;
-            document_keys.extend(
-                fetch_response
-                    .contents
-                    .unwrap_or_default()
-                    .into_iter()
-                    .filter_map(|o| Some(S3ObjectKey(o.key?))),
-            );
-
-            match fetch_response.continuation_token {
-                Some(new_token) => continuation_token = Some(new_token),
-                None => break,
-            }
-        }
-
-        Ok(document_keys)
-    }
-
-    async fn upload(
-        &self,
-        from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
-        from_size_bytes: usize,
-        to: &Self::RemoteObjectId,
-        metadata: Option<StorageMetadata>,
-    ) -> anyhow::Result<()> {
-        let _guard = self
-            .concurrency_limiter
-            .acquire()
-            .await
-            .context("Concurrency limiter semaphore got closed during S3 upload")?;
-        self.client
-            .put_object(PutObjectRequest {
-                body: Some(StreamingBody::new_with_size(
-                    ReaderStream::new(from),
-                    from_size_bytes,
-                )),
-                bucket: self.bucket_name.clone(),
-                key: to.key().to_owned(),
-                metadata: metadata.map(|m| m.0),
-                ..PutObjectRequest::default()
-            })
-            .await?;
-        Ok(())
-    }
-
-    async fn download(
-        &self,
-        from: &Self::RemoteObjectId,
-        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>> {
-        let _guard = self
-            .concurrency_limiter
-            .acquire()
-            .await
-            .context("Concurrency limiter semaphore got closed during S3 download")?;
-        let object_output = self
-            .client
-            .get_object(GetObjectRequest {
-                bucket: self.bucket_name.clone(),
-                key: from.key().to_owned(),
-                ..GetObjectRequest::default()
-            })
-            .await?;
-
-        if let Some(body) = object_output.body {
-            let mut from = io::BufReader::new(body.into_async_read());
-            io::copy(&mut from, to).await?;
-        }
-
-        Ok(object_output.metadata.map(StorageMetadata))
-    }
-
-    async fn download_byte_range(
-        &self,
-        from: &Self::RemoteObjectId,
-        start_inclusive: u64,
-        end_exclusive: Option<u64>,
-        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>> {
-        // S3 accepts ranges as https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35
-        // and needs both ends to be exclusive
-        let end_inclusive = end_exclusive.map(|end| end.saturating_sub(1));
-        let range = Some(match end_inclusive {
-            Some(end_inclusive) => format!("bytes={}-{}", start_inclusive, end_inclusive),
-            None => format!("bytes={}-", start_inclusive),
-        });
-        let _guard = self
-            .concurrency_limiter
-            .acquire()
-            .await
-            .context("Concurrency limiter semaphore got closed during S3 range download")?;
-        let object_output = self
-            .client
-            .get_object(GetObjectRequest {
-                bucket: self.bucket_name.clone(),
-                key: from.key().to_owned(),
-                range,
-                ..GetObjectRequest::default()
-            })
-            .await?;
-
-        if let Some(body) = object_output.body {
-            let mut from = io::BufReader::new(body.into_async_read());
-            io::copy(&mut from, to).await?;
-        }
-
-        Ok(object_output.metadata.map(StorageMetadata))
-    }
-
-    async fn delete(&self, path: &Self::RemoteObjectId) -> anyhow::Result<()> {
-        let _guard = self
-            .concurrency_limiter
-            .acquire()
-            .await
-            .context("Concurrency limiter semaphore got closed during S3 delete")?;
-        self.client
-            .delete_object(DeleteObjectRequest {
-                bucket: self.bucket_name.clone(),
-                key: path.key().to_owned(),
-                ..DeleteObjectRequest::default()
-            })
-            .await?;
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use tempfile::tempdir;
-
-    use super::*;
-
-    #[test]
-    fn download_destination() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-        let local_path = workdir.join("one").join("two").join("test_name");
-        let relative_path = local_path.strip_prefix(&workdir)?;
-
-        let key = S3ObjectKey(format!(
-            "{}{}",
-            S3_PREFIX_SEPARATOR,
-            relative_path
-                .iter()
-                .map(|segment| segment.to_str().unwrap())
-                .collect::<Vec<_>>()
-                .join(&S3_PREFIX_SEPARATOR.to_string()),
-        ));
-
-        assert_eq!(
-            local_path,
-            key.download_destination(&workdir, None),
-            "Download destination should consist of s3 path joined with the workdir prefix"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn storage_path_positive() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
-        let segment_1 = "matching";
-        let segment_2 = "file";
-        let local_path = &workdir.join(segment_1).join(segment_2);
-
-        let storage = dummy_storage(workdir);
-
-        let expected_key = S3ObjectKey(format!(
-            "{}{S3_PREFIX_SEPARATOR}{segment_1}{S3_PREFIX_SEPARATOR}{segment_2}",
-            storage.prefix_in_bucket.as_deref().unwrap_or_default(),
-        ));
-
-        let actual_key = storage
-            .remote_object_id(local_path)
-            .expect("Matching path should map to S3 path normally");
-        assert_eq!(
-            expected_key,
-            actual_key,
-            "S3 key from the matching path should contain all segments after the workspace prefix, separated with S3 separator"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn storage_path_negatives() -> anyhow::Result<()> {
-        #[track_caller]
-        fn storage_path_error(storage: &S3Bucket, mismatching_path: &Path) -> String {
-            match storage.remote_object_id(mismatching_path) {
-                Ok(wrong_key) => panic!(
-                    "Expected path '{}' to error, but got S3 key: {:?}",
-                    mismatching_path.display(),
-                    wrong_key,
-                ),
-                Err(e) => e.to_string(),
-            }
-        }
-
-        let workdir = tempdir()?.path().to_owned();
-        let storage = dummy_storage(workdir.clone());
-
-        let error_message = storage_path_error(&storage, &workdir);
-        assert!(
-            error_message.contains("Prefix and the path are equal"),
-            "Message '{}' does not contain the required string",
-            error_message
-        );
-
-        let mismatching_path = PathBuf::from("somewhere").join("else");
-        let error_message = storage_path_error(&storage, &mismatching_path);
-        assert!(
-            error_message.contains(mismatching_path.to_str().unwrap()),
-            "Error should mention wrong path"
-        );
-        assert!(
-            error_message.contains(workdir.to_str().unwrap()),
-            "Error should mention server workdir"
-        );
-        assert!(
-            error_message.contains("is not prefixed with"),
-            "Message '{}' does not contain a required string",
-            error_message
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn local_path_positive() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-        let storage = dummy_storage(workdir.clone());
-        let timeline_dir = workdir.join("timelines").join("test_timeline");
-        let relative_timeline_path = timeline_dir.strip_prefix(&workdir)?;
-
-        let s3_key = create_s3_key(
-            &relative_timeline_path.join("not a metadata"),
-            storage.prefix_in_bucket.as_deref(),
-        );
-        assert_eq!(
-            s3_key.download_destination(&workdir, storage.prefix_in_bucket.as_deref()),
-            storage
-                .local_path(&s3_key)
-                .expect("For a valid input, valid S3 info should be parsed"),
-            "Should be able to parse metadata out of the correctly named remote delta file"
-        );
-
-        let s3_key = create_s3_key(
-            &relative_timeline_path.join("metadata"),
-            storage.prefix_in_bucket.as_deref(),
-        );
-        assert_eq!(
-            s3_key.download_destination(&workdir, storage.prefix_in_bucket.as_deref()),
-            storage
-                .local_path(&s3_key)
-                .expect("For a valid input, valid S3 info should be parsed"),
-            "Should be able to parse metadata out of the correctly named remote metadata file"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn download_destination_matches_original_path() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-        let original_path = workdir
-            .join("timelines")
-            .join("some_timeline")
-            .join("some name");
-
-        let dummy_storage = dummy_storage(workdir);
-
-        let key = dummy_storage.remote_object_id(&original_path)?;
-        let download_destination = dummy_storage.local_path(&key)?;
-
-        assert_eq!(
-            original_path, download_destination,
-            "'original path -> storage key -> matching fs path' transformation should produce the same path as the input one for the correct path"
-        );
-
-        Ok(())
-    }
-
-    fn dummy_storage(workdir: PathBuf) -> S3Bucket {
-        S3Bucket {
-            workdir,
-            client: S3Client::new("us-east-1".parse().unwrap()),
-            bucket_name: "dummy-bucket".to_string(),
-            prefix_in_bucket: Some("dummy_prefix/".to_string()),
-            concurrency_limiter: Semaphore::new(1),
-        }
-    }
-
-    fn create_s3_key(relative_file_path: &Path, prefix: Option<&str>) -> S3ObjectKey {
-        S3ObjectKey(relative_file_path.iter().fold(
-            prefix.unwrap_or_default().to_string(),
-            |mut path_string, segment| {
-                path_string.push(S3_PREFIX_SEPARATOR);
-                path_string.push_str(segment.to_str().unwrap());
-                path_string
-            },
-        ))
-    }
-}

libs/utils/scripts/restore_from_wal.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-PG_BIN=$1
-WAL_PATH=$2
-DATA_DIR=$3
-PORT=$4
-SYSID=`od -A n -j 24 -N 8 -t d8 $WAL_PATH/000000010000000000000002* | cut -c 3-`
-rm -fr $DATA_DIR
-env -i LD_LIBRARY_PATH=$PG_BIN/../lib $PG_BIN/initdb -E utf8 -U zenith_admin -D $DATA_DIR --sysid=$SYSID
-echo port=$PORT >> $DATA_DIR/postgresql.conf
-REDO_POS=0x`$PG_BIN/pg_controldata -D $DATA_DIR | fgrep "REDO location"| cut -c 42-`
-declare -i WAL_SIZE=$REDO_POS+114
-$PG_BIN/pg_ctl -D $DATA_DIR -l logfile start
-$PG_BIN/pg_ctl -D $DATA_DIR -l logfile stop -m immediate
-cp $DATA_DIR/pg_wal/000000010000000000000001 .
-cp $WAL_PATH/* $DATA_DIR/pg_wal/
-if [ -f $DATA_DIR/pg_wal/*.partial ]
-then
-	(cd $DATA_DIR/pg_wal ; for partial in \*.partial ; do mv $partial `basename $partial .partial` ; done)
-fi
-dd if=000000010000000000000001 of=$DATA_DIR/pg_wal/000000010000000000000001 bs=$WAL_SIZE count=1 conv=notrunc
-rm -f 000000010000000000000001

libs/utils/scripts/restore_from_wal_archive.sh

@@ -1,20 +0,0 @@
-PG_BIN=$1
-WAL_PATH=$2
-DATA_DIR=$3
-PORT=$4
-SYSID=`od -A n -j 24 -N 8 -t d8 $WAL_PATH/000000010000000000000002* | cut -c 3-`
-rm -fr $DATA_DIR /tmp/pg_wals
-mkdir /tmp/pg_wals
-env -i LD_LIBRARY_PATH=$PG_BIN/../lib $PG_BIN/initdb -E utf8 -U zenith_admin -D $DATA_DIR --sysid=$SYSID
-echo port=$PORT >> $DATA_DIR/postgresql.conf
-REDO_POS=0x`$PG_BIN/pg_controldata -D $DATA_DIR | fgrep "REDO location"| cut -c 42-`
-declare -i WAL_SIZE=$REDO_POS+114
-cp $WAL_PATH/* /tmp/pg_wals
-if [ -f $DATA_DIR/pg_wal/*.partial ]
-then
-	(cd /tmp/pg_wals ; for partial in \*.partial ; do  mv $partial `basename $partial .partial` ; done)
-fi
-dd if=$DATA_DIR/pg_wal/000000010000000000000001 of=/tmp/pg_wals/000000010000000000000001 bs=$WAL_SIZE count=1 conv=notrunc
-echo > $DATA_DIR/recovery.signal
-rm -f $DATA_DIR/pg_wal/*
-echo "restore_command = 'cp /tmp/pg_wals/%f %p'" >> $DATA_DIR/postgresql.conf

libs/utils/src/lib.rs

@@ -1,97 +0,0 @@
-//! `utils` is intended to be a place to put code that is shared
-//! between other crates in this repository.
-
-#![allow(clippy::manual_range_contains)]
-
-/// `Lsn` type implements common tasks on Log Sequence Numbers
-pub mod lsn;
-/// SeqWait allows waiting for a future sequence number to arrive
-pub mod seqwait;
-
-/// append only ordered map implemented with a Vec
-pub mod vec_map;
-
-// Async version of SeqWait. Currently unused.
-// pub mod seqwait_async;
-
-pub mod bin_ser;
-pub mod postgres_backend;
-pub mod pq_proto;
-
-// dealing with connstring parsing and handy access to it's parts
-pub mod connstring;
-
-// helper functions for creating and fsyncing directories/trees
-pub mod crashsafe_dir;
-
-// common authentication routines
-pub mod auth;
-
-// utility functions and helper traits for unified unique id generation/serialization etc.
-pub mod zid;
-// http endpoint utils
-pub mod http;
-
-// socket splitting utils
-pub mod sock_split;
-
-// common log initialisation routine
-pub mod logging;
-
-// Misc
-pub mod accum;
-pub mod shutdown;
-
-// Tools for calling certain async methods in sync contexts
-pub mod sync;
-
-// Utility for binding TcpListeners with proper socket options.
-pub mod tcp_listener;
-
-// Utility for putting a raw file descriptor into non-blocking mode
-pub mod nonblock;
-
-// Default signal handling
-pub mod signals;
-
-/// This is a shortcut to embed git sha into binaries and avoid copying the same build script to all packages
-///
-/// we have several cases:
-/// * building locally from git repo
-/// * building in CI from git repo
-/// * building in docker (either in CI or locally)
-///
-/// One thing to note is that .git is not available in docker (and it is bad to include it there).
-/// So everything becides docker build is covered by git_version crate, and docker uses a `GIT_VERSION` argument to get the value required.
-/// It takes variable from build process env and puts it to the rustc env. And then we can retrieve it here by using env! macro.
-/// Git version received from environment variable used as a fallback in git_version invokation.
-/// And to avoid running buildscript every recompilation, we use rerun-if-env-changed option.
-/// So the build script will be run only when GIT_VERSION envvar has changed.
-///
-/// Why not to use buildscript to get git commit sha directly without procmacro from different crate?
-/// Caching and workspaces complicates that. In case `utils` is not
-/// recompiled due to caching then version may become outdated.
-/// git_version crate handles that case by introducing a dependency on .git internals via include_bytes! macro,
-/// so if we changed the index state git_version will pick that up and rerun the macro.
-///
-/// Note that with git_version prefix is `git:` and in case of git version from env its `git-env:`.
-///
-/// #############################################################################################
-/// TODO this macro is not the way the library is intended to be used, see https://github.com/neondatabase/neon/issues/1565 for details.
-/// We use `cachepot` to reduce our current CI build times: https://github.com/neondatabase/cloud/pull/1033#issuecomment-1100935036
-/// Yet, it seems to ignore the GIT_VERSION env variable, passed to Docker build, even with build.rs that contains
-/// `println!("cargo:rerun-if-env-changed=GIT_VERSION");` code for cachepot cache invalidation.
-/// The problem needs further investigation and regular `const` declaration instead of a macro.
-#[macro_export]
-macro_rules! project_git_version {
-    ($const_identifier:ident) => {
-        const $const_identifier: &str = git_version::git_version!(
-            prefix = "git:",
-            fallback = concat!(
-                "git-env:",
-                env!("GIT_VERSION", "Missing GIT_VERSION envvar")
-            ),
-            args = ["--abbrev=40", "--always", "--dirty=-modified"] // always use full sha
-        );
-    };
-}

pageserver/Cargo.toml

@@ -3,32 +3,25 @@ name = "pageserver"
 version = "0.1.0"
 edition = "2021"
 
-[features]
-# It is simpler infra-wise to have failpoints enabled by default
-# It shouldn't affect perf in any way because failpoints
-# are not placed in hot code paths
-default = ["failpoints"]
-profiling = ["pprof"]
-failpoints = ["fail/failpoints"]
-
 [dependencies]
+bookfile = { git = "https://github.com/neondatabase/bookfile.git", branch="main" }
 chrono = "0.4.19"
 rand = "0.8.3"
 regex = "1.4.5"
 bytes = { version = "1.0.1", features = ['serde'] }
 byteorder = "1.4.3"
 futures = "0.3.13"
-hex = "0.4.3"
 hyper = "0.14"
-itertools = "0.10.3"
 lazy_static = "1.4.0"
+log = "0.4.14"
 clap = "3.0"
 daemonize = "0.4.1"
-tokio = { version = "1.17", features = ["process", "sync", "macros", "fs", "rt", "io-util", "time"] }
-postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+itertools = "0.10.3"
+tokio = { version = "1.11", features = ["process", "sync", "macros", "fs", "rt", "io-util", "time"] }
+postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
+postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
+postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
+tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
 tokio-stream = "0.1.8"
 anyhow = { version = "1.0", features = ["backtrace"] }
 crc32c = "0.6.0"
@@ -38,27 +31,27 @@ humantime = "2.1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 serde_with = "1.12.0"
-humantime-serde = "1.1.1"
-
-pprof = { git = "https://github.com/neondatabase/pprof-rs.git", branch = "wallclock-profiling", features = ["flamegraph"], optional = true }
 
 toml_edit = { version = "0.13", features = ["easy"] }
 scopeguard = "1.1.0"
+async-trait = "0.1"
 const_format = "0.2.21"
 tracing = "0.1.27"
+tracing-futures = "0.2"
 signal-hook = "0.3.10"
 url = "2"
 nix = "0.23"
 once_cell = "1.8.0"
 crossbeam-utils = "0.8.5"
 fail = "0.5.0"
-git-version = "0.3.5"
 
-postgres_ffi = { path = "../libs/postgres_ffi" }
-metrics = { path = "../libs/metrics" }
-utils = { path = "../libs/utils" }
-remote_storage = { path = "../libs/remote_storage" }
-workspace_hack = { version = "0.1", path = "../workspace_hack" }
+rust-s3 = { version = "0.28", default-features = false, features = ["no-verify-ssl", "tokio-rustls-tls"] }
+async-compression = {version = "0.3", features = ["zstd", "tokio"]}
+
+postgres_ffi = { path = "../postgres_ffi" }
+zenith_metrics = { path = "../zenith_metrics" }
+zenith_utils = { path = "../zenith_utils" }
+workspace_hack = { path = "../workspace_hack" }
 
 [dev-dependencies]
 hex-literal = "0.3"

pageserver/README.md

@@ -13,7 +13,7 @@ keeps track of WAL records which are not synced to S3 yet.
 
 The Page Server consists of multiple threads that operate on a shared
 repository of page versions:
-```
+
                                            | WAL
                                            V
                                    +--------------+
@@ -46,7 +46,7 @@ Legend:
 
 --->   Data flow
 <---
-```
+
 
 Page Service
 ------------
@@ -135,7 +135,7 @@ The backup service is disabled by default and can be enabled to interact with a
 
 CLI examples:
 * Local FS: `${PAGESERVER_BIN} -c "remote_storage={local_path='/some/local/path/'}"`
-* AWS S3  : `env AWS_ACCESS_KEY_ID='SOMEKEYAAAAASADSAH*#' AWS_SECRET_ACCESS_KEY='SOMEsEcReTsd292v' ${PAGESERVER_BIN} -c "remote_storage={bucket_name='some-sample-bucket',bucket_region='eu-north-1', prefix_in_bucket='/test_prefix/'}"`
+* AWS S3  : `${PAGESERVER_BIN} -c "remote_storage={bucket_name='some-sample-bucket',bucket_region='eu-north-1', prefix_in_bucket='/test_prefix/',access_key_id='SOMEKEYAAAAASADSAH*#',secret_access_key='SOMEsEcReTsd292v'}"`
 
 For Amazon AWS S3, a key id and secret access key could be located in `~/.aws/credentials` if awscli was ever configured to work with the desired bucket, on the AWS Settings page for a certain user. Also note, that the bucket names does not contain any protocols when used on AWS.
 For local S3 installations, refer to the their documentation for name format and credentials.
@@ -155,9 +155,11 @@ or
 bucket_name = 'some-sample-bucket'
 bucket_region = 'eu-north-1'
 prefix_in_bucket = '/test_prefix/'
+access_key_id = 'SOMEKEYAAAAASADSAH*#'
+secret_access_key = 'SOMEsEcReTsd292v'
 ```
 
-`AWS_SECRET_ACCESS_KEY` and `AWS_ACCESS_KEY_ID` env variables can be used to specify the S3 credentials if needed.
+Also, `AWS_SECRET_ACCESS_KEY` and `AWS_ACCESS_KEY_ID` variables can be used to specify the credentials instead of any of the ways above.
 
 TODO: Sharding
 --------------------

pageserver/src/basebackup.rs

@@ -10,34 +10,36 @@
 //! This module is responsible for creation of such tarball
 //! from data stored in object storage.
 //!
-use anyhow::{anyhow, ensure, Context, Result};
+use anyhow::{anyhow, Context, Result};
 use bytes::{BufMut, BytesMut};
+use itertools::Itertools;
+use log::*;
 use std::fmt::Write as FmtWrite;
 use std::io;
 use std::io::Write;
 use std::sync::Arc;
 use std::time::SystemTime;
 use tar::{Builder, EntryType, Header};
-use tracing::*;
 
-use crate::reltag::SlruKind;
+use crate::relish::*;
 use crate::repository::Timeline;
-use crate::DatadirTimelineImpl;
 use postgres_ffi::xlog_utils::*;
 use postgres_ffi::*;
-use utils::lsn::Lsn;
+use zenith_utils::lsn::Lsn;
 
 /// This is short-living object only for the time of tarball creation,
 /// created mostly to avoid passing a lot of parameters between various functions
 /// used for constructing tarball.
 pub struct Basebackup<'a> {
     ar: Builder<&'a mut dyn Write>,
-    timeline: &'a Arc<DatadirTimelineImpl>,
+    timeline: &'a Arc<dyn Timeline>,
     pub lsn: Lsn,
     prev_record_lsn: Lsn,
+    full_backup: bool,
 }
 
-// Create basebackup with non-rel data in it. Omit relational data.
+// Create basebackup with non-rel data in it.
+// Only include relational data if 'full_backup' is true.
 //
 // Currently we use empty lsn in two cases:
 //  * During the basebackup right after timeline creation
@@ -47,8 +49,10 @@ pub struct Basebackup<'a> {
 impl<'a> Basebackup<'a> {
     pub fn new(
         write: &'a mut dyn Write,
-        timeline: &'a Arc<DatadirTimelineImpl>,
+        timeline: &'a Arc<dyn Timeline>,
         req_lsn: Option<Lsn>,
+        prev_lsn: Option<Lsn>,
+        full_backup: bool,
     ) -> Result<Basebackup<'a>> {
         // Compute postgres doesn't have any previous WAL files, but the first
         // record that it's going to write needs to include the LSN of the
@@ -65,14 +69,13 @@ impl<'a> Basebackup<'a> {
         // prev_lsn to Lsn(0) if we cannot provide the correct value.
         let (backup_prev, backup_lsn) = if let Some(req_lsn) = req_lsn {
             // Backup was requested at a particular LSN. Wait for it to arrive.
-            info!("waiting for {}", req_lsn);
-            timeline.tline.wait_lsn(req_lsn)?;
+            timeline.wait_lsn(req_lsn)?;
 
             // If the requested point is the end of the timeline, we can
             // provide prev_lsn. (get_last_record_rlsn() might return it as
             // zero, though, if no WAL has been generated on this timeline
             // yet.)
-            let end_of_timeline = timeline.tline.get_last_record_rlsn();
+            let end_of_timeline = timeline.get_last_record_rlsn();
             if req_lsn == end_of_timeline.last {
                 (end_of_timeline.prev, req_lsn)
             } else {
@@ -80,20 +83,31 @@ impl<'a> Basebackup<'a> {
             }
         } else {
             // Backup was requested at end of the timeline.
-            let end_of_timeline = timeline.tline.get_last_record_rlsn();
+            let end_of_timeline = timeline.get_last_record_rlsn();
             (end_of_timeline.prev, end_of_timeline.last)
         };
 
+        // Consolidate the derived and the provided prev_lsn values
+        let prev_lsn = if let Some(provided_prev_lsn) = prev_lsn {
+            if backup_prev != Lsn(0) {
+                anyhow::ensure!(backup_prev == provided_prev_lsn)
+            }
+            provided_prev_lsn
+        } else {
+            backup_prev
+        };
+
         info!(
-            "taking basebackup lsn={}, prev_lsn={}",
-            backup_lsn, backup_prev
+            "taking basebackup lsn={}, prev_lsn={} (full_backup={})",
+            backup_lsn, prev_lsn, full_backup
         );
 
         Ok(Basebackup {
             ar: Builder::new(write),
             timeline,
             lsn: backup_lsn,
-            prev_record_lsn: backup_prev,
+            prev_record_lsn: prev_lsn,
+            full_backup,
         })
     }
 
@@ -117,22 +131,27 @@ impl<'a> Basebackup<'a> {
         }
 
         // Gather non-relational files from object storage pages.
-        for kind in [
-            SlruKind::Clog,
-            SlruKind::MultiXactOffsets,
-            SlruKind::MultiXactMembers,
-        ] {
-            for segno in self.timeline.list_slru_segments(kind, self.lsn)? {
-                self.add_slru_segment(kind, segno)?;
+        for obj in self.timeline.list_nonrels(self.lsn)? {
+            match obj {
+                RelishTag::Slru { slru, segno } => {
+                    self.add_slru_segment(slru, segno)?;
+                }
+                RelishTag::FileNodeMap { spcnode, dbnode } => {
+                    self.add_relmap_file(spcnode, dbnode)?;
+                }
+                RelishTag::TwoPhase { xid } => {
+                    self.add_twophase_file(xid)?;
+                }
+                _ => {}
             }
         }
 
-        // Create tablespace directories
-        for ((spcnode, dbnode), has_relmap_file) in self.timeline.list_dbdirs(self.lsn)? {
-            self.add_dbdir(spcnode, dbnode, has_relmap_file)?;
-        }
-        for xid in self.timeline.list_twophase_files(self.lsn)? {
-            self.add_twophase_file(xid)?;
+        // Gather relational files if we are doing a full backup.
+        if self.full_backup {
+            let all_rels = self.timeline.list_rels(0, 0, self.lsn)?;
+            for rel in all_rels {
+                self.add_rel(rel)?;
+            }
         }
 
         // Generate pg_control and bootstrap WAL segment.
@@ -142,29 +161,79 @@ impl<'a> Basebackup<'a> {
         Ok(())
     }
 
+    fn add_rel(&mut self, rel: RelishTag) -> anyhow::Result<()> {
+        let tag = match rel {
+            RelishTag::Relation(tag) => tag,
+            _ => {
+                return Err(anyhow!("expected RelishTag::Rel, got {:?}", rel));
+            }
+        };
+
+        // Function that adds relation segment data to archive
+        let mut add_file = |segment_index, data: &Vec<u8>| -> anyhow::Result<()> {
+            let file_name = tag.to_segfile_name(segment_index as u32);
+            let header = new_tar_header(&file_name, data.len() as u64)?;
+            self.ar.append(&header, data.as_slice())?;
+            Ok(())
+        };
+
+        let nblocks = match self.timeline.get_relish_size(rel, self.lsn)? {
+            Some(nblocks) => nblocks,
+            None => {
+                warn!("rel {} is truncated in timeline", tag);
+                return Ok(());
+            }
+        };
+
+        // If the relation is empty, create an empty file
+        if nblocks == 0 {
+            add_file(0, &vec![])?;
+            return Ok(());
+        }
+
+        // Add a file for each chunk of blocks (aka segment)
+        let chunks = (0..nblocks).chunks(pg_constants::RELSEG_SIZE as usize);
+        for (seg, blocks) in chunks.into_iter().enumerate() {
+            let mut segment_data: Vec<u8> = vec![];
+            for blknum in blocks {
+                let img = self.timeline.get_page_at_lsn(rel, blknum, self.lsn)?;
+                segment_data.extend_from_slice(&img[..]);
+            }
+
+            add_file(seg, &segment_data)?;
+        }
+
+        Ok(())
+    }
+
     //
     // Generate SLRU segment files from repository.
     //
     fn add_slru_segment(&mut self, slru: SlruKind, segno: u32) -> anyhow::Result<()> {
-        let nblocks = self.timeline.get_slru_segment_size(slru, segno, self.lsn)?;
+        let seg_size = self
+            .timeline
+            .get_relish_size(RelishTag::Slru { slru, segno }, self.lsn)?;
+
+        if seg_size == None {
+            trace!(
+                "SLRU segment {}/{:>04X} was truncated",
+                slru.to_str(),
+                segno
+            );
+            return Ok(());
+        }
+
+        let nblocks = seg_size.unwrap();
 
         let mut slru_buf: Vec<u8> =
             Vec::with_capacity(nblocks as usize * pg_constants::BLCKSZ as usize);
         for blknum in 0..nblocks {
-            let img = self
-                .timeline
-                .get_slru_page_at_lsn(slru, segno, blknum, self.lsn)?;
+            let img =
+                self.timeline
+                    .get_page_at_lsn(RelishTag::Slru { slru, segno }, blknum, self.lsn)?;
+            assert!(img.len() == pg_constants::BLCKSZ as usize);
 
-            if slru == SlruKind::Clog {
-                ensure!(
-                    img.len() == pg_constants::BLCKSZ as usize
-                        || img.len() == pg_constants::BLCKSZ as usize + 8
-                );
-            } else {
-                ensure!(img.len() == pg_constants::BLCKSZ as usize);
-            }
-
-            slru_buf.extend_from_slice(&img[..pg_constants::BLCKSZ as usize]);
+            slru_buf.extend_from_slice(&img);
         }
 
         let segname = format!("{}/{:>04X}", slru.to_str(), segno);
@@ -176,26 +245,16 @@ impl<'a> Basebackup<'a> {
     }
 
     //
-    // Include database/tablespace directories.
+    // Extract pg_filenode.map files from repository
+    // Along with them also send PG_VERSION for each database.
     //
-    // Each directory contains a PG_VERSION file, and the default database
-    // directories also contain pg_filenode.map files.
-    //
-    fn add_dbdir(
-        &mut self,
-        spcnode: u32,
-        dbnode: u32,
-        has_relmap_file: bool,
-    ) -> anyhow::Result<()> {
-        let relmap_img = if has_relmap_file {
-            let img = self.timeline.get_relmap_file(spcnode, dbnode, self.lsn)?;
-            ensure!(img.len() == 512);
-            Some(img)
-        } else {
-            None
-        };
-
-        if spcnode == pg_constants::GLOBALTABLESPACE_OID {
+    fn add_relmap_file(&mut self, spcnode: u32, dbnode: u32) -> anyhow::Result<()> {
+        let img = self.timeline.get_page_at_lsn(
+            RelishTag::FileNodeMap { spcnode, dbnode },
+            0,
+            self.lsn,
+        )?;
+        let path = if spcnode == pg_constants::GLOBALTABLESPACE_OID {
             let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
             let header = new_tar_header("PG_VERSION", version_bytes.len() as u64)?;
             self.ar.append(&header, version_bytes)?;
@@ -203,51 +262,26 @@ impl<'a> Basebackup<'a> {
             let header = new_tar_header("global/PG_VERSION", version_bytes.len() as u64)?;
             self.ar.append(&header, version_bytes)?;
 
-            if let Some(img) = relmap_img {
-                // filenode map for global tablespace
-                let header = new_tar_header("global/pg_filenode.map", img.len() as u64)?;
-                self.ar.append(&header, &img[..])?;
-            } else {
-                warn!("global/pg_filenode.map is missing");
-            }
+            String::from("global/pg_filenode.map") // filenode map for global tablespace
         } else {
-            // User defined tablespaces are not supported. However, as
-            // a special case, if a tablespace/db directory is
-            // completely empty, we can leave it out altogether. This
-            // makes taking a base backup after the 'tablespace'
-            // regression test pass, because the test drops the
-            // created tablespaces after the tests.
-            //
-            // FIXME: this wouldn't be necessary, if we handled
-            // XLOG_TBLSPC_DROP records. But we probably should just
-            // throw an error on CREATE TABLESPACE in the first place.
-            if !has_relmap_file
-                && self
-                    .timeline
-                    .list_rels(spcnode, dbnode, self.lsn)?
-                    .is_empty()
-            {
-                return Ok(());
-            }
             // User defined tablespaces are not supported
-            ensure!(spcnode == pg_constants::DEFAULTTABLESPACE_OID);
+            assert!(spcnode == pg_constants::DEFAULTTABLESPACE_OID);
 
             // Append dir path for each database
             let path = format!("base/{}", dbnode);
             let header = new_tar_header_dir(&path)?;
             self.ar.append(&header, &mut io::empty())?;
 
-            if let Some(img) = relmap_img {
-                let dst_path = format!("base/{}/PG_VERSION", dbnode);
-                let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
-                let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
-                self.ar.append(&header, version_bytes)?;
+            let dst_path = format!("base/{}/PG_VERSION", dbnode);
+            let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
+            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
+            self.ar.append(&header, version_bytes)?;
 
-                let relmap_path = format!("base/{}/pg_filenode.map", dbnode);
-                let header = new_tar_header(&relmap_path, img.len() as u64)?;
-                self.ar.append(&header, &img[..])?;
-            }
+            format!("base/{}/pg_filenode.map", dbnode)
         };
+        assert!(img.len() == 512);
+        let header = new_tar_header(&path, img.len() as u64)?;
+        self.ar.append(&header, &img[..])?;
         Ok(())
     }
 
@@ -255,7 +289,9 @@ impl<'a> Basebackup<'a> {
     // Extract twophase state files
     //
     fn add_twophase_file(&mut self, xid: TransactionId) -> anyhow::Result<()> {
-        let img = self.timeline.get_twophase_file(xid, self.lsn)?;
+        let img = self
+            .timeline
+            .get_page_at_lsn(RelishTag::TwoPhase { xid }, 0, self.lsn)?;
 
         let mut buf = BytesMut::new();
         buf.extend_from_slice(&img[..]);
@@ -275,11 +311,11 @@ impl<'a> Basebackup<'a> {
     fn add_pgcontrol_file(&mut self) -> anyhow::Result<()> {
         let checkpoint_bytes = self
             .timeline
-            .get_checkpoint(self.lsn)
+            .get_page_at_lsn(RelishTag::Checkpoint, 0, self.lsn)
             .context("failed to get checkpoint bytes")?;
         let pg_control_bytes = self
             .timeline
-            .get_control_file(self.lsn)
+            .get_page_at_lsn(RelishTag::ControlFile, 0, self.lsn)
             .context("failed get control bytes")?;
         let mut pg_control = ControlFileData::decode(&pg_control_bytes)?;
         let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
@@ -300,7 +336,7 @@ impl<'a> Basebackup<'a> {
         // add zenith.signal file
         let mut zenith_signal = String::new();
         if self.prev_record_lsn == Lsn(0) {
-            if self.lsn == self.timeline.tline.get_ancestor_lsn() {
+            if self.lsn == self.timeline.get_ancestor_lsn() {
                 write!(zenith_signal, "PREV LSN: none")?;
             } else {
                 write!(zenith_signal, "PREV LSN: invalid")?;
@@ -323,9 +359,8 @@ impl<'a> Basebackup<'a> {
         let wal_file_name = XLogFileName(PG_TLI, segno, pg_constants::WAL_SEGMENT_SIZE);
         let wal_file_path = format!("pg_wal/{}", wal_file_name);
         let header = new_tar_header(&wal_file_path, pg_constants::WAL_SEGMENT_SIZE as u64)?;
-        let wal_seg = generate_wal_segment(segno, pg_control.system_identifier)
-            .map_err(|e| anyhow!(e).context("Failed generating wal segment"))?;
-        ensure!(wal_seg.len() == pg_constants::WAL_SEGMENT_SIZE);
+        let wal_seg = generate_wal_segment(segno, pg_control.system_identifier);
+        assert!(wal_seg.len() == pg_constants::WAL_SEGMENT_SIZE);
         self.ar.append(&header, &wal_seg[..])?;
         Ok(())
     }

pageserver/src/bin/dump_layerfile.rs

@@ -4,12 +4,9 @@
 use anyhow::Result;
 use clap::{App, Arg};
 use pageserver::layered_repository::dump_layerfile_from_path;
-use pageserver::page_cache;
 use pageserver::virtual_file;
 use std::path::PathBuf;
-use utils::project_git_version;
-
-project_git_version!(GIT_VERSION);
+use zenith_utils::GIT_VERSION;
 
 fn main() -> Result<()> {
     let arg_matches = App::new("Zenith dump_layerfile utility")
@@ -27,9 +24,8 @@ fn main() -> Result<()> {
 
     // Basic initialization of things that don't change after startup
     virtual_file::init(10);
-    page_cache::init(100);
 
-    dump_layerfile_from_path(&path, true)?;
+    dump_layerfile_from_path(&path)?;
 
     Ok(())
 }

pageserver/src/bin/pageserver.rs

@@ -2,46 +2,39 @@
 
 use std::{env, path::Path, str::FromStr};
 use tracing::*;
+use zenith_utils::{
+    auth::JwtAuth,
+    logging,
+    postgres_backend::AuthType,
+    tcp_listener,
+    zid::{ZTenantId, ZTimelineId},
+    GIT_VERSION,
+};
 
 use anyhow::{bail, Context, Result};
 
 use clap::{App, Arg};
 use daemonize::Daemonize;
 
-use fail::FailScenario;
 use pageserver::{
     config::{defaults::*, PageServerConf},
-    http, page_cache, page_service, profiling, tenant_mgr, thread_mgr,
+    http, page_cache, page_service,
+    remote_storage::{self, SyncStartupData},
+    repository::TimelineSyncStatusUpdate,
+    tenant_mgr, thread_mgr,
     thread_mgr::ThreadKind,
     timelines, virtual_file, LOG_FILE_NAME,
 };
-use utils::{
-    auth::JwtAuth,
-    http::endpoint,
-    logging,
-    postgres_backend::AuthType,
-    project_git_version,
-    shutdown::exit_now,
-    signals::{self, Signal},
-    tcp_listener,
-    zid::{ZTenantId, ZTimelineId},
-};
+use zenith_utils::http::endpoint;
+use zenith_utils::postgres_backend;
+use zenith_utils::shutdown::exit_now;
+use zenith_utils::signals::{self, Signal};
 
-project_git_version!(GIT_VERSION);
-
-fn version() -> String {
-    format!(
-        "{GIT_VERSION} profiling:{} failpoints:{}",
-        cfg!(feature = "profiling"),
-        fail::has_failpoints()
-    )
-}
-
-fn main() -> anyhow::Result<()> {
-    metrics::set_common_metrics_prefix("pageserver");
+fn main() -> Result<()> {
+    zenith_metrics::set_common_metrics_prefix("pageserver");
     let arg_matches = App::new("Zenith page server")
         .about("Materializes WAL stream to pages and serves them to the postgres")
-        .version(&*version())
+        .version(GIT_VERSION)
         .arg(
             Arg::new("daemonize")
                 .short('d')
@@ -86,23 +79,8 @@ fn main() -> anyhow::Result<()> {
                 .help("Additional configuration overrides of the ones from the toml config file (or new ones to add there).
                 Any option has to be a valid toml document, example: `-c=\"foo='hey'\"` `-c=\"foo={value=1}\"`"),
         )
-        .arg(
-            Arg::new("enabled-features")
-                .long("enabled-features")
-                .takes_value(false)
-                .help("Show enabled compile time features"),
-        )
         .get_matches();
 
-    if arg_matches.is_present("enabled-features") {
-        let features: &[&str] = &[
-            #[cfg(feature = "failpoints")]
-            "failpoints",
-        ];
-        println!("{{\"features\": {features:?} }}");
-        return Ok(());
-    }
-
     let workdir = Path::new(arg_matches.value_of("workdir").unwrap_or(".zenith"));
     let workdir = workdir
         .canonicalize()
@@ -138,7 +116,7 @@ fn main() -> anyhow::Result<()> {
         // We're initializing the repo, so there's no config file yet
         DEFAULT_CONFIG_FILE
             .parse::<toml_edit::Document>()
-            .context("could not parse built-in config file")?
+            .expect("could not parse built-in config file")
     } else {
         // Supplement the CLI arguments with the config file
         let cfg_file_contents = std::fs::read_to_string(&cfg_file_path)
@@ -183,17 +161,10 @@ fn main() -> anyhow::Result<()> {
     // as a ref.
     let conf: &'static PageServerConf = Box::leak(Box::new(conf));
 
-    // If failpoints are used, terminate the whole pageserver process if they are hit.
-    let scenario = FailScenario::setup();
-    if fail::has_failpoints() {
-        std::panic::set_hook(Box::new(|_| {
-            std::process::exit(1);
-        }));
-    }
-
     // Basic initialization of things that don't change after startup
     virtual_file::init(conf.max_file_descriptors);
-    page_cache::init(conf.page_cache_size);
+
+    page_cache::init(conf);
 
     // Create repo and exit if init was requested
     if init {
@@ -206,19 +177,17 @@ fn main() -> anyhow::Result<()> {
                 cfg_file_path.display()
             )
         })?;
+        Ok(())
     } else {
-        start_pageserver(conf, daemonize).context("Failed to start pageserver")?;
+        start_pageserver(conf, daemonize).context("Failed to start pageserver")
     }
-
-    scenario.teardown();
-    Ok(())
 }
 
 fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()> {
     // Initialize logger
     let log_file = logging::init(LOG_FILE_NAME, daemonize)?;
 
-    info!("version: {GIT_VERSION}");
+    info!("version: {}", GIT_VERSION);
 
     // TODO: Check that it looks like a valid repository before going further
 
@@ -241,9 +210,7 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
 
         // There shouldn't be any logging to stdin/stdout. Redirect it to the main log so
         // that we will see any accidental manual fprintf's or backtraces.
-        let stdout = log_file
-            .try_clone()
-            .with_context(|| format!("Failed to clone log file '{:?}'", log_file))?;
+        let stdout = log_file.try_clone().unwrap();
         let stderr = log_file;
 
         let daemonize = Daemonize::new()
@@ -264,8 +231,46 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
 
     let signals = signals::install_shutdown_handlers()?;
 
-    // start profiler (if enabled)
-    let profiler_guard = profiling::init_profiler(conf);
+    // Initialize repositories with locally available timelines.
+    // Timelines that are only partially available locally (remote storage has more data than this pageserver)
+    // are scheduled for download and added to the repository once download is completed.
+    let SyncStartupData {
+        remote_index,
+        local_timeline_init_statuses,
+    } = remote_storage::start_local_timeline_sync(conf)
+        .context("Failed to set up local files sync with external storage")?;
+
+    for (tenant_id, local_timeline_init_statuses) in local_timeline_init_statuses {
+        // initialize local tenant
+        let repo = tenant_mgr::load_local_repo(conf, tenant_id, &remote_index);
+        for (timeline_id, init_status) in local_timeline_init_statuses {
+            match init_status {
+                remote_storage::LocalTimelineInitStatus::LocallyComplete => {
+                    debug!("timeline {} for tenant {} is locally complete, registering it in repository", tenant_id, timeline_id);
+                    // Lets fail here loudly to be on the safe side.
+                    // XXX: It may be a better api to actually distinguish between repository startup
+                    //   and processing of newly downloaded timelines.
+                    repo.apply_timeline_remote_sync_status_update(
+                        timeline_id,
+                        TimelineSyncStatusUpdate::Downloaded,
+                    )
+                    .with_context(|| {
+                        format!(
+                            "Failed to bootstrap timeline {} for tenant {}",
+                            timeline_id, tenant_id
+                        )
+                    })?
+                }
+                remote_storage::LocalTimelineInitStatus::NeedsSync => {
+                    debug!(
+                        "timeline {} for tenant {} needs sync, \
+                         so skipped for adding into repository until sync is finished",
+                        tenant_id, timeline_id
+                    );
+                }
+            }
+        }
+    }
 
     // initialize authentication for incoming connections
     let auth = match &conf.auth_type {
@@ -278,8 +283,6 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
     };
     info!("Using auth: {:#?}", conf.auth_type);
 
-    let remote_index = tenant_mgr::init_tenant_mgr(conf)?;
-
     // Spawn a new thread for the http endpoint
     // bind before launching separate thread so the error reported before startup exits
     let auth_cloned = auth.clone();
@@ -288,9 +291,8 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
         None,
         None,
         "http_endpoint_thread",
-        true,
         move || {
-            let router = http::make_router(conf, auth_cloned, remote_index)?;
+            let router = http::make_router(conf, auth_cloned, remote_index);
             endpoint::serve_thread_main(router, http_listener, thread_mgr::shutdown_watcher())
         },
     )?;
@@ -302,7 +304,6 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
         None,
         None,
         "libpq endpoint thread",
-        true,
         move || page_service::thread_main(conf, auth, pageserver_listener, conf.auth_type),
     )?;
 
@@ -312,7 +313,6 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
                 "Got {}. Terminating in immediate shutdown mode",
                 signal.name()
             );
-            profiling::exit_profiler(conf, &profiler_guard);
             std::process::exit(111);
         }
 
@@ -321,9 +321,38 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
                 "Got {}. Terminating gracefully in fast shutdown mode",
                 signal.name()
             );
-            profiling::exit_profiler(conf, &profiler_guard);
-            pageserver::shutdown_pageserver(0);
+            shutdown_pageserver();
             unreachable!()
         }
     })
 }
+
+fn shutdown_pageserver() {
+    // Shut down the libpq endpoint thread. This prevents new connections from
+    // being accepted.
+    thread_mgr::shutdown_threads(Some(ThreadKind::LibpqEndpointListener), None, None);
+
+    // Shut down any page service threads.
+    postgres_backend::set_pgbackend_shutdown_requested();
+    thread_mgr::shutdown_threads(Some(ThreadKind::PageRequestHandler), None, None);
+
+    // Shut down all the tenants. This flushes everything to disk and kills
+    // the checkpoint and GC threads.
+    tenant_mgr::shutdown_all_tenants();
+
+    // Stop syncing with remote storage.
+    //
+    // FIXME: Does this wait for the sync thread to finish syncing what's queued up?
+    // Should it?
+    thread_mgr::shutdown_threads(Some(ThreadKind::StorageSync), None, None);
+
+    // Shut down the HTTP endpoint last, so that you can still check the server's
+    // status while it's shutting down.
+    thread_mgr::shutdown_threads(Some(ThreadKind::HttpEndpointListener), None, None);
+
+    // There should be nothing left, but let's be sure
+    thread_mgr::shutdown_threads(None, None, None);
+
+    info!("Shut down successfully completed");
+    std::process::exit(0);
+}

pageserver/src/bin/pageserver_zst.rs

@@ -0,0 +1,334 @@
+//! A CLI helper to deal with remote storage (S3, usually) blobs as archives.
+//! See [`compression`] for more details about the archives.
+
+use std::{collections::BTreeSet, path::Path};
+
+use anyhow::{bail, ensure, Context};
+use clap::{App, Arg};
+use pageserver::{
+    layered_repository::metadata::{TimelineMetadata, METADATA_FILE_NAME},
+    remote_storage::compression,
+};
+use tokio::{fs, io};
+use zenith_utils::GIT_VERSION;
+
+const LIST_SUBCOMMAND: &str = "list";
+const ARCHIVE_ARG_NAME: &str = "archive";
+
+const EXTRACT_SUBCOMMAND: &str = "extract";
+const TARGET_DIRECTORY_ARG_NAME: &str = "target_directory";
+
+const CREATE_SUBCOMMAND: &str = "create";
+const SOURCE_DIRECTORY_ARG_NAME: &str = "source_directory";
+
+#[tokio::main(flavor = "current_thread")]
+async fn main() -> anyhow::Result<()> {
+    let arg_matches = App::new("pageserver zst blob [un]compressor utility")
+        .version(GIT_VERSION)
+        .subcommands(vec![
+            App::new(LIST_SUBCOMMAND)
+                .about("List the archive contents")
+                .arg(
+                    Arg::new(ARCHIVE_ARG_NAME)
+                        .required(true)
+                        .takes_value(true)
+                        .help("An archive to list the contents of"),
+                ),
+            App::new(EXTRACT_SUBCOMMAND)
+                .about("Extracts the archive into the directory")
+                .arg(
+                    Arg::new(ARCHIVE_ARG_NAME)
+                        .required(true)
+                        .takes_value(true)
+                        .help("An archive to extract"),
+                )
+                .arg(
+                    Arg::new(TARGET_DIRECTORY_ARG_NAME)
+                        .required(false)
+                        .takes_value(true)
+                        .help("A directory to extract the archive into. Optional, will use the current directory if not specified"),
+                ),
+            App::new(CREATE_SUBCOMMAND)
+                .about("Creates an archive with the contents of a directory (only the first level files are taken, metadata file has to be present in the same directory)")
+                .arg(
+                    Arg::new(SOURCE_DIRECTORY_ARG_NAME)
+                        .required(true)
+                        .takes_value(true)
+                        .help("A directory to use for creating the archive"),
+                )
+                .arg(
+                    Arg::new(TARGET_DIRECTORY_ARG_NAME)
+                        .required(false)
+                        .takes_value(true)
+                        .help("A directory to create the archive in. Optional, will use the current directory if not specified"),
+                ),
+        ])
+        .get_matches();
+
+    let subcommand_name = match arg_matches.subcommand_name() {
+        Some(name) => name,
+        None => bail!("No subcommand specified"),
+    };
+
+    let subcommand_matches = match arg_matches.subcommand_matches(subcommand_name) {
+        Some(matches) => matches,
+        None => bail!(
+            "No subcommand arguments were recognized for subcommand '{}'",
+            subcommand_name
+        ),
+    };
+
+    let target_dir = Path::new(
+        subcommand_matches
+            .value_of(TARGET_DIRECTORY_ARG_NAME)
+            .unwrap_or("./"),
+    );
+
+    match subcommand_name {
+        LIST_SUBCOMMAND => {
+            let archive = match subcommand_matches.value_of(ARCHIVE_ARG_NAME) {
+                Some(archive) => Path::new(archive),
+                None => bail!("No '{}' argument is specified", ARCHIVE_ARG_NAME),
+            };
+            list_archive(archive).await
+        }
+        EXTRACT_SUBCOMMAND => {
+            let archive = match subcommand_matches.value_of(ARCHIVE_ARG_NAME) {
+                Some(archive) => Path::new(archive),
+                None => bail!("No '{}' argument is specified", ARCHIVE_ARG_NAME),
+            };
+            extract_archive(archive, target_dir).await
+        }
+        CREATE_SUBCOMMAND => {
+            let source_dir = match subcommand_matches.value_of(SOURCE_DIRECTORY_ARG_NAME) {
+                Some(source) => Path::new(source),
+                None => bail!("No '{}' argument is specified", SOURCE_DIRECTORY_ARG_NAME),
+            };
+            create_archive(source_dir, target_dir).await
+        }
+        unknown => bail!("Unknown subcommand {}", unknown),
+    }
+}
+
+async fn list_archive(archive: &Path) -> anyhow::Result<()> {
+    let archive = archive.canonicalize().with_context(|| {
+        format!(
+            "Failed to get the absolute path for the archive path '{}'",
+            archive.display()
+        )
+    })?;
+    ensure!(
+        archive.is_file(),
+        "Path '{}' is not an archive file",
+        archive.display()
+    );
+    println!("Listing an archive at path '{}'", archive.display());
+    let archive_name = match archive.file_name().and_then(|name| name.to_str()) {
+        Some(name) => name,
+        None => bail!(
+            "Failed to get the archive name from the path '{}'",
+            archive.display()
+        ),
+    };
+
+    let archive_bytes = fs::read(&archive)
+        .await
+        .context("Failed to read the archive bytes")?;
+
+    let header = compression::read_archive_header(archive_name, &mut archive_bytes.as_slice())
+        .await
+        .context("Failed to read the archive header")?;
+
+    let empty_path = Path::new("");
+    println!("-------------------------------");
+
+    let longest_path_in_archive = header
+        .files
+        .iter()
+        .filter_map(|file| Some(file.subpath.as_path(empty_path).to_str()?.len()))
+        .max()
+        .unwrap_or_default()
+        .max(METADATA_FILE_NAME.len());
+
+    for regular_file in &header.files {
+        println!(
+            "File: {:width$} uncompressed size: {} bytes",
+            regular_file.subpath.as_path(empty_path).display(),
+            regular_file.size,
+            width = longest_path_in_archive,
+        )
+    }
+    println!(
+        "File: {:width$} uncompressed size: {} bytes",
+        METADATA_FILE_NAME,
+        header.metadata_file_size,
+        width = longest_path_in_archive,
+    );
+    println!("-------------------------------");
+
+    Ok(())
+}
+
+async fn extract_archive(archive: &Path, target_dir: &Path) -> anyhow::Result<()> {
+    let archive = archive.canonicalize().with_context(|| {
+        format!(
+            "Failed to get the absolute path for the archive path '{}'",
+            archive.display()
+        )
+    })?;
+    ensure!(
+        archive.is_file(),
+        "Path '{}' is not an archive file",
+        archive.display()
+    );
+    let archive_name = match archive.file_name().and_then(|name| name.to_str()) {
+        Some(name) => name,
+        None => bail!(
+            "Failed to get the archive name from the path '{}'",
+            archive.display()
+        ),
+    };
+
+    if !target_dir.exists() {
+        fs::create_dir_all(target_dir).await.with_context(|| {
+            format!(
+                "Failed to create the target dir at path '{}'",
+                target_dir.display()
+            )
+        })?;
+    }
+    let target_dir = target_dir.canonicalize().with_context(|| {
+        format!(
+            "Failed to get the absolute path for the target dir path '{}'",
+            target_dir.display()
+        )
+    })?;
+    ensure!(
+        target_dir.is_dir(),
+        "Path '{}' is not a directory",
+        target_dir.display()
+    );
+    let mut dir_contents = fs::read_dir(&target_dir)
+        .await
+        .context("Failed to list the target directory contents")?;
+    let dir_entry = dir_contents
+        .next_entry()
+        .await
+        .context("Failed to list the target directory contents")?;
+    ensure!(
+        dir_entry.is_none(),
+        "Target directory '{}' is not empty",
+        target_dir.display()
+    );
+
+    println!(
+        "Extracting an archive at path '{}' into directory '{}'",
+        archive.display(),
+        target_dir.display()
+    );
+
+    let mut archive_file = fs::File::open(&archive).await.with_context(|| {
+        format!(
+            "Failed to get the archive name from the path '{}'",
+            archive.display()
+        )
+    })?;
+    let header = compression::read_archive_header(archive_name, &mut archive_file)
+        .await
+        .context("Failed to read the archive header")?;
+    compression::uncompress_with_header(&BTreeSet::new(), &target_dir, header, &mut archive_file)
+        .await
+        .context("Failed to extract the archive")
+}
+
+async fn create_archive(source_dir: &Path, target_dir: &Path) -> anyhow::Result<()> {
+    let source_dir = source_dir.canonicalize().with_context(|| {
+        format!(
+            "Failed to get the absolute path for the source dir path '{}'",
+            source_dir.display()
+        )
+    })?;
+    ensure!(
+        source_dir.is_dir(),
+        "Path '{}' is not a directory",
+        source_dir.display()
+    );
+
+    if !target_dir.exists() {
+        fs::create_dir_all(target_dir).await.with_context(|| {
+            format!(
+                "Failed to create the target dir at path '{}'",
+                target_dir.display()
+            )
+        })?;
+    }
+    let target_dir = target_dir.canonicalize().with_context(|| {
+        format!(
+            "Failed to get the absolute path for the target dir path '{}'",
+            target_dir.display()
+        )
+    })?;
+    ensure!(
+        target_dir.is_dir(),
+        "Path '{}' is not a directory",
+        target_dir.display()
+    );
+
+    println!(
+        "Compressing directory '{}' and creating resulting archive in directory '{}'",
+        source_dir.display(),
+        target_dir.display()
+    );
+
+    let mut metadata_file_contents = None;
+    let mut files_co_archive = Vec::new();
+
+    let mut source_dir_contents = fs::read_dir(&source_dir)
+        .await
+        .context("Failed to read the source directory contents")?;
+
+    while let Some(source_dir_entry) = source_dir_contents
+        .next_entry()
+        .await
+        .context("Failed to read a source dir entry")?
+    {
+        let entry_path = source_dir_entry.path();
+        if entry_path.is_file() {
+            if entry_path.file_name().and_then(|name| name.to_str()) == Some(METADATA_FILE_NAME) {
+                let metadata_bytes = fs::read(entry_path)
+                    .await
+                    .context("Failed to read metata file bytes in the source dir")?;
+                metadata_file_contents = Some(
+                    TimelineMetadata::from_bytes(&metadata_bytes)
+                        .context("Failed to parse metata file contents in the source dir")?,
+                );
+            } else {
+                files_co_archive.push(entry_path);
+            }
+        }
+    }
+
+    let metadata = match metadata_file_contents {
+        Some(metadata) => metadata,
+        None => bail!(
+            "No metadata file found in the source dir '{}', cannot create the archive",
+            source_dir.display()
+        ),
+    };
+
+    let _ = compression::archive_files_as_stream(
+        &source_dir,
+        files_co_archive.iter(),
+        &metadata,
+        move |mut archive_streamer, archive_name| async move {
+            let archive_target = target_dir.join(&archive_name);
+            let mut archive_file = fs::File::create(&archive_target).await?;
+            io::copy(&mut archive_streamer, &mut archive_file).await?;
+            Ok(archive_target)
+        },
+    )
+    .await
+    .context("Failed to create an archive")?;
+
+    Ok(())
+}

pageserver/src/bin/update_metadata.rs

@@ -6,9 +6,8 @@ use clap::{App, Arg};
 use pageserver::layered_repository::metadata::TimelineMetadata;
 use std::path::PathBuf;
 use std::str::FromStr;
-use utils::{lsn::Lsn, project_git_version};
-
-project_git_version!(GIT_VERSION);
+use zenith_utils::lsn::Lsn;
+use zenith_utils::GIT_VERSION;
 
 fn main() -> Result<()> {
     let arg_matches = App::new("Zenith update metadata utility")

pageserver/src/config.rs

@@ -4,25 +4,22 @@
 //! file, or on the command line.
 //! See also `settings.md` for better description on every parameter.
 
-use anyhow::{anyhow, bail, ensure, Context, Result};
-use remote_storage::{RemoteStorageConfig, RemoteStorageKind, S3Config};
+use anyhow::{bail, ensure, Context, Result};
+use toml_edit;
+use toml_edit::{Document, Item};
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId};
+
+use std::convert::TryInto;
 use std::env;
 use std::num::{NonZeroU32, NonZeroUsize};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::time::Duration;
-use toml_edit;
-use toml_edit::{Document, Item};
-use utils::{
-    postgres_backend::AuthType,
-    zid::{ZNodeId, ZTenantId, ZTimelineId},
-};
 
 use crate::layered_repository::TIMELINES_SEGMENT_NAME;
-use crate::tenant_config::{TenantConf, TenantConfOpt};
 
 pub mod defaults {
-    use crate::tenant_config::defaults::*;
     use const_format::formatcp;
 
     pub const DEFAULT_PG_LISTEN_PORT: u16 = 64000;
@@ -30,10 +27,21 @@ pub mod defaults {
     pub const DEFAULT_HTTP_LISTEN_PORT: u16 = 9898;
     pub const DEFAULT_HTTP_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_HTTP_LISTEN_PORT}");
 
+    // FIXME: This current value is very low. I would imagine something like 1 GB or 10 GB
+    // would be more appropriate. But a low value forces the code to be exercised more,
+    // which is good for now to trigger bugs.
+    pub const DEFAULT_CHECKPOINT_DISTANCE: u64 = 256 * 1024 * 1024;
+    pub const DEFAULT_CHECKPOINT_PERIOD: &str = "1 s";
+
+    pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
+    pub const DEFAULT_GC_PERIOD: &str = "100 s";
+
     pub const DEFAULT_WAIT_LSN_TIMEOUT: &str = "60 s";
     pub const DEFAULT_WAL_REDO_TIMEOUT: &str = "60 s";
 
     pub const DEFAULT_SUPERUSER: &str = "zenith_admin";
+    pub const DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNC: usize = 100;
+    pub const DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS: u32 = 10;
 
     pub const DEFAULT_PAGE_CACHE_SIZE: usize = 8192;
     pub const DEFAULT_MAX_FILE_DESCRIPTORS: usize = 100;
@@ -48,6 +56,12 @@ pub mod defaults {
 #listen_pg_addr = '{DEFAULT_PG_LISTEN_ADDR}'
 #listen_http_addr = '{DEFAULT_HTTP_LISTEN_ADDR}'
 
+#checkpoint_distance = {DEFAULT_CHECKPOINT_DISTANCE} # in bytes
+#checkpoint_period = '{DEFAULT_CHECKPOINT_PERIOD}'
+
+#gc_period = '{DEFAULT_GC_PERIOD}'
+#gc_horizon = {DEFAULT_GC_HORIZON}
+
 #wait_lsn_timeout = '{DEFAULT_WAIT_LSN_TIMEOUT}'
 #wal_redo_timeout = '{DEFAULT_WAL_REDO_TIMEOUT}'
 
@@ -56,17 +70,6 @@ pub mod defaults {
 # initial superuser role name to use when creating a new tenant
 #initial_superuser_name = '{DEFAULT_SUPERUSER}'
 
-# [tenant_config]
-#checkpoint_distance = {DEFAULT_CHECKPOINT_DISTANCE} # in bytes
-#compaction_target_size = {DEFAULT_COMPACTION_TARGET_SIZE} # in bytes
-#compaction_period = '{DEFAULT_COMPACTION_PERIOD}'
-#compaction_threshold = '{DEFAULT_COMPACTION_THRESHOLD}'
-
-#gc_period = '{DEFAULT_GC_PERIOD}'
-#gc_horizon = {DEFAULT_GC_HORIZON}
-#image_creation_threshold = {DEFAULT_IMAGE_CREATION_THRESHOLD}
-#pitr_interval = '{DEFAULT_PITR_INTERVAL}'
-
 # [remote_storage]
 
 "###
@@ -84,6 +87,15 @@ pub struct PageServerConf {
     /// Example (default): 127.0.0.1:9898
     pub listen_http_addr: String,
 
+    // Flush out an inmemory layer, if it's holding WAL older than this
+    // This puts a backstop on how much WAL needs to be re-digested if the
+    // page server crashes.
+    pub checkpoint_distance: u64,
+    pub checkpoint_period: Duration,
+
+    pub gc_horizon: u64,
+    pub gc_period: Duration,
+
     // Timeout when waiting for WAL receiver to catch up to an LSN given in a GetPage@LSN call.
     pub wait_lsn_timeout: Duration,
     // How long to wait for WAL redo to complete.
@@ -108,28 +120,6 @@ pub struct PageServerConf {
 
     pub auth_validation_public_key_path: Option<PathBuf>,
     pub remote_storage_config: Option<RemoteStorageConfig>,
-
-    pub profiling: ProfilingConfig,
-    pub default_tenant_conf: TenantConf,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum ProfilingConfig {
-    Disabled,
-    PageRequests,
-}
-
-impl FromStr for ProfilingConfig {
-    type Err = anyhow::Error;
-
-    fn from_str(s: &str) -> Result<ProfilingConfig, Self::Err> {
-        let result = match s {
-            "disabled"  => ProfilingConfig::Disabled,
-            "page_requests"  => ProfilingConfig::PageRequests,
-            _ => bail!("invalid value \"{s}\" for profiling option, valid values are \"disabled\" and \"page_requests\""),
-        };
-        Ok(result)
-    }
 }
 
 // use dedicated enum for builder to better indicate the intention
@@ -154,6 +144,12 @@ struct PageServerConfigBuilder {
 
     listen_http_addr: BuilderValue<String>,
 
+    checkpoint_distance: BuilderValue<u64>,
+    checkpoint_period: BuilderValue<Duration>,
+
+    gc_horizon: BuilderValue<u64>,
+    gc_period: BuilderValue<Duration>,
+
     wait_lsn_timeout: BuilderValue<Duration>,
     wal_redo_timeout: BuilderValue<Duration>,
 
@@ -173,8 +169,6 @@ struct PageServerConfigBuilder {
     remote_storage_config: BuilderValue<Option<RemoteStorageConfig>>,
 
     id: BuilderValue<ZNodeId>,
-
-    profiling: BuilderValue<ProfilingConfig>,
 }
 
 impl Default for PageServerConfigBuilder {
@@ -184,6 +178,12 @@ impl Default for PageServerConfigBuilder {
         Self {
             listen_pg_addr: Set(DEFAULT_PG_LISTEN_ADDR.to_string()),
             listen_http_addr: Set(DEFAULT_HTTP_LISTEN_ADDR.to_string()),
+            checkpoint_distance: Set(DEFAULT_CHECKPOINT_DISTANCE),
+            checkpoint_period: Set(humantime::parse_duration(DEFAULT_CHECKPOINT_PERIOD)
+                .expect("cannot parse default checkpoint period")),
+            gc_horizon: Set(DEFAULT_GC_HORIZON),
+            gc_period: Set(humantime::parse_duration(DEFAULT_GC_PERIOD)
+                .expect("cannot parse default gc period")),
             wait_lsn_timeout: Set(humantime::parse_duration(DEFAULT_WAIT_LSN_TIMEOUT)
                 .expect("cannot parse default wait lsn timeout")),
             wal_redo_timeout: Set(humantime::parse_duration(DEFAULT_WAL_REDO_TIMEOUT)
@@ -199,7 +199,6 @@ impl Default for PageServerConfigBuilder {
             auth_validation_public_key_path: Set(None),
             remote_storage_config: Set(None),
             id: NotSet,
-            profiling: Set(ProfilingConfig::Disabled),
         }
     }
 }
@@ -213,6 +212,22 @@ impl PageServerConfigBuilder {
         self.listen_http_addr = BuilderValue::Set(listen_http_addr)
     }
 
+    pub fn checkpoint_distance(&mut self, checkpoint_distance: u64) {
+        self.checkpoint_distance = BuilderValue::Set(checkpoint_distance)
+    }
+
+    pub fn checkpoint_period(&mut self, checkpoint_period: Duration) {
+        self.checkpoint_period = BuilderValue::Set(checkpoint_period)
+    }
+
+    pub fn gc_horizon(&mut self, gc_horizon: u64) {
+        self.gc_horizon = BuilderValue::Set(gc_horizon)
+    }
+
+    pub fn gc_period(&mut self, gc_period: Duration) {
+        self.gc_period = BuilderValue::Set(gc_period)
+    }
+
     pub fn wait_lsn_timeout(&mut self, wait_lsn_timeout: Duration) {
         self.wait_lsn_timeout = BuilderValue::Set(wait_lsn_timeout)
     }
@@ -260,50 +275,110 @@ impl PageServerConfigBuilder {
         self.id = BuilderValue::Set(node_id)
     }
 
-    pub fn profiling(&mut self, profiling: ProfilingConfig) {
-        self.profiling = BuilderValue::Set(profiling)
-    }
-
     pub fn build(self) -> Result<PageServerConf> {
         Ok(PageServerConf {
             listen_pg_addr: self
                 .listen_pg_addr
-                .ok_or(anyhow!("missing listen_pg_addr"))?,
+                .ok_or(anyhow::anyhow!("missing listen_pg_addr"))?,
             listen_http_addr: self
                 .listen_http_addr
-                .ok_or(anyhow!("missing listen_http_addr"))?,
+                .ok_or(anyhow::anyhow!("missing listen_http_addr"))?,
+            checkpoint_distance: self
+                .checkpoint_distance
+                .ok_or(anyhow::anyhow!("missing checkpoint_distance"))?,
+            checkpoint_period: self
+                .checkpoint_period
+                .ok_or(anyhow::anyhow!("missing checkpoint_period"))?,
+            gc_horizon: self
+                .gc_horizon
+                .ok_or(anyhow::anyhow!("missing gc_horizon"))?,
+            gc_period: self.gc_period.ok_or(anyhow::anyhow!("missing gc_period"))?,
             wait_lsn_timeout: self
                 .wait_lsn_timeout
-                .ok_or(anyhow!("missing wait_lsn_timeout"))?,
+                .ok_or(anyhow::anyhow!("missing wait_lsn_timeout"))?,
             wal_redo_timeout: self
                 .wal_redo_timeout
-                .ok_or(anyhow!("missing wal_redo_timeout"))?,
-            superuser: self.superuser.ok_or(anyhow!("missing superuser"))?,
+                .ok_or(anyhow::anyhow!("missing wal_redo_timeout"))?,
+            superuser: self.superuser.ok_or(anyhow::anyhow!("missing superuser"))?,
             page_cache_size: self
                 .page_cache_size
-                .ok_or(anyhow!("missing page_cache_size"))?,
+                .ok_or(anyhow::anyhow!("missing page_cache_size"))?,
             max_file_descriptors: self
                 .max_file_descriptors
-                .ok_or(anyhow!("missing max_file_descriptors"))?,
-            workdir: self.workdir.ok_or(anyhow!("missing workdir"))?,
+                .ok_or(anyhow::anyhow!("missing max_file_descriptors"))?,
+            workdir: self.workdir.ok_or(anyhow::anyhow!("missing workdir"))?,
             pg_distrib_dir: self
                 .pg_distrib_dir
-                .ok_or(anyhow!("missing pg_distrib_dir"))?,
-            auth_type: self.auth_type.ok_or(anyhow!("missing auth_type"))?,
+                .ok_or(anyhow::anyhow!("missing pg_distrib_dir"))?,
+            auth_type: self.auth_type.ok_or(anyhow::anyhow!("missing auth_type"))?,
             auth_validation_public_key_path: self
                 .auth_validation_public_key_path
-                .ok_or(anyhow!("missing auth_validation_public_key_path"))?,
+                .ok_or(anyhow::anyhow!("missing auth_validation_public_key_path"))?,
             remote_storage_config: self
                 .remote_storage_config
-                .ok_or(anyhow!("missing remote_storage_config"))?,
-            id: self.id.ok_or(anyhow!("missing id"))?,
-            profiling: self.profiling.ok_or(anyhow!("missing profiling"))?,
-            // TenantConf is handled separately
-            default_tenant_conf: TenantConf::default(),
+                .ok_or(anyhow::anyhow!("missing remote_storage_config"))?,
+            id: self.id.ok_or(anyhow::anyhow!("missing id"))?,
         })
     }
 }
 
+/// External backup storage configuration, enough for creating a client for that storage.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct RemoteStorageConfig {
+    /// Max allowed number of concurrent sync operations between pageserver and the remote storage.
+    pub max_concurrent_sync: NonZeroUsize,
+    /// Max allowed errors before the sync task is considered failed and evicted.
+    pub max_sync_errors: NonZeroU32,
+    /// The storage connection configuration.
+    pub storage: RemoteStorageKind,
+}
+
+/// A kind of a remote storage to connect to, with its connection configuration.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum RemoteStorageKind {
+    /// Storage based on local file system.
+    /// Specify a root folder to place all stored relish data into.
+    LocalFs(PathBuf),
+    /// AWS S3 based storage, storing all relishes into the root
+    /// of the S3 bucket from the config.
+    AwsS3(S3Config),
+}
+
+/// AWS S3 bucket coordinates and access credentials to manage the bucket contents (read and write).
+#[derive(Clone, PartialEq, Eq)]
+pub struct S3Config {
+    /// Name of the bucket to connect to.
+    pub bucket_name: String,
+    /// The region where the bucket is located at.
+    pub bucket_region: String,
+    /// A "subfolder" in the bucket, to use the same bucket separately by multiple pageservers at once.
+    pub prefix_in_bucket: Option<String>,
+    /// "Login" to use when connecting to bucket.
+    /// Can be empty for cases like AWS k8s IAM
+    /// where we can allow certain pods to connect
+    /// to the bucket directly without any credentials.
+    pub access_key_id: Option<String>,
+    /// "Password" to use when connecting to bucket.
+    pub secret_access_key: Option<String>,
+    /// A base URL to send S3 requests to.
+    /// By default, the endpoint is derived from a region name, assuming it's
+    /// an AWS S3 region name, erroring on wrong region name.
+    /// Endpoint provides a way to support other S3 flavors and their regions.
+    ///
+    /// Example: `http://127.0.0.1:5000`
+    pub endpoint: Option<String>,
+}
+
+impl std::fmt::Debug for S3Config {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("S3Config")
+            .field("bucket_name", &self.bucket_name)
+            .field("bucket_region", &self.bucket_region)
+            .field("prefix_in_bucket", &self.prefix_in_bucket)
+            .finish()
+    }
+}
+
 impl PageServerConf {
     //
     // Repository paths, relative to workdir.
@@ -345,12 +420,14 @@ impl PageServerConf {
         let mut builder = PageServerConfigBuilder::default();
         builder.workdir(workdir.to_owned());
 
-        let mut t_conf: TenantConfOpt = Default::default();
-
         for (key, item) in toml.iter() {
             match key {
                 "listen_pg_addr" => builder.listen_pg_addr(parse_toml_string(key, item)?),
                 "listen_http_addr" => builder.listen_http_addr(parse_toml_string(key, item)?),
+                "checkpoint_distance" => builder.checkpoint_distance(parse_toml_u64(key, item)?),
+                "checkpoint_period" => builder.checkpoint_period(parse_toml_duration(key, item)?),
+                "gc_horizon" => builder.gc_horizon(parse_toml_u64(key, item)?),
+                "gc_period" => builder.gc_period(parse_toml_duration(key, item)?),
                 "wait_lsn_timeout" => builder.wait_lsn_timeout(parse_toml_duration(key, item)?),
                 "wal_redo_timeout" => builder.wal_redo_timeout(parse_toml_duration(key, item)?),
                 "initial_superuser_name" => builder.superuser(parse_toml_string(key, item)?),
@@ -364,16 +441,12 @@ impl PageServerConf {
                 "auth_validation_public_key_path" => builder.auth_validation_public_key_path(Some(
                     PathBuf::from(parse_toml_string(key, item)?),
                 )),
-                "auth_type" => builder.auth_type(parse_toml_from_str(key, item)?),
+                "auth_type" => builder.auth_type(parse_toml_auth_type(key, item)?),
                 "remote_storage" => {
                     builder.remote_storage_config(Some(Self::parse_remote_storage_config(item)?))
                 }
-                "tenant_config" => {
-                    t_conf = Self::parse_toml_tenant_conf(item)?;
-                }
                 "id" => builder.id(ZNodeId(parse_toml_u64(key, item)?)),
-                "profiling" => builder.profiling(parse_toml_from_str(key, item)?),
-                _ => bail!("unrecognized pageserver option '{key}'"),
+                _ => bail!("unrecognized pageserver option '{}'", key),
             }
         }
 
@@ -399,75 +472,41 @@ impl PageServerConf {
             );
         }
 
-        conf.default_tenant_conf = t_conf.merge(TenantConf::default());
-
         Ok(conf)
     }
 
-    // subroutine of parse_and_validate to parse `[tenant_conf]` section
-
-    pub fn parse_toml_tenant_conf(item: &toml_edit::Item) -> Result<TenantConfOpt> {
-        let mut t_conf: TenantConfOpt = Default::default();
-        if let Some(checkpoint_distance) = item.get("checkpoint_distance") {
-            t_conf.checkpoint_distance =
-                Some(parse_toml_u64("checkpoint_distance", checkpoint_distance)?);
-        }
-
-        if let Some(compaction_target_size) = item.get("compaction_target_size") {
-            t_conf.compaction_target_size = Some(parse_toml_u64(
-                "compaction_target_size",
-                compaction_target_size,
-            )?);
-        }
-
-        if let Some(compaction_period) = item.get("compaction_period") {
-            t_conf.compaction_period =
-                Some(parse_toml_duration("compaction_period", compaction_period)?);
-        }
-
-        if let Some(compaction_threshold) = item.get("compaction_threshold") {
-            t_conf.compaction_threshold =
-                Some(parse_toml_u64("compaction_threshold", compaction_threshold)?.try_into()?);
-        }
-
-        if let Some(gc_horizon) = item.get("gc_horizon") {
-            t_conf.gc_horizon = Some(parse_toml_u64("gc_horizon", gc_horizon)?);
-        }
-
-        if let Some(gc_period) = item.get("gc_period") {
-            t_conf.gc_period = Some(parse_toml_duration("gc_period", gc_period)?);
-        }
-
-        if let Some(pitr_interval) = item.get("pitr_interval") {
-            t_conf.pitr_interval = Some(parse_toml_duration("pitr_interval", pitr_interval)?);
-        }
-
-        Ok(t_conf)
-    }
-
     /// subroutine of parse_config(), to parse the `[remote_storage]` table.
     fn parse_remote_storage_config(toml: &toml_edit::Item) -> anyhow::Result<RemoteStorageConfig> {
         let local_path = toml.get("local_path");
         let bucket_name = toml.get("bucket_name");
         let bucket_region = toml.get("bucket_region");
 
-        let max_concurrent_syncs = NonZeroUsize::new(
-            parse_optional_integer("max_concurrent_syncs", toml)?
-                .unwrap_or(remote_storage::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS),
-        )
-        .context("Failed to parse 'max_concurrent_syncs' as a positive integer")?;
-
-        let max_sync_errors = NonZeroU32::new(
-            parse_optional_integer("max_sync_errors", toml)?
-                .unwrap_or(remote_storage::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS),
-        )
-        .context("Failed to parse 'max_sync_errors' as a positive integer")?;
-
-        let concurrency_limit = NonZeroUsize::new(
-            parse_optional_integer("concurrency_limit", toml)?
-                .unwrap_or(remote_storage::DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT),
-        )
-        .context("Failed to parse 'concurrency_limit' as a positive integer")?;
+        let max_concurrent_sync: NonZeroUsize = if let Some(s) = toml.get("max_concurrent_sync") {
+            parse_toml_u64("max_concurrent_sync", s)
+                .and_then(|toml_u64| {
+                    toml_u64.try_into().with_context(|| {
+                        format!("'max_concurrent_sync' value {} is too large", toml_u64)
+                    })
+                })
+                .ok()
+                .and_then(NonZeroUsize::new)
+                .context("'max_concurrent_sync' must be a non-zero positive integer")?
+        } else {
+            NonZeroUsize::new(defaults::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNC).unwrap()
+        };
+        let max_sync_errors: NonZeroU32 = if let Some(s) = toml.get("max_sync_errors") {
+            parse_toml_u64("max_sync_errors", s)
+                .and_then(|toml_u64| {
+                    toml_u64.try_into().with_context(|| {
+                        format!("'max_sync_errors' value {} is too large", toml_u64)
+                    })
+                })
+                .ok()
+                .and_then(NonZeroU32::new)
+                .context("'max_sync_errors' must be a non-zero positive integer")?
+        } else {
+            NonZeroU32::new(defaults::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS).unwrap()
+        };
 
         let storage = match (local_path, bucket_name, bucket_region) {
             (None, None, None) => bail!("no 'local_path' nor 'bucket_name' option"),
@@ -480,6 +519,16 @@ impl PageServerConf {
             (None, Some(bucket_name), Some(bucket_region)) => RemoteStorageKind::AwsS3(S3Config {
                 bucket_name: parse_toml_string("bucket_name", bucket_name)?,
                 bucket_region: parse_toml_string("bucket_region", bucket_region)?,
+                access_key_id: toml
+                    .get("access_key_id")
+                    .map(|access_key_id| parse_toml_string("access_key_id", access_key_id))
+                    .transpose()?,
+                secret_access_key: toml
+                    .get("secret_access_key")
+                    .map(|secret_access_key| {
+                        parse_toml_string("secret_access_key", secret_access_key)
+                    })
+                    .transpose()?,
                 prefix_in_bucket: toml
                     .get("prefix_in_bucket")
                     .map(|prefix_in_bucket| parse_toml_string("prefix_in_bucket", prefix_in_bucket))
@@ -488,7 +537,6 @@ impl PageServerConf {
                     .get("endpoint")
                     .map(|endpoint| parse_toml_string("endpoint", endpoint))
                     .transpose()?,
-                concurrency_limit,
             }),
             (Some(local_path), None, None) => RemoteStorageKind::LocalFs(PathBuf::from(
                 parse_toml_string("local_path", local_path)?,
@@ -497,7 +545,7 @@ impl PageServerConf {
         };
 
         Ok(RemoteStorageConfig {
-            max_concurrent_syncs,
+            max_concurrent_sync,
             max_sync_errors,
             storage,
         })
@@ -505,13 +553,17 @@ impl PageServerConf {
 
     #[cfg(test)]
     pub fn test_repo_dir(test_name: &str) -> PathBuf {
-        PathBuf::from(format!("../tmp_check/test_{test_name}"))
+        PathBuf::from(format!("../tmp_check/test_{}", test_name))
     }
 
     #[cfg(test)]
     pub fn dummy_conf(repo_dir: PathBuf) -> Self {
         PageServerConf {
             id: ZNodeId(0),
+            checkpoint_distance: defaults::DEFAULT_CHECKPOINT_DISTANCE,
+            checkpoint_period: Duration::from_secs(10),
+            gc_horizon: defaults::DEFAULT_GC_HORIZON,
+            gc_period: Duration::from_secs(10),
             wait_lsn_timeout: Duration::from_secs(60),
             wal_redo_timeout: Duration::from_secs(60),
             page_cache_size: defaults::DEFAULT_PAGE_CACHE_SIZE,
@@ -524,8 +576,6 @@ impl PageServerConf {
             auth_type: AuthType::Trust,
             auth_validation_public_key_path: None,
             remote_storage_config: None,
-            profiling: ProfilingConfig::Disabled,
-            default_tenant_conf: TenantConf::dummy_conf(),
         }
     }
 }
@@ -535,7 +585,7 @@ impl PageServerConf {
 fn parse_toml_string(name: &str, item: &Item) -> Result<String> {
     let s = item
         .as_str()
-        .with_context(|| format!("configure option {name} is not a string"))?;
+        .with_context(|| format!("configure option {} is not a string", name))?;
     Ok(s.to_string())
 }
 
@@ -544,46 +594,26 @@ fn parse_toml_u64(name: &str, item: &Item) -> Result<u64> {
     // for our use, though.
     let i: i64 = item
         .as_integer()
-        .with_context(|| format!("configure option {name} is not an integer"))?;
+        .with_context(|| format!("configure option {} is not an integer", name))?;
     if i < 0 {
-        bail!("configure option {name} cannot be negative");
+        bail!("configure option {} cannot be negative", name);
     }
     Ok(i as u64)
 }
 
-fn parse_optional_integer<I, E>(name: &str, item: &toml_edit::Item) -> anyhow::Result<Option<I>>
-where
-    I: TryFrom<i64, Error = E>,
-    E: std::error::Error + Send + Sync + 'static,
-{
-    let toml_integer = match item.get(name) {
-        Some(item) => item
-            .as_integer()
-            .with_context(|| format!("configure option {name} is not an integer"))?,
-        None => return Ok(None),
-    };
-
-    I::try_from(toml_integer)
-        .map(Some)
-        .with_context(|| format!("configure option {name} is too large"))
-}
-
 fn parse_toml_duration(name: &str, item: &Item) -> Result<Duration> {
     let s = item
         .as_str()
-        .with_context(|| format!("configure option {name} is not a string"))?;
+        .with_context(|| format!("configure option {} is not a string", name))?;
 
     Ok(humantime::parse_duration(s)?)
 }
 
-fn parse_toml_from_str<T>(name: &str, item: &Item) -> Result<T>
-where
-    T: FromStr<Err = anyhow::Error>,
-{
+fn parse_toml_auth_type(name: &str, item: &Item) -> Result<AuthType> {
     let v = item
         .as_str()
-        .with_context(|| format!("configure option {name} is not a string"))?;
-    T::from_str(v)
+        .with_context(|| format!("configure option {} is not a string", name))?;
+    AuthType::from_str(v)
 }
 
 #[cfg(test)]
@@ -600,6 +630,12 @@ mod tests {
 listen_pg_addr = '127.0.0.1:64000'
 listen_http_addr = '127.0.0.1:9898'
 
+checkpoint_distance = 111 # in bytes
+checkpoint_period = '111 s'
+
+gc_period = '222 s'
+gc_horizon = 222
+
 wait_lsn_timeout = '111 s'
 wal_redo_timeout = '111 s'
 
@@ -620,8 +656,10 @@ id = 10
         let config_string = format!("pg_distrib_dir='{}'\nid=10", pg_distrib_dir.display());
         let toml = config_string.parse()?;
 
-        let parsed_config = PageServerConf::parse_and_validate(&toml, &workdir)
-            .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e}"));
+        let parsed_config =
+            PageServerConf::parse_and_validate(&toml, &workdir).unwrap_or_else(|e| {
+                panic!("Failed to parse config '{}', reason: {}", config_string, e)
+            });
 
         assert_eq!(
             parsed_config,
@@ -629,6 +667,10 @@ id = 10
                 id: ZNodeId(10),
                 listen_pg_addr: defaults::DEFAULT_PG_LISTEN_ADDR.to_string(),
                 listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(),
+                checkpoint_distance: defaults::DEFAULT_CHECKPOINT_DISTANCE,
+                checkpoint_period: humantime::parse_duration(defaults::DEFAULT_CHECKPOINT_PERIOD)?,
+                gc_horizon: defaults::DEFAULT_GC_HORIZON,
+                gc_period: humantime::parse_duration(defaults::DEFAULT_GC_PERIOD)?,
                 wait_lsn_timeout: humantime::parse_duration(defaults::DEFAULT_WAIT_LSN_TIMEOUT)?,
                 wal_redo_timeout: humantime::parse_duration(defaults::DEFAULT_WAL_REDO_TIMEOUT)?,
                 superuser: defaults::DEFAULT_SUPERUSER.to_string(),
@@ -639,8 +681,6 @@ id = 10
                 auth_type: AuthType::Trust,
                 auth_validation_public_key_path: None,
                 remote_storage_config: None,
-                profiling: ProfilingConfig::Disabled,
-                default_tenant_conf: TenantConf::default(),
             },
             "Correct defaults should be used when no config values are provided"
         );
@@ -654,13 +694,16 @@ id = 10
         let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?;
 
         let config_string = format!(
-            "{ALL_BASE_VALUES_TOML}pg_distrib_dir='{}'",
+            "{}pg_distrib_dir='{}'",
+            ALL_BASE_VALUES_TOML,
             pg_distrib_dir.display()
         );
         let toml = config_string.parse()?;
 
-        let parsed_config = PageServerConf::parse_and_validate(&toml, &workdir)
-            .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e}"));
+        let parsed_config =
+            PageServerConf::parse_and_validate(&toml, &workdir).unwrap_or_else(|e| {
+                panic!("Failed to parse config '{}', reason: {}", config_string, e)
+            });
 
         assert_eq!(
             parsed_config,
@@ -668,6 +711,10 @@ id = 10
                 id: ZNodeId(10),
                 listen_pg_addr: "127.0.0.1:64000".to_string(),
                 listen_http_addr: "127.0.0.1:9898".to_string(),
+                checkpoint_distance: 111,
+                checkpoint_period: Duration::from_secs(111),
+                gc_horizon: 222,
+                gc_period: Duration::from_secs(222),
                 wait_lsn_timeout: Duration::from_secs(111),
                 wal_redo_timeout: Duration::from_secs(111),
                 superuser: "zzzz".to_string(),
@@ -678,8 +725,6 @@ id = 10
                 auth_type: AuthType::Trust,
                 auth_validation_public_key_path: None,
                 remote_storage_config: None,
-                profiling: ProfilingConfig::Disabled,
-                default_tenant_conf: TenantConf::default(),
             },
             "Should be able to parse all basic config values correctly"
         );
@@ -708,33 +753,37 @@ local_path = '{}'"#,
 
         for remote_storage_config_str in identical_toml_declarations {
             let config_string = format!(
-                r#"{ALL_BASE_VALUES_TOML}
+                r#"{}
 pg_distrib_dir='{}'
 
-{remote_storage_config_str}"#,
+{}"#,
+                ALL_BASE_VALUES_TOML,
                 pg_distrib_dir.display(),
+                remote_storage_config_str,
             );
 
             let toml = config_string.parse()?;
 
             let parsed_remote_storage_config = PageServerConf::parse_and_validate(&toml, &workdir)
-                .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e}"))
+                .unwrap_or_else(|e| {
+                    panic!("Failed to parse config '{}', reason: {}", config_string, e)
+                })
                 .remote_storage_config
                 .expect("Should have remote storage config for the local FS");
 
             assert_eq!(
-                parsed_remote_storage_config,
-                RemoteStorageConfig {
-                    max_concurrent_syncs: NonZeroUsize::new(
-                        remote_storage::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS
-                    )
+            parsed_remote_storage_config,
+            RemoteStorageConfig {
+                max_concurrent_sync: NonZeroUsize::new(
+                    defaults::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNC
+                )
+                .unwrap(),
+                max_sync_errors: NonZeroU32::new(defaults::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS)
                     .unwrap(),
-                    max_sync_errors: NonZeroU32::new(remote_storage::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS)
-                        .unwrap(),
-                    storage: RemoteStorageKind::LocalFs(local_storage_path.clone()),
-                },
-                "Remote storage config should correctly parse the local FS config and fill other storage defaults"
-            );
+                storage: RemoteStorageKind::LocalFs(local_storage_path.clone()),
+            },
+            "Remote storage config should correctly parse the local FS config and fill other storage defaults"
+        );
         }
         Ok(())
     }
@@ -747,55 +796,63 @@ pg_distrib_dir='{}'
         let bucket_name = "some-sample-bucket".to_string();
         let bucket_region = "eu-north-1".to_string();
         let prefix_in_bucket = "test_prefix".to_string();
+        let access_key_id = "SOMEKEYAAAAASADSAH*#".to_string();
+        let secret_access_key = "SOMEsEcReTsd292v".to_string();
         let endpoint = "http://localhost:5000".to_string();
-        let max_concurrent_syncs = NonZeroUsize::new(111).unwrap();
+        let max_concurrent_sync = NonZeroUsize::new(111).unwrap();
         let max_sync_errors = NonZeroU32::new(222).unwrap();
-        let s3_concurrency_limit = NonZeroUsize::new(333).unwrap();
 
         let identical_toml_declarations = &[
             format!(
                 r#"[remote_storage]
-max_concurrent_syncs = {max_concurrent_syncs}
-max_sync_errors = {max_sync_errors}
-bucket_name = '{bucket_name}'
-bucket_region = '{bucket_region}'
-prefix_in_bucket = '{prefix_in_bucket}'
-endpoint = '{endpoint}'
-concurrency_limit = {s3_concurrency_limit}"#
+max_concurrent_sync = {}
+max_sync_errors = {}
+bucket_name = '{}'
+bucket_region = '{}'
+prefix_in_bucket = '{}'
+access_key_id = '{}'
+secret_access_key = '{}'
+endpoint = '{}'"#,
+                max_concurrent_sync, max_sync_errors, bucket_name, bucket_region, prefix_in_bucket, access_key_id, secret_access_key, endpoint
             ),
             format!(
-                "remote_storage={{max_concurrent_syncs={max_concurrent_syncs}, max_sync_errors={max_sync_errors}, bucket_name='{bucket_name}',\
-                bucket_region='{bucket_region}', prefix_in_bucket='{prefix_in_bucket}', endpoint='{endpoint}', concurrency_limit={s3_concurrency_limit}}}",
+                "remote_storage={{max_concurrent_sync={}, max_sync_errors={}, bucket_name='{}', bucket_region='{}', prefix_in_bucket='{}', access_key_id='{}', secret_access_key='{}', endpoint='{}'}}",
+                max_concurrent_sync, max_sync_errors, bucket_name, bucket_region, prefix_in_bucket, access_key_id, secret_access_key, endpoint
             ),
         ];
 
         for remote_storage_config_str in identical_toml_declarations {
             let config_string = format!(
-                r#"{ALL_BASE_VALUES_TOML}
+                r#"{}
 pg_distrib_dir='{}'
 
-{remote_storage_config_str}"#,
+{}"#,
+                ALL_BASE_VALUES_TOML,
                 pg_distrib_dir.display(),
+                remote_storage_config_str,
             );
 
             let toml = config_string.parse()?;
 
             let parsed_remote_storage_config = PageServerConf::parse_and_validate(&toml, &workdir)
-                .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e}"))
+                .unwrap_or_else(|e| {
+                    panic!("Failed to parse config '{}', reason: {}", config_string, e)
+                })
                 .remote_storage_config
                 .expect("Should have remote storage config for S3");
 
             assert_eq!(
                 parsed_remote_storage_config,
                 RemoteStorageConfig {
-                    max_concurrent_syncs,
+                    max_concurrent_sync,
                     max_sync_errors,
                     storage: RemoteStorageKind::AwsS3(S3Config {
                         bucket_name: bucket_name.clone(),
                         bucket_region: bucket_region.clone(),
+                        access_key_id: Some(access_key_id.clone()),
+                        secret_access_key: Some(secret_access_key.clone()),
                         prefix_in_bucket: Some(prefix_in_bucket.clone()),
-                        endpoint: Some(endpoint.clone()),
-                        concurrency_limit: s3_concurrency_limit,
+                        endpoint: Some(endpoint.clone())
                     }),
                 },
                 "Remote storage config should correctly parse the S3 config"

pageserver/src/http/models.rs

@@ -1,10 +1,12 @@
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
-use utils::{
+use zenith_utils::{
     lsn::Lsn,
     zid::{ZNodeId, ZTenantId, ZTimelineId},
 };
 
+use crate::timelines::{LocalTimelineInfo, TimelineInfo};
+
 #[serde_as]
 #[derive(Serialize, Deserialize)]
 pub struct TimelineCreateRequest {
@@ -20,19 +22,11 @@ pub struct TimelineCreateRequest {
 }
 
 #[serde_as]
-#[derive(Serialize, Deserialize, Default)]
+#[derive(Serialize, Deserialize)]
 pub struct TenantCreateRequest {
     #[serde(default)]
     #[serde_as(as = "Option<DisplayFromStr>")]
     pub new_tenant_id: Option<ZTenantId>,
-    pub checkpoint_distance: Option<u64>,
-    pub compaction_target_size: Option<u64>,
-    pub compaction_period: Option<String>,
-    pub compaction_threshold: Option<usize>,
-    pub gc_horizon: Option<u64>,
-    pub gc_period: Option<String>,
-    pub image_creation_threshold: Option<usize>,
-    pub pitr_interval: Option<String>,
 }
 
 #[serde_as]
@@ -40,48 +34,155 @@ pub struct TenantCreateRequest {
 #[serde(transparent)]
 pub struct TenantCreateResponse(#[serde_as(as = "DisplayFromStr")] pub ZTenantId);
 
-#[derive(Serialize)]
-pub struct StatusResponse {
-    pub id: ZNodeId,
-}
-
-impl TenantCreateRequest {
-    pub fn new(new_tenant_id: Option<ZTenantId>) -> TenantCreateRequest {
-        TenantCreateRequest {
-            new_tenant_id,
-            ..Default::default()
-        }
-    }
+#[derive(Clone)]
+pub enum TimelineInfoV1 {
+    Local {
+        timeline_id: ZTimelineId,
+        tenant_id: ZTenantId,
+        last_record_lsn: Lsn,
+        prev_record_lsn: Option<Lsn>,
+        ancestor_timeline_id: Option<ZTimelineId>,
+        ancestor_lsn: Option<Lsn>,
+        disk_consistent_lsn: Lsn,
+        current_logical_size: Option<usize>,
+        current_logical_size_non_incremental: Option<usize>,
+    },
+    Remote {
+        timeline_id: ZTimelineId,
+        tenant_id: ZTenantId,
+        disk_consistent_lsn: Lsn,
+    },
 }
 
 #[serde_as]
 #[derive(Serialize, Deserialize)]
-pub struct TenantConfigRequest {
-    pub tenant_id: ZTenantId,
-    #[serde(default)]
+pub struct TimelineInfoResponseV1 {
+    pub kind: String,
+    #[serde_as(as = "DisplayFromStr")]
+    timeline_id: ZTimelineId,
+    #[serde_as(as = "DisplayFromStr")]
+    tenant_id: ZTenantId,
+    #[serde_as(as = "DisplayFromStr")]
+    disk_consistent_lsn: Lsn,
     #[serde_as(as = "Option<DisplayFromStr>")]
-    pub checkpoint_distance: Option<u64>,
-    pub compaction_target_size: Option<u64>,
-    pub compaction_period: Option<String>,
-    pub compaction_threshold: Option<usize>,
-    pub gc_horizon: Option<u64>,
-    pub gc_period: Option<String>,
-    pub image_creation_threshold: Option<usize>,
-    pub pitr_interval: Option<String>,
+    last_record_lsn: Option<Lsn>,
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    prev_record_lsn: Option<Lsn>,
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    ancestor_timeline_id: Option<ZTimelineId>,
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    ancestor_lsn: Option<Lsn>,
+    current_logical_size: Option<usize>,
+    current_logical_size_non_incremental: Option<usize>,
 }
 
-impl TenantConfigRequest {
-    pub fn new(tenant_id: ZTenantId) -> TenantConfigRequest {
-        TenantConfigRequest {
-            tenant_id,
-            checkpoint_distance: None,
-            compaction_target_size: None,
-            compaction_period: None,
-            compaction_threshold: None,
-            gc_horizon: None,
-            gc_period: None,
-            image_creation_threshold: None,
-            pitr_interval: None,
+impl From<TimelineInfoV1> for TimelineInfoResponseV1 {
+    fn from(other: TimelineInfoV1) -> Self {
+        match other {
+            TimelineInfoV1::Local {
+                timeline_id,
+                tenant_id,
+                last_record_lsn,
+                prev_record_lsn,
+                ancestor_timeline_id,
+                ancestor_lsn,
+                disk_consistent_lsn,
+                current_logical_size,
+                current_logical_size_non_incremental,
+            } => TimelineInfoResponseV1 {
+                kind: "Local".to_owned(),
+                timeline_id,
+                tenant_id,
+                disk_consistent_lsn,
+                last_record_lsn: Some(last_record_lsn),
+                prev_record_lsn,
+                ancestor_timeline_id,
+                ancestor_lsn,
+                current_logical_size,
+                current_logical_size_non_incremental,
+            },
+            TimelineInfoV1::Remote {
+                timeline_id,
+                tenant_id,
+                disk_consistent_lsn,
+            } => TimelineInfoResponseV1 {
+                kind: "Remote".to_owned(),
+                timeline_id,
+                tenant_id,
+                disk_consistent_lsn,
+                last_record_lsn: None,
+                prev_record_lsn: None,
+                ancestor_timeline_id: None,
+                ancestor_lsn: None,
+                current_logical_size: None,
+                current_logical_size_non_incremental: None,
+            },
         }
     }
 }
+
+impl TryFrom<TimelineInfoResponseV1> for TimelineInfoV1 {
+    type Error = anyhow::Error;
+
+    fn try_from(other: TimelineInfoResponseV1) -> anyhow::Result<Self> {
+        Ok(match other.kind.as_str() {
+            "Local" => TimelineInfoV1::Local {
+                timeline_id: other.timeline_id,
+                tenant_id: other.tenant_id,
+                last_record_lsn: other.last_record_lsn.ok_or(anyhow::anyhow!(
+                    "Local timeline should have last_record_lsn"
+                ))?,
+                prev_record_lsn: other.prev_record_lsn,
+                ancestor_timeline_id: other.ancestor_timeline_id.map(ZTimelineId::from),
+                ancestor_lsn: other.ancestor_lsn,
+                disk_consistent_lsn: other.disk_consistent_lsn,
+                current_logical_size: other.current_logical_size,
+                current_logical_size_non_incremental: other.current_logical_size_non_incremental,
+            },
+            "Remote" => TimelineInfoV1::Remote {
+                timeline_id: other.timeline_id,
+                tenant_id: other.tenant_id,
+                disk_consistent_lsn: other.disk_consistent_lsn,
+            },
+            unknown => anyhow::bail!("Unknown timeline kind: {}", unknown),
+        })
+    }
+}
+
+fn from_local(
+    tenant_id: ZTenantId,
+    timeline_id: ZTimelineId,
+    local: &LocalTimelineInfo,
+) -> TimelineInfoV1 {
+    TimelineInfoV1::Local {
+        timeline_id,
+        tenant_id,
+        last_record_lsn: local.last_record_lsn,
+        prev_record_lsn: local.prev_record_lsn,
+        ancestor_timeline_id: local.ancestor_timeline_id.map(ZTimelineId::from),
+        ancestor_lsn: local.ancestor_lsn,
+        disk_consistent_lsn: local.disk_consistent_lsn,
+        current_logical_size: local.current_logical_size,
+        current_logical_size_non_incremental: local.current_logical_size_non_incremental,
+    }
+}
+
+impl From<TimelineInfo> for TimelineInfoV1 {
+    fn from(t: TimelineInfo) -> Self {
+        match (t.local.as_ref(), t.remote.as_ref()) {
+            (None, None) => unreachable!(),
+            (None, Some(remote)) => TimelineInfoV1::Remote {
+                timeline_id: t.timeline_id,
+                tenant_id: t.tenant_id,
+                disk_consistent_lsn: remote.remote_consistent_lsn.unwrap_or(Lsn(0)),
+            },
+            (Some(local), None) => from_local(t.tenant_id, t.timeline_id, local),
+            (Some(local), Some(_)) => from_local(t.tenant_id, t.timeline_id, local),
+        }
+    }
+}
+
+#[derive(Serialize)]
+pub struct StatusResponse {
+    pub id: ZNodeId,
+}

pageserver/src/http/openapi_spec.yml

@@ -18,7 +18,7 @@ paths:
               schema:
                 type: object
                 required:
-                  - id
+                - id
                 properties:
                   id:
                     type: integer
@@ -122,110 +122,6 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/Error"
-
-
-  /v1/tenant/{tenant_id}/timeline/{timeline_id}/attach:
-    parameters:
-      - name: tenant_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-      - name: timeline_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-    post:
-      description: Attach remote timeline
-      responses:
-        "200":
-          description: Timeline attaching scheduled
-        "400":
-          description: Error when no tenant id found in path or no timeline id
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
-        "404":
-          description: Timeline not found
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/NotFoundError"
-        "409":
-          description: Timeline download is already in progress
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ConflictError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-
-
-  /v1/tenant/{tenant_id}/timeline/{timeline_id}/detach:
-    parameters:
-      - name: tenant_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-      - name: timeline_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-    post:
-      description: Detach local timeline
-      responses:
-        "200":
-          description: Timeline detached
-        "400":
-          description: Error when no tenant id found in path or no timeline id
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-
-
   /v1/tenant/{tenant_id}/timeline/:
     parameters:
       - name: tenant_id
@@ -252,7 +148,6 @@ paths:
                   format: hex
                 ancestor_start_lsn:
                   type: string
-                  format: hex
       responses:
         "201":
           description: TimelineInfo
@@ -283,7 +178,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/ConflictError"
+                $ref: "#/components/schemas/AlreadyExistsError"
         "500":
           description: Generic operation error
           content:
@@ -328,7 +223,11 @@ paths:
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/TenantCreateInfo"
+              type: object
+              properties:
+                new_tenant_id:
+                  type: string
+                  format: hex
       responses:
         "201":
           description: New tenant created successfully
@@ -360,55 +259,14 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/ConflictError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-  /v1/tenant/config:
-    put:
-      description: |
-        Update tenant's config.
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: "#/components/schemas/TenantConfigInfo"
-      responses:
-        "200":
-          description: OK
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  $ref: "#/components/schemas/TenantInfo"
-        "400":
-          description: Malformed tenant config request
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
+                $ref: "#/components/schemas/AlreadyExistsError"
         "500":
           description: Generic operation error
           content:
             application/json:
               schema:
                 $ref: "#/components/schemas/Error"
+
 components:
   securitySchemes:
     JWT:
@@ -426,50 +284,12 @@ components:
           type: string
         state:
           type: string
-    TenantCreateInfo:
-      type: object
-      properties:
-        new_tenant_id:
-          type: string
-          format: hex
-        tenant_id:
-          type: string
-          format: hex
-        gc_period:
-          type: string
-        gc_horizon:
-          type: integer
-        pitr_interval:
-          type: string
-        checkpoint_distance:
-          type: integer
-        compaction_period:
-          type: string
-        compaction_threshold:
-          type: string
-    TenantConfigInfo:
-      type: object
-      properties:
-        tenant_id:
-          type: string
-          format: hex
-        gc_period:
-          type: string
-        gc_horizon:
-          type: integer
-        pitr_interval:
-          type: string
-        checkpoint_distance:
-          type: integer
-        compaction_period:
-          type: string
-        compaction_threshold:
-          type: string
     TimelineInfo:
       type: object
       required:
         - timeline_id
         - tenant_id
+        - disk_consistent_lsn
       properties:
         timeline_id:
           type: string
@@ -477,45 +297,17 @@ components:
         tenant_id:
           type: string
           format: hex
-        local:
-          $ref: "#/components/schemas/LocalTimelineInfo"
-        remote:
-          $ref: "#/components/schemas/RemoteTimelineInfo"
-    RemoteTimelineInfo:
-      type: object
-      required:
-        - awaits_download
-        - remote_consistent_lsn
-      properties:
-        awaits_download:
-          type: boolean
-        remote_consistent_lsn:
-          type: string
-          format: hex
-    LocalTimelineInfo:
-      type: object
-      required:
-        - last_record_lsn
-        - disk_consistent_lsn
-        - timeline_state
-      properties:
         last_record_lsn:
           type: string
-          format: hex
-        disk_consistent_lsn:
-          type: string
-          format: hex
-        timeline_state:
+        prev_record_lsn:
           type: string
         ancestor_timeline_id:
           type: string
           format: hex
         ancestor_lsn:
           type: string
-          format: hex
-        prev_record_lsn:
+        disk_consistent_lsn:
           type: string
-          format: hex
         current_logical_size:
           type: integer
         current_logical_size_non_incremental:
@@ -535,6 +327,13 @@ components:
       properties:
         msg:
           type: string
+    AlreadyExistsError:
+      type: object
+      required:
+        - msg
+      properties:
+        msg:
+          type: string
     ForbiddenError:
       type: object
       required:
@@ -542,20 +341,6 @@ components:
       properties:
         msg:
           type: string
-    NotFoundError:
-      type: object
-      required:
-        - msg
-      properties:
-        msg:
-          type: string
-    ConflictError:
-      type: object
-      required:
-        - msg
-      properties:
-        msg:
-          type: string
 
 security:
   - JWT: []

pageserver/src/http/routes.rs

@@ -1,67 +1,58 @@
 use std::sync::Arc;
 
-use anyhow::{Context, Result};
+use anyhow::Result;
 use hyper::StatusCode;
 use hyper::{Body, Request, Response, Uri};
-use remote_storage::GenericRemoteStorage;
+use tokio::sync::RwLock;
 use tracing::*;
+use zenith_utils::auth::JwtAuth;
+use zenith_utils::http::endpoint::attach_openapi_ui;
+use zenith_utils::http::endpoint::auth_middleware;
+use zenith_utils::http::endpoint::check_permission;
+use zenith_utils::http::error::ApiError;
+use zenith_utils::http::{
+    endpoint,
+    error::HttpErrorBody,
+    json::{json_request, json_response},
+    request::parse_request_param,
+};
+use zenith_utils::http::{RequestExt, RouterBuilder};
+use zenith_utils::zid::{ZTenantTimelineId, ZTimelineId};
 
 use super::models::{
-    StatusResponse, TenantConfigRequest, TenantCreateRequest, TenantCreateResponse,
-    TimelineCreateRequest,
+    StatusResponse, TenantCreateRequest, TenantCreateResponse, TimelineCreateRequest,
+    TimelineInfoResponseV1, TimelineInfoV1,
 };
-use crate::repository::Repository;
-use crate::storage_sync;
-use crate::storage_sync::index::{RemoteIndex, RemoteTimeline};
-use crate::tenant_config::TenantConfOpt;
-use crate::timelines::{LocalTimelineInfo, RemoteTimelineInfo, TimelineInfo};
-use crate::{config::PageServerConf, tenant_mgr, timelines};
-use utils::{
-    auth::JwtAuth,
-    http::{
-        endpoint::{self, attach_openapi_ui, auth_middleware, check_permission},
-        error::{ApiError, HttpErrorBody},
-        json::{json_request, json_response},
-        request::parse_request_param,
-        RequestExt, RouterBuilder,
-    },
-    zid::{ZTenantId, ZTenantTimelineId, ZTimelineId},
+use crate::remote_storage::{schedule_timeline_download, RemoteTimelineIndex};
+use crate::timelines::{
+    extract_remote_timeline_info, LocalTimelineInfo, RemoteTimelineInfo, TimelineInfo,
 };
+use crate::{config::PageServerConf, tenant_mgr, timelines, ZTenantId};
 
+#[derive(Debug)]
 struct State {
     conf: &'static PageServerConf,
     auth: Option<Arc<JwtAuth>>,
-    remote_index: RemoteIndex,
+    remote_index: Arc<RwLock<RemoteTimelineIndex>>,
     allowlist_routes: Vec<Uri>,
-    remote_storage: Option<GenericRemoteStorage>,
 }
 
 impl State {
     fn new(
         conf: &'static PageServerConf,
         auth: Option<Arc<JwtAuth>>,
-        remote_index: RemoteIndex,
-    ) -> anyhow::Result<Self> {
+        remote_index: Arc<RwLock<RemoteTimelineIndex>>,
+    ) -> Self {
         let allowlist_routes = ["/v1/status", "/v1/doc", "/swagger.yml"]
             .iter()
             .map(|v| v.parse().unwrap())
             .collect::<Vec<_>>();
-        // Note that this remote storage is created separately from the main one in the sync_loop.
-        // It's fine since it's stateless and some code duplication saves us from bloating the code around with generics.
-        let remote_storage = conf
-            .remote_storage_config
-            .as_ref()
-            .map(|storage_config| GenericRemoteStorage::new(conf.workdir.clone(), storage_config))
-            .transpose()
-            .context("Failed to init generic remote storage")?;
-
-        Ok(Self {
+        Self {
             conf,
             auth,
             allowlist_routes,
             remote_index,
-            remote_storage,
-        })
+        }
     }
 }
 
@@ -81,7 +72,10 @@ fn get_config(request: &Request<Body>) -> &'static PageServerConf {
 // healthcheck handler
 async fn status_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let config = get_config(&request);
-    json_response(StatusCode::OK, StatusResponse { id: config.id })
+    Ok(json_response(
+        StatusCode::OK,
+        StatusResponse { id: config.id },
+    )?)
 }
 
 async fn timeline_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -120,28 +114,18 @@ async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>,
     .await
     .map_err(ApiError::from_err)??;
 
+    let remote_index = get_state(&request).remote_index.read().await;
     let mut response_data = Vec::with_capacity(local_timeline_infos.len());
     for (timeline_id, local_timeline_info) in local_timeline_infos {
         response_data.push(TimelineInfo {
             tenant_id,
             timeline_id,
             local: Some(local_timeline_info),
-            remote: get_state(&request)
-                .remote_index
-                .read()
-                .await
-                .timeline_entry(&ZTenantTimelineId {
-                    tenant_id,
-                    timeline_id,
-                })
-                .map(|remote_entry| RemoteTimelineInfo {
-                    remote_consistent_lsn: remote_entry.metadata.disk_consistent_lsn(),
-                    awaits_download: remote_entry.awaits_download,
-                }),
+            remote: extract_remote_timeline_info(tenant_id, timeline_id, &remote_index),
         })
     }
 
-    json_response(StatusCode::OK, response_data)
+    Ok(json_response(StatusCode::OK, response_data)?)
 }
 
 // Gate non incremental logical size calculation behind a flag
@@ -159,54 +143,48 @@ fn get_include_non_incremental_logical_size(request: &Request<Body>) -> bool {
         .unwrap_or(false)
 }
 
-async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+// common part for v1 and v2 handlers
+async fn timeline_detail_common(request: Request<Body>) -> Result<TimelineInfo, ApiError> {
     let tenant_id: ZTenantId = parse_request_param(&request, "tenant_id")?;
     check_permission(&request, Some(tenant_id))?;
 
     let timeline_id: ZTimelineId = parse_request_param(&request, "timeline_id")?;
     let include_non_incremental_logical_size = get_include_non_incremental_logical_size(&request);
 
-    let (local_timeline_info, remote_timeline_info) = async {
-        // any error here will render local timeline as None
-        // XXX .in_current_span does not attach messages in spawn_blocking future to current future's span
-        let local_timeline_info = tokio::task::spawn_blocking(move || {
-            let repo = tenant_mgr::get_repository_for_tenant(tenant_id)?;
-            let local_timeline = {
-                repo.get_timeline(timeline_id)
-                    .as_ref()
-                    .map(|timeline| {
-                        LocalTimelineInfo::from_repo_timeline(
-                            tenant_id,
-                            timeline_id,
-                            timeline,
-                            include_non_incremental_logical_size,
-                        )
-                    })
-                    .transpose()?
-            };
-            Ok::<_, anyhow::Error>(local_timeline)
-        })
-        .await
-        .ok()
-        .and_then(|r| r.ok())
-        .flatten();
+    let span = info_span!("timeline_detail_handler", tenant = %tenant_id, timeline = %timeline_id);
 
-        let remote_timeline_info = {
-            let remote_index_read = get_state(&request).remote_index.read().await;
-            remote_index_read
-                .timeline_entry(&ZTenantTimelineId {
-                    tenant_id,
-                    timeline_id,
-                })
-                .map(|remote_entry| RemoteTimelineInfo {
-                    remote_consistent_lsn: remote_entry.metadata.disk_consistent_lsn(),
-                    awaits_download: remote_entry.awaits_download,
+    let (local_timeline_info, span) = tokio::task::spawn_blocking(move || {
+        let entered = span.entered();
+        let repo = tenant_mgr::get_repository_for_tenant(tenant_id)?;
+        let local_timeline = {
+            repo.get_timeline(timeline_id)
+                .map(|timeline| {
+                    LocalTimelineInfo::from_repo_timeline(
+                        timeline,
+                        include_non_incremental_logical_size,
+                    )
                 })
+                .transpose()?
         };
-        (local_timeline_info, remote_timeline_info)
-    }
-    .instrument(info_span!("timeline_detail_handler", tenant = %tenant_id, timeline = %timeline_id))
-    .await;
+        Ok::<_, anyhow::Error>((local_timeline, entered.exit()))
+    })
+    .await
+    .map_err(ApiError::from_err)??;
+
+    let remote_timeline_info = {
+        let remote_index_read = get_state(&request).remote_index.read().await;
+        remote_index_read
+            .timeline_entry(&ZTenantTimelineId {
+                tenant_id,
+                timeline_id,
+            })
+            .map(|remote_entry| RemoteTimelineInfo {
+                remote_consistent_lsn: remote_entry.disk_consistent_lsn(),
+                awaits_download: remote_entry.get_awaits_download(),
+            })
+    };
+
+    let _enter = span.entered();
 
     if local_timeline_info.is_none() && remote_timeline_info.is_none() {
         return Err(ApiError::NotFound(
@@ -214,14 +192,27 @@ async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body
         ));
     }
 
-    let timeline_info = TimelineInfo {
+    Ok(TimelineInfo {
         tenant_id,
         timeline_id,
         local: local_timeline_info,
         remote: remote_timeline_info,
-    };
+    })
+}
 
-    json_response(StatusCode::OK, timeline_info)
+// TODO remove when console adopts v2
+async fn timeline_detail_handler_v1(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let timeline_info = timeline_detail_common(request).await?;
+    Ok(json_response(
+        StatusCode::OK,
+        TimelineInfoResponseV1::from(TimelineInfoV1::from(timeline_info)),
+    )?)
+}
+
+async fn timeline_detail_handler_v2(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let timeline_info = timeline_detail_common(request).await?;
+
+    Ok(json_response(StatusCode::OK, timeline_info)?)
 }
 
 async fn timeline_attach_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -229,100 +220,38 @@ async fn timeline_attach_handler(request: Request<Body>) -> Result<Response<Body
     check_permission(&request, Some(tenant_id))?;
 
     let timeline_id: ZTimelineId = parse_request_param(&request, "timeline_id")?;
-    info!(
-        "Handling timeline {} attach for tenant: {}",
-        timeline_id, tenant_id,
-    );
+    let span = info_span!("timeline_attach_handler", tenant = %tenant_id, timeline = %timeline_id);
 
-    tokio::task::spawn_blocking(move || {
-        if tenant_mgr::get_local_timeline_with_load(tenant_id, timeline_id).is_ok() {
-            // TODO: maybe answer with 309 Not Modified here?
+    let span = tokio::task::spawn_blocking(move || {
+        let entered = span.entered();
+        if tenant_mgr::get_timeline_for_tenant_load(tenant_id, timeline_id).is_ok() {
             anyhow::bail!("Timeline is already present locally")
         };
-        Ok(())
+        Ok(entered.exit())
     })
     .await
     .map_err(ApiError::from_err)??;
 
-    let sync_id = ZTenantTimelineId {
-        tenant_id,
-        timeline_id,
-    };
-    let state = get_state(&request);
-    let remote_index = &state.remote_index;
+    let mut remote_index_write = get_state(&request).remote_index.write().await;
 
-    let mut index_accessor = remote_index.write().await;
-    if let Some(remote_timeline) = index_accessor.timeline_entry_mut(&sync_id) {
-        if remote_timeline.awaits_download {
-            return Err(ApiError::Conflict(
-                "Timeline download is already in progress".to_string(),
-            ));
-        }
-
-        remote_timeline.awaits_download = true;
-        storage_sync::schedule_layer_download(tenant_id, timeline_id);
-        return json_response(StatusCode::ACCEPTED, ());
-    } else {
-        // no timeline in the index, release the lock to make the potentially lengthy download opetation
-        drop(index_accessor);
-    }
-
-    let new_timeline = match try_download_index_part_data(state, sync_id).await {
-        Ok(Some(mut new_timeline)) => {
-            tokio::fs::create_dir_all(state.conf.timeline_path(&timeline_id, &tenant_id))
-                .await
-                .context("Failed to create new timeline directory")?;
-            new_timeline.awaits_download = true;
-            new_timeline
-        }
-        Ok(None) => return Err(ApiError::NotFound("Unknown remote timeline".to_string())),
-        Err(e) => {
-            error!("Failed to retrieve remote timeline data: {:?}", e);
-            return Err(ApiError::NotFound(
-                "Failed to retrieve remote timeline".to_string(),
-            ));
-        }
-    };
-
-    let mut index_accessor = remote_index.write().await;
-    match index_accessor.timeline_entry_mut(&sync_id) {
-        Some(remote_timeline) => {
-            if remote_timeline.awaits_download {
-                return Err(ApiError::Conflict(
-                    "Timeline download is already in progress".to_string(),
-                ));
-            }
-            remote_timeline.awaits_download = true;
-        }
-        None => index_accessor.add_timeline_entry(sync_id, new_timeline),
-    }
-    storage_sync::schedule_layer_download(tenant_id, timeline_id);
-    json_response(StatusCode::ACCEPTED, ())
-}
-
-async fn try_download_index_part_data(
-    state: &State,
-    sync_id: ZTenantTimelineId,
-) -> anyhow::Result<Option<RemoteTimeline>> {
-    let index_part = match state.remote_storage.as_ref() {
-        Some(GenericRemoteStorage::Local(local_storage)) => {
-            storage_sync::download_index_part(state.conf, local_storage, sync_id).await
-        }
-        Some(GenericRemoteStorage::S3(s3_storage)) => {
-            storage_sync::download_index_part(state.conf, s3_storage, sync_id).await
-        }
-        None => return Ok(None),
-    }
-    .with_context(|| format!("Failed to download index part for timeline {sync_id}"))?;
-
-    let timeline_path = state
-        .conf
-        .timeline_path(&sync_id.timeline_id, &sync_id.tenant_id);
-    RemoteTimeline::from_index_part(&timeline_path, index_part)
-        .map(Some)
-        .with_context(|| {
-            format!("Failed to convert index part into remote timeline for timeline {sync_id}")
+    let _enter = span.entered(); // entered guard cannot live across awaits (non Send)
+    let index_entry = remote_index_write
+        .timeline_entry_mut(&ZTenantTimelineId {
+            tenant_id,
+            timeline_id,
         })
+        .ok_or_else(|| ApiError::BadRequest("Unknown remote timeline".to_string()))?;
+
+    if index_entry.get_awaits_download() {
+        return Err(ApiError::NotFound(
+            "Timeline download is already in progress".to_string(),
+        ));
+    }
+
+    index_entry.set_awaits_download(true);
+    schedule_timeline_download(tenant_id, timeline_id);
+
+    Ok(json_response(StatusCode::ACCEPTED, ())?)
 }
 
 async fn timeline_detach_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -335,13 +264,13 @@ async fn timeline_detach_handler(request: Request<Body>) -> Result<Response<Body
         let _enter =
             info_span!("timeline_detach_handler", tenant = %tenant_id, timeline = %timeline_id)
                 .entered();
-        let state = get_state(&request);
-        tenant_mgr::detach_timeline(state.conf, tenant_id, timeline_id)
+        let repo = tenant_mgr::get_repository_for_tenant(tenant_id)?;
+        repo.detach_timeline(timeline_id)
     })
     .await
     .map_err(ApiError::from_err)??;
 
-    json_response(StatusCode::OK, ())
+    Ok(json_response(StatusCode::OK, ())?)
 }
 
 async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -353,9 +282,9 @@ async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, A
         crate::tenant_mgr::list_tenants()
     })
     .await
-    .map_err(ApiError::from_err)?;
+    .map_err(ApiError::from_err)??;
 
-    json_response(StatusCode::OK, response_data)
+    Ok(json_response(StatusCode::OK, response_data)?)
 }
 
 async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -363,29 +292,7 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
     check_permission(&request, None)?;
 
     let request_data: TenantCreateRequest = json_request(&mut request).await?;
-    let remote_index = get_state(&request).remote_index.clone();
-
-    let mut tenant_conf = TenantConfOpt::default();
-    if let Some(gc_period) = request_data.gc_period {
-        tenant_conf.gc_period =
-            Some(humantime::parse_duration(&gc_period).map_err(ApiError::from_err)?);
-    }
-    tenant_conf.gc_horizon = request_data.gc_horizon;
-    tenant_conf.image_creation_threshold = request_data.image_creation_threshold;
-
-    if let Some(pitr_interval) = request_data.pitr_interval {
-        tenant_conf.pitr_interval =
-            Some(humantime::parse_duration(&pitr_interval).map_err(ApiError::from_err)?);
-    }
-
-    tenant_conf.checkpoint_distance = request_data.checkpoint_distance;
-    tenant_conf.compaction_target_size = request_data.compaction_target_size;
-    tenant_conf.compaction_threshold = request_data.compaction_threshold;
-
-    if let Some(compaction_period) = request_data.compaction_period {
-        tenant_conf.compaction_period =
-            Some(humantime::parse_duration(&compaction_period).map_err(ApiError::from_err)?);
-    }
+    let remote_index = Arc::clone(&get_state(&request).remote_index);
 
     let target_tenant_id = request_data
         .new_tenant_id
@@ -394,9 +301,8 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
 
     let new_tenant_id = tokio::task::spawn_blocking(move || {
         let _enter = info_span!("tenant_create", tenant = ?target_tenant_id).entered();
-        let conf = get_config(&request);
 
-        tenant_mgr::create_tenant_repository(conf, tenant_conf, target_tenant_id, remote_index)
+        tenant_mgr::create_tenant_repository(get_config(&request), target_tenant_id, remote_index)
     })
     .await
     .map_err(ApiError::from_err)??;
@@ -407,45 +313,6 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
     })
 }
 
-async fn tenant_config_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let request_data: TenantConfigRequest = json_request(&mut request).await?;
-    let tenant_id = request_data.tenant_id;
-    // check for management permission
-    check_permission(&request, Some(tenant_id))?;
-
-    let mut tenant_conf: TenantConfOpt = Default::default();
-    if let Some(gc_period) = request_data.gc_period {
-        tenant_conf.gc_period =
-            Some(humantime::parse_duration(&gc_period).map_err(ApiError::from_err)?);
-    }
-    tenant_conf.gc_horizon = request_data.gc_horizon;
-    tenant_conf.image_creation_threshold = request_data.image_creation_threshold;
-
-    if let Some(pitr_interval) = request_data.pitr_interval {
-        tenant_conf.pitr_interval =
-            Some(humantime::parse_duration(&pitr_interval).map_err(ApiError::from_err)?);
-    }
-
-    tenant_conf.checkpoint_distance = request_data.checkpoint_distance;
-    tenant_conf.compaction_target_size = request_data.compaction_target_size;
-    tenant_conf.compaction_threshold = request_data.compaction_threshold;
-
-    if let Some(compaction_period) = request_data.compaction_period {
-        tenant_conf.compaction_period =
-            Some(humantime::parse_duration(&compaction_period).map_err(ApiError::from_err)?);
-    }
-
-    tokio::task::spawn_blocking(move || {
-        let _enter = info_span!("tenant_config", tenant = ?tenant_id).entered();
-
-        tenant_mgr::update_tenant_config(tenant_conf, tenant_id)
-    })
-    .await
-    .map_err(ApiError::from_err)??;
-
-    json_response(StatusCode::OK, ())
-}
-
 async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
     json_response(
         StatusCode::NOT_FOUND,
@@ -456,8 +323,8 @@ async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
 pub fn make_router(
     conf: &'static PageServerConf,
     auth: Option<Arc<JwtAuth>>,
-    remote_index: RemoteIndex,
-) -> anyhow::Result<RouterBuilder<hyper::Body, ApiError>> {
+    remote_index: Arc<RwLock<RemoteTimelineIndex>>,
+) -> RouterBuilder<hyper::Body, ApiError> {
     let spec = include_bytes!("openapi_spec.yml");
     let mut router = attach_openapi_ui(endpoint::make_router(), spec, "/swagger.yml", "/v1/doc");
     if auth.is_some() {
@@ -471,19 +338,20 @@ pub fn make_router(
         }))
     }
 
-    Ok(router
-        .data(Arc::new(
-            State::new(conf, auth, remote_index).context("Failed to initialize router state")?,
-        ))
+    router
+        .data(Arc::new(State::new(conf, auth, remote_index)))
         .get("/v1/status", status_handler)
         .get("/v1/tenant", tenant_list_handler)
         .post("/v1/tenant", tenant_create_handler)
-        .put("/v1/tenant/config", tenant_config_handler)
         .get("/v1/tenant/:tenant_id/timeline", timeline_list_handler)
         .post("/v1/tenant/:tenant_id/timeline", timeline_create_handler)
         .get(
             "/v1/tenant/:tenant_id/timeline/:timeline_id",
-            timeline_detail_handler,
+            timeline_detail_handler_v1,
+        )
+        .get(
+            "/v2/tenant/:tenant_id/timeline/:timeline_id",
+            timeline_detail_handler_v2,
         )
         .post(
             "/v1/tenant/:tenant_id/timeline/:timeline_id/attach",
@@ -493,5 +361,5 @@ pub fn make_router(
             "/v1/tenant/:tenant_id/timeline/:timeline_id/detach",
             timeline_detach_handler,
         )
-        .any(handler_404))
+        .any(handler_404)
 }

pageserver/src/import_datadir.rs

@@ -11,16 +11,15 @@ use anyhow::{bail, ensure, Context, Result};
 use bytes::Bytes;
 use tracing::*;
 
-use crate::pgdatadir_mapping::*;
-use crate::reltag::{RelTag, SlruKind};
-use crate::repository::Repository;
+use crate::relish::*;
+use crate::repository::*;
 use crate::walingest::WalIngest;
 use postgres_ffi::relfile_utils::*;
 use postgres_ffi::waldecoder::*;
 use postgres_ffi::xlog_utils::*;
+use postgres_ffi::Oid;
 use postgres_ffi::{pg_constants, ControlFileData, DBState_DB_SHUTDOWNED};
-use postgres_ffi::{Oid, TransactionId};
-use utils::lsn::Lsn;
+use zenith_utils::lsn::Lsn;
 
 ///
 /// Import all relation data pages from local disk into the repository.
@@ -28,47 +27,42 @@ use utils::lsn::Lsn;
 /// This is currently only used to import a cluster freshly created by initdb.
 /// The code that deals with the checkpoint would not work right if the
 /// cluster was not shut down cleanly.
-pub fn import_timeline_from_postgres_datadir<R: Repository>(
+pub fn import_timeline_from_postgres_datadir(
     path: &Path,
-    tline: &mut DatadirTimeline<R>,
+    writer: &dyn TimelineWriter,
     lsn: Lsn,
 ) -> Result<()> {
     let mut pg_control: Option<ControlFileData> = None;
 
-    let mut modification = tline.begin_modification(lsn);
-    modification.init_empty()?;
-
     // Scan 'global'
-    let mut relfiles: Vec<PathBuf> = Vec::new();
     for direntry in fs::read_dir(path.join("global"))? {
         let direntry = direntry?;
         match direntry.file_name().to_str() {
             None => continue,
 
             Some("pg_control") => {
-                pg_control = Some(import_control_file(&mut modification, &direntry.path())?);
-            }
-            Some("pg_filenode.map") => {
-                import_relmap_file(
-                    &mut modification,
-                    pg_constants::GLOBALTABLESPACE_OID,
-                    0,
-                    &direntry.path(),
-                )?;
+                pg_control = Some(import_control_file(writer, lsn, &direntry.path())?);
             }
+            Some("pg_filenode.map") => import_nonrel_file(
+                writer,
+                lsn,
+                RelishTag::FileNodeMap {
+                    spcnode: pg_constants::GLOBALTABLESPACE_OID,
+                    dbnode: 0,
+                },
+                &direntry.path(),
+            )?,
 
-            // Load any relation files into the page server (but only after the other files)
-            _ => relfiles.push(direntry.path()),
+            // Load any relation files into the page server
+            _ => import_relfile(
+                &direntry.path(),
+                writer,
+                lsn,
+                pg_constants::GLOBALTABLESPACE_OID,
+                0,
+            )?,
         }
     }
-    for relfile in relfiles {
-        import_relfile(
-            &mut modification,
-            &relfile,
-            pg_constants::GLOBALTABLESPACE_OID,
-            0,
-        )?;
-    }
 
     // Scan 'base'. It contains database dirs, the database OID is the filename.
     // E.g. 'base/12345', where 12345 is the database OID.
@@ -76,62 +70,60 @@ pub fn import_timeline_from_postgres_datadir<R: Repository>(
         let direntry = direntry?;
 
         //skip all temporary files
-        if direntry.file_name().to_string_lossy() == "pgsql_tmp" {
+        if direntry.file_name().to_str().unwrap() == "pgsql_tmp" {
             continue;
         }
 
-        let dboid = direntry.file_name().to_string_lossy().parse::<u32>()?;
+        let dboid = direntry.file_name().to_str().unwrap().parse::<u32>()?;
 
-        let mut relfiles: Vec<PathBuf> = Vec::new();
         for direntry in fs::read_dir(direntry.path())? {
             let direntry = direntry?;
             match direntry.file_name().to_str() {
                 None => continue,
 
-                Some("PG_VERSION") => {
-                    //modification.put_dbdir_creation(pg_constants::DEFAULTTABLESPACE_OID, dboid)?;
-                }
-                Some("pg_filenode.map") => import_relmap_file(
-                    &mut modification,
-                    pg_constants::DEFAULTTABLESPACE_OID,
-                    dboid,
+                Some("PG_VERSION") => continue,
+                Some("pg_filenode.map") => import_nonrel_file(
+                    writer,
+                    lsn,
+                    RelishTag::FileNodeMap {
+                        spcnode: pg_constants::DEFAULTTABLESPACE_OID,
+                        dbnode: dboid,
+                    },
                     &direntry.path(),
                 )?,
 
                 // Load any relation files into the page server
-                _ => relfiles.push(direntry.path()),
+                _ => import_relfile(
+                    &direntry.path(),
+                    writer,
+                    lsn,
+                    pg_constants::DEFAULTTABLESPACE_OID,
+                    dboid,
+                )?,
             }
         }
-        for relfile in relfiles {
-            import_relfile(
-                &mut modification,
-                &relfile,
-                pg_constants::DEFAULTTABLESPACE_OID,
-                dboid,
-            )?;
-        }
     }
     for entry in fs::read_dir(path.join("pg_xact"))? {
         let entry = entry?;
-        import_slru_file(&mut modification, SlruKind::Clog, &entry.path())?;
+        import_slru_file(writer, lsn, SlruKind::Clog, &entry.path())?;
     }
     for entry in fs::read_dir(path.join("pg_multixact").join("members"))? {
         let entry = entry?;
-        import_slru_file(&mut modification, SlruKind::MultiXactMembers, &entry.path())?;
+        import_slru_file(writer, lsn, SlruKind::MultiXactMembers, &entry.path())?;
     }
     for entry in fs::read_dir(path.join("pg_multixact").join("offsets"))? {
         let entry = entry?;
-        import_slru_file(&mut modification, SlruKind::MultiXactOffsets, &entry.path())?;
+        import_slru_file(writer, lsn, SlruKind::MultiXactOffsets, &entry.path())?;
     }
     for entry in fs::read_dir(path.join("pg_twophase"))? {
         let entry = entry?;
-        let xid = u32::from_str_radix(&entry.path().to_string_lossy(), 16)?;
-        import_twophase_file(&mut modification, xid, &entry.path())?;
+        let xid = u32::from_str_radix(entry.path().to_str().unwrap(), 16)?;
+        import_nonrel_file(writer, lsn, RelishTag::TwoPhase { xid }, &entry.path())?;
     }
     // TODO: Scan pg_tblspc
 
     // We're done importing all the data files.
-    modification.commit()?;
+    writer.advance_last_record_lsn(lsn);
 
     // We expect the Postgres server to be shut down cleanly.
     let pg_control = pg_control.context("pg_control file not found")?;
@@ -149,7 +141,7 @@ pub fn import_timeline_from_postgres_datadir<R: Repository>(
     // *after* the checkpoint record. And crucially, it initializes the 'prev_lsn'.
     import_wal(
         &path.join("pg_wal"),
-        tline,
+        writer,
         Lsn(pg_control.checkPointCopy.redo),
         lsn,
     )?;
@@ -158,53 +150,46 @@ pub fn import_timeline_from_postgres_datadir<R: Repository>(
 }
 
 // subroutine of import_timeline_from_postgres_datadir(), to load one relation file.
-fn import_relfile<R: Repository>(
-    modification: &mut DatadirModification<R>,
+fn import_relfile(
     path: &Path,
+    timeline: &dyn TimelineWriter,
+    lsn: Lsn,
     spcoid: Oid,
     dboid: Oid,
-) -> anyhow::Result<()> {
+) -> Result<()> {
     // Does it look like a relation file?
     trace!("importing rel file {}", path.display());
 
-    let (relnode, forknum, segno) = parse_relfilename(&path.file_name().unwrap().to_string_lossy())
-        .map_err(|e| {
-            warn!("unrecognized file in postgres datadir: {:?} ({})", path, e);
-            e
-        })?;
+    let p = parse_relfilename(path.file_name().unwrap().to_str().unwrap());
+    if let Err(e) = p {
+        warn!("unrecognized file in postgres datadir: {:?} ({})", path, e);
+        return Err(e.into());
+    }
+    let (relnode, forknum, segno) = p.unwrap();
 
     let mut file = File::open(path)?;
     let mut buf: [u8; 8192] = [0u8; 8192];
 
-    let len = file.metadata().unwrap().len();
-    ensure!(len % pg_constants::BLCKSZ as u64 == 0);
-    let nblocks = len / pg_constants::BLCKSZ as u64;
-
-    if segno != 0 {
-        todo!();
-    }
-
-    let rel = RelTag {
-        spcnode: spcoid,
-        dbnode: dboid,
-        relnode,
-        forknum,
-    };
-    modification.put_rel_creation(rel, nblocks as u32)?;
-
     let mut blknum: u32 = segno * (1024 * 1024 * 1024 / pg_constants::BLCKSZ as u32);
     loop {
         let r = file.read_exact(&mut buf);
         match r {
             Ok(_) => {
-                modification.put_rel_page_image(rel, blknum, Bytes::copy_from_slice(&buf))?;
+                let rel = RelTag {
+                    spcnode: spcoid,
+                    dbnode: dboid,
+                    relnode,
+                    forknum,
+                };
+                let tag = RelishTag::Relation(rel);
+                timeline.put_page_image(tag, blknum, lsn, Bytes::copy_from_slice(&buf))?;
             }
 
             // TODO: UnexpectedEof is expected
             Err(err) => match err.kind() {
                 std::io::ErrorKind::UnexpectedEof => {
                     // reached EOF. That's expected.
-                    ensure!(blknum == nblocks as u32, "unexpected EOF");
+                    // FIXME: maybe check that we read the full length of the file?
                     break;
                 }
                 _ => {
@@ -218,28 +203,16 @@ fn import_relfile<R: Repository>(
     Ok(())
 }
 
-/// Import a relmapper (pg_filenode.map) file into the repository
-fn import_relmap_file<R: Repository>(
-    modification: &mut DatadirModification<R>,
-    spcnode: Oid,
-    dbnode: Oid,
-    path: &Path,
-) -> Result<()> {
-    let mut file = File::open(path)?;
-    let mut buffer = Vec::new();
-    // read the whole file
-    file.read_to_end(&mut buffer)?;
-
-    trace!("importing relmap file {}", path.display());
-
-    modification.put_relmap_file(spcnode, dbnode, Bytes::copy_from_slice(&buffer[..]))?;
-    Ok(())
-}
-
-/// Import a twophase state file (pg_twophase/<xid>) into the repository
-fn import_twophase_file<R: Repository>(
-    modification: &mut DatadirModification<R>,
-    xid: TransactionId,
+///
+/// Import a "non-blocky" file into the repository
+///
+/// This is used for small files like the control file, twophase files etc. that
+/// are just slurped into the repository as one blob.
+///
+fn import_nonrel_file(
+    timeline: &dyn TimelineWriter,
+    lsn: Lsn,
+    tag: RelishTag,
     path: &Path,
 ) -> Result<()> {
     let mut file = File::open(path)?;
@@ -249,7 +222,7 @@ fn import_twophase_file<R: Repository>(
 
     trace!("importing non-rel file {}", path.display());
 
-    modification.put_twophase_file(xid, Bytes::copy_from_slice(&buffer[..]))?;
+    timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buffer[..]))?;
     Ok(())
 }
 
@@ -258,8 +231,9 @@ fn import_twophase_file<R: Repository>(
 ///
 /// The control file is imported as is, but we also extract the checkpoint record
 /// from it and store it separated.
-fn import_control_file<R: Repository>(
-    modification: &mut DatadirModification<R>,
+fn import_control_file(
+    timeline: &dyn TimelineWriter,
+    lsn: Lsn,
     path: &Path,
 ) -> Result<ControlFileData> {
     let mut file = File::open(path)?;
@@ -270,12 +244,17 @@ fn import_control_file<R: Repository>(
     trace!("importing control file {}", path.display());
 
     // Import it as ControlFile
-    modification.put_control_file(Bytes::copy_from_slice(&buffer[..]))?;
+    timeline.put_page_image(
+        RelishTag::ControlFile,
+        0,
+        lsn,
+        Bytes::copy_from_slice(&buffer[..]),
+    )?;
 
     // Extract the checkpoint record and import it separately.
     let pg_control = ControlFileData::decode(&buffer)?;
-    let checkpoint_bytes = pg_control.checkPointCopy.encode()?;
-    modification.put_checkpoint(checkpoint_bytes)?;
+    let checkpoint_bytes = pg_control.checkPointCopy.encode();
+    timeline.put_page_image(RelishTag::Checkpoint, 0, lsn, checkpoint_bytes)?;
 
     Ok(pg_control)
 }
@@ -283,34 +262,28 @@ fn import_control_file<R: Repository>(
 ///
 /// Import an SLRU segment file
 ///
-fn import_slru_file<R: Repository>(
-    modification: &mut DatadirModification<R>,
+fn import_slru_file(
+    timeline: &dyn TimelineWriter,
+    lsn: Lsn,
     slru: SlruKind,
     path: &Path,
 ) -> Result<()> {
-    trace!("importing slru file {}", path.display());
-
+    // Does it look like an SLRU file?
     let mut file = File::open(path)?;
     let mut buf: [u8; 8192] = [0u8; 8192];
-    let segno = u32::from_str_radix(&path.file_name().unwrap().to_string_lossy(), 16)?;
+    let segno = u32::from_str_radix(path.file_name().unwrap().to_str().unwrap(), 16)?;
 
-    let len = file.metadata().unwrap().len();
-    ensure!(len % pg_constants::BLCKSZ as u64 == 0); // we assume SLRU block size is the same as BLCKSZ
-    let nblocks = len / pg_constants::BLCKSZ as u64;
-
-    ensure!(nblocks <= pg_constants::SLRU_PAGES_PER_SEGMENT as u64);
-
-    modification.put_slru_segment_creation(slru, segno, nblocks as u32)?;
+    trace!("importing slru file {}", path.display());
 
     let mut rpageno = 0;
     loop {
         let r = file.read_exact(&mut buf);
         match r {
             Ok(_) => {
-                modification.put_slru_page_image(
-                    slru,
-                    segno,
+                timeline.put_page_image(
+                    RelishTag::Slru { slru, segno },
                     rpageno,
+                    lsn,
                     Bytes::copy_from_slice(&buf),
                 )?;
             }
@@ -319,7 +292,7 @@ fn import_slru_file<R: Repository>(
             Err(err) => match err.kind() {
                 std::io::ErrorKind::UnexpectedEof => {
                     // reached EOF. That's expected.
-                    ensure!(rpageno == nblocks as u32, "unexpected EOF");
+                    // FIXME: maybe check that we read the full length of the file?
                     break;
                 }
                 _ => {
@@ -328,6 +301,8 @@ fn import_slru_file<R: Repository>(
             },
         };
         rpageno += 1;
+
+        // TODO: Check that the file isn't unexpectedly large, not larger than SLRU_PAGES_PER_SEGMENT pages
     }
 
     Ok(())
@@ -335,9 +310,9 @@ fn import_slru_file<R: Repository>(
 
 /// Scan PostgreSQL WAL files in given directory and load all records between
 /// 'startpoint' and 'endpoint' into the repository.
-fn import_wal<R: Repository>(
+fn import_wal(
     walpath: &Path,
-    tline: &mut DatadirTimeline<R>,
+    writer: &dyn TimelineWriter,
     startpoint: Lsn,
     endpoint: Lsn,
 ) -> Result<()> {
@@ -347,7 +322,7 @@ fn import_wal<R: Repository>(
     let mut offset = startpoint.segment_offset(pg_constants::WAL_SEGMENT_SIZE);
     let mut last_lsn = startpoint;
 
-    let mut walingest = WalIngest::new(tline, startpoint)?;
+    let mut walingest = WalIngest::new(writer.deref(), startpoint)?;
 
     while last_lsn <= endpoint {
         // FIXME: assume postgresql tli 1 for now
@@ -380,7 +355,7 @@ fn import_wal<R: Repository>(
         let mut nrecords = 0;
         while last_lsn <= endpoint {
             if let Some((lsn, recdata)) = waldecoder.poll_decode()? {
-                walingest.ingest_record(tline, recdata, lsn)?;
+                walingest.ingest_record(writer, recdata, lsn)?;
                 last_lsn = lsn;
 
                 nrecords += 1;

pageserver/src/keyspace.rs

@@ -1,131 +0,0 @@
-use crate::repository::{key_range_size, singleton_range, Key};
-use postgres_ffi::pg_constants;
-use std::ops::Range;
-
-///
-/// Represents a set of Keys, in a compact form.
-///
-#[derive(Clone, Debug)]
-pub struct KeySpace {
-    /// Contiguous ranges of keys that belong to the key space. In key order,
-    /// and with no overlap.
-    pub ranges: Vec<Range<Key>>,
-}
-
-impl KeySpace {
-    ///
-    /// Partition a key space into roughly chunks of roughly 'target_size' bytes
-    /// in each patition.
-    ///
-    pub fn partition(&self, target_size: u64) -> KeyPartitioning {
-        // Assume that each value is 8k in size.
-        let target_nblocks = (target_size / pg_constants::BLCKSZ as u64) as usize;
-
-        let mut parts = Vec::new();
-        let mut current_part = Vec::new();
-        let mut current_part_size: usize = 0;
-        for range in &self.ranges {
-            // If appending the next contiguous range in the keyspace to the current
-            // partition would cause it to be too large, start a new partition.
-            let this_size = key_range_size(range) as usize;
-            if current_part_size + this_size > target_nblocks && !current_part.is_empty() {
-                parts.push(KeySpace {
-                    ranges: current_part,
-                });
-                current_part = Vec::new();
-                current_part_size = 0;
-            }
-
-            // If the next range is larger than 'target_size', split it into
-            // 'target_size' chunks.
-            let mut remain_size = this_size;
-            let mut start = range.start;
-            while remain_size > target_nblocks {
-                let next = start.add(target_nblocks as u32);
-                parts.push(KeySpace {
-                    ranges: vec![start..next],
-                });
-                start = next;
-                remain_size -= target_nblocks
-            }
-            current_part.push(start..range.end);
-            current_part_size += remain_size;
-        }
-
-        // add last partition that wasn't full yet.
-        if !current_part.is_empty() {
-            parts.push(KeySpace {
-                ranges: current_part,
-            });
-        }
-
-        KeyPartitioning { parts }
-    }
-}
-
-///
-/// Represents a partitioning of the key space.
-///
-/// The only kind of partitioning we do is to partition the key space into
-/// partitions that are roughly equal in physical size (see KeySpace::partition).
-/// But this data structure could represent any partitioning.
-///
-#[derive(Clone, Debug, Default)]
-pub struct KeyPartitioning {
-    pub parts: Vec<KeySpace>,
-}
-
-impl KeyPartitioning {
-    pub fn new() -> Self {
-        KeyPartitioning { parts: Vec::new() }
-    }
-}
-
-///
-/// A helper object, to collect a set of keys and key ranges into a KeySpace
-/// object. This takes care of merging adjacent keys and key ranges into
-/// contiguous ranges.
-///
-#[derive(Clone, Debug, Default)]
-pub struct KeySpaceAccum {
-    accum: Option<Range<Key>>,
-
-    ranges: Vec<Range<Key>>,
-}
-
-impl KeySpaceAccum {
-    pub fn new() -> Self {
-        Self {
-            accum: None,
-            ranges: Vec::new(),
-        }
-    }
-
-    pub fn add_key(&mut self, key: Key) {
-        self.add_range(singleton_range(key))
-    }
-
-    pub fn add_range(&mut self, range: Range<Key>) {
-        match self.accum.as_mut() {
-            Some(accum) => {
-                if range.start == accum.end {
-                    accum.end = range.end;
-                } else {
-                    assert!(range.start > accum.end);
-                    self.ranges.push(accum.clone());
-                    *accum = range;
-                }
-            }
-            None => self.accum = Some(range),
-        }
-    }
-
-    pub fn to_keyspace(mut self) -> KeySpace {
-        if let Some(accum) = self.accum.take() {
-            self.ranges.push(accum);
-        }
-        KeySpace {
-            ranges: self.ranges,
-        }
-    }
-}

pageserver/src/layered_repository/README.md

@@ -1,44 +1,40 @@
 # Overview
 
-The main responsibility of the Page Server is to process the incoming WAL, and
-reprocess it into a format that allows reasonably quick access to any page
-version. The page server slices the incoming WAL per relation and page, and
-packages the sliced WAL into suitably-sized "layer files". The layer files
-contain all the history of the database, back to some reasonable retention
-period. This system replaces the base backups and the WAL archive used in a
-traditional PostgreSQL installation. The layer files are immutable, they are not
-modified in-place after creation. New layer files are created for new incoming
-WAL, and old layer files are removed when they are no longer needed.
-
 The on-disk format is based on immutable files. The page server receives a
 stream of incoming WAL, parses the WAL records to determine which pages they
-apply to, and accumulates the incoming changes in memory. Whenever enough WAL
-has been accumulated in memory, it is written out to a new immutable file. That
-process accumulates "L0 delta files" on disk. When enough L0 files have been
-accumulated, they are merged and re-partitioned into L1 files, and old files
-that are no longer needed are removed by Garbage Collection (GC).
+apply to, and accumulates the incoming changes in memory. Every now and then,
+the accumulated changes are written out to new immutable files. This process is
+called checkpointing. Old versions of on-disk files that are not needed by any
+timeline are removed by GC process.
+
+The main responsibility of the Page Server is to process the incoming WAL, and
+reprocess it into a format that allows reasonably quick access to any page
+version.
 
 The incoming WAL contains updates to arbitrary pages in the system. The
 distribution depends on the workload: the updates could be totally random, or
 there could be a long stream of updates to a single relation when data is bulk
-loaded, for example, or something in between.
+loaded, for example, or something in between. The page server slices the
+incoming WAL per relation and page, and packages the sliced WAL into
+suitably-sized "layer files". The layer files contain all the history of the
+database, back to some reasonable retention period. This system replaces the
+base backups and the WAL archive used in a traditional PostgreSQL
+installation. The layer files are immutable, they are not modified in-place
+after creation. New layer files are created for new incoming WAL, and old layer
+files are removed when they are no longer needed. We could also replace layer
+files with new files that contain the same information, merging small files for
+example, but that hasn't been implemented yet.
 
-```
-Cloud Storage                   Page Server                           Safekeeper
-                        L1               L0             Memory            WAL
 
-+----+               +----+----+
-|AAAA|               |AAAA|AAAA|      +---+-----+         |
-+----+               +----+----+      |   |     |         |AA
-|BBBB|               |BBBB|BBBB|      |BB | AA  |         |BB
-+----+----+          +----+----+      |C  | BB  |         |CC
-|CCCC|CCCC|  <----   |CCCC|CCCC| <--- |D  | CC  |  <---   |DDD     <----   ADEBAABED
-+----+----+          +----+----+      |   | DDD |         |E
-|DDDD|DDDD|          |DDDD|DDDD|      |E  |     |         |
-+----+----+          +----+----+      |   |     |
-|EEEE|               |EEEE|EEEE|      +---+-----+
-+----+               +----+----+
-```
+Cloud Storage                   Page Server                   Safekeeper
+                     Local disk                Memory            WAL
+
+|AAAA|               |AAAA|AAAA|               |AA
+|BBBB|               |BBBB|BBBB|               |
+|CCCC|CCCC|  <----   |CCCC|CCCC|CCCC|   <---   |CC     <----   ADEBAABED
+|DDDD|DDDD|          |DDDD|DDDD|               |DDD
+|EEEE|               |EEEE|EEEE|EEEE|          |E
+
 
 In this illustration, WAL is received as a stream from the Safekeeper, from the
 right.  It is immediately captured by the page server and stored quickly in
@@ -46,29 +42,39 @@ memory. The page server memory can be thought of as a quick "reorder buffer",
 used to hold the incoming WAL and reorder it so that we keep the WAL records for
 the same page and relation close to each other.
 
-From the page server memory, whenever enough WAL has been accumulated, it is flushed
-to disk into a new L0 layer file, and the memory is released.
-
-When enough L0 files have been accumulated, they are merged together and sliced
-per key-space, producing a new set of files where each file contains a more
-narrow key range, but larger LSN range.
+From the page server memory, whenever enough WAL has been accumulated for one
+relation segment, it is moved to local disk, as a new layer file, and the memory
+is released.
 
 From the local disk, the layers are further copied to Cloud Storage, for
 long-term archival. After a layer has been copied to Cloud Storage, it can be
 removed from local disk, although we currently keep everything locally for fast
 access. If a layer is needed that isn't found locally, it is fetched from Cloud
-Storage and stored in local disk. L0 and L1 files are both uploaded to Cloud
-Storage.
+Storage and stored in local disk.
+
+# Terms used in layered repository
+
+- Relish - one PostgreSQL relation or similarly treated file.
+- Segment - one slice of a Relish that is stored in a LayeredTimeline.
+- Layer -  specific version of a relish Segment in a range of LSNs.
 
 # Layer map
 
-The LayerMap tracks what layers exist in a timeline.
+The LayerMap tracks what layers exist for all the relishes in a timeline.
+
+LayerMap consists of two data structures:
+- segs - All the layers keyed by segment tag
+- open_layers - data structure that hold all open layers ordered by oldest_pending_lsn for quick access during checkpointing. oldest_pending_lsn is the LSN of the oldest page version stored in this layer.
+
+All operations that update InMemory Layers should update both structures to keep them up-to-date.
+
+- LayeredTimeline - implements Timeline interface.
+
+All methods of LayeredTimeline are aware of its ancestors and return data taking them into account.
+TODO: Are there any exceptions to this?
+For example, timeline.list_rels(lsn) will return all segments that are visible in this timeline at the LSN,
+including ones that were not modified in this timeline and thus don't have a layer in the timeline's LayerMap.
 
-Currently, the layer map is just a resizeable array (Vec). On a GetPage@LSN or
-other read request, the layer map scans through the array to find the right layer
-that contains the data for the requested page. The read-code in LayeredTimeline
-is aware of the ancestor, and returns data from the ancestor timeline if it's
-not found on the current timeline.
 
 # Different kinds of layers
 
@@ -86,11 +92,11 @@ To avoid OOM errors, InMemory layers can be spilled to disk into ephemeral file.
 TODO: Clarify the difference between Closed, Historic and Frozen.
 
 There are two kinds of OnDisk layers:
-- ImageLayer represents a snapshot of all the keys in a particular range, at one
-  particular LSN. Any keys that are not present in the ImageLayer are known not
-  to exist at that LSN.
-- DeltaLayer represents a collection of WAL records or page images in a range of
-  LSNs, for a range of keys.
+- ImageLayer represents an image or a snapshot of a 10 MB relish segment, at one particular LSN.
+- DeltaLayer represents a collection of WAL records or page images in a range of LSNs, for one
+  relish segment.
+
+Dropped segments are always represented on disk by DeltaLayer.
 
 # Layer life cycle
 
@@ -103,83 +109,74 @@ layer or a delta layer, it is a valid end bound. An image layer represents
 snapshot at one LSN, so end_lsn is always the snapshot LSN + 1
 
 Every layer starts its life as an Open In-Memory layer. When the page server
-receives the first WAL record for a timeline, it creates a new In-Memory layer
-for it, and puts it to the layer map. Later, when the layer becomes full, its
-contents are written to disk, as an on-disk layers.
+receives the first WAL record for a segment, it creates a new In-Memory layer
+for it, and puts it to the layer map. Later, the layer is old enough, its
+contents are written to disk, as On-Disk layers. This process is called
+"evicting" a layer.
 
-Flushing a layer is a two-step process: First, the layer is marked as closed, so
-that it no longer accepts new WAL records, and a new in-memory layer is created
-to hold any WAL after that point. After this first step, the layer is a Closed
+Layer eviction is a two-step process: First, the layer is marked as closed, so
+that it no longer accepts new WAL records, and the layer map is updated
+accordingly. If a new WAL record for that segment arrives after this step, a new
+Open layer is created to hold it. After this first step, the layer is a Closed
 InMemory state. This first step is called "freezing" the layer.
 
-In the second step, a new Delta layers is created, containing all the data from
-the Frozen InMemory layer. When it has been created and flushed to disk, the
-original frozen layer is replaced with the new layers in the layer map, and the
-original frozen layer is dropped, releasing the memory.
+In the second step, new Delta and Image layers are created, containing all the
+data in the Frozen InMemory layer. When the new layers are ready, the original
+frozen layer is replaced with the new layers in the layer map, and the original
+frozen layer is dropped, releasing the memory.
 
 # Layer files (On-disk layers)
 
-The files are called "layer files". Each layer file covers a range of keys, and
-a range of LSNs (or a single LSN, in case of image layers). You can think of it
-as a rectangle in the two-dimensional key-LSN space. The layer files for each
-timeline are stored in the timeline's subdirectory under
-`.zenith/tenants/<tenantid>/timelines`.
+The files are called "layer files". Each layer file corresponds
+to one RELISH_SEG_SIZE slice of a PostgreSQL relation fork or
+non-rel file in a range of LSNs. The layer files
+for each timeline are stored in the timeline's subdirectory under
+.zenith/tenants/<tenantid>/timelines.
 
-There are two kind of layer files: images, and delta layers. An image file
-contains a snapshot of all keys at a particular LSN, whereas a delta file
-contains modifications to a segment - mostly in the form of WAL records - in a
-range of LSN.
+There are two kind of layer file: base images, and deltas. A base
+image file contains a layer of a segment as it was at one LSN,
+whereas a delta file contains modifications to a segment - mostly in
+the form of WAL records - in a range of LSN
 
-image file:
+base image file:
 
-```
-    000000067F000032BE0000400000000070B6-000000067F000032BE0000400000000080B6__00000000346BC568
-              start key                          end key                           LSN
-```
-
-
-The first parts define the key range that the layer covers. See
-pgdatadir_mapping.rs for how the key space is used. The last part is the LSN.
+    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>
 
 delta file:
 
-Delta files are named similarly, but they cover a range of LSNs:
+    rel_<spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>_<end LSN>
 
-```
-    000000067F000032BE0000400000000020B6-000000067F000032BE0000400000000030B6__000000578C6B29-0000000057A50051
-              start key                          end key                          start LSN     end LSN
-```
+For example:
 
-A delta file contains all the key-values in the key-range that were updated in
-the LSN range. If a key has not been modified, there is no trace of it in the
-delta layer.
+    rel_1663_13990_2609_0_10_000000000169C348
+    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000
 
+In addition to the relations, with "rel_*" prefix, we use the same
+format for storing various smaller files from the PostgreSQL data
+directory. They will use different suffixes and the naming scheme up
+to the LSNs vary. The Zenith source code uses the term "relish" to
+mean "a relation, or other file that's treated like a relation in the
+storage" For example, a base image of a CLOG segment would be named
+like this:
 
-A delta layer file can cover a part of the overall key space, as in the previous
-example, or the whole key range like this:
+    pg_xact_0000_0_00000000198B06B0
 
-```
-    000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__000000578C6B29-0000000057A50051
-```
+There is no difference in how the relation and non-relation files are
+managed, except that the first part of file names is different.
+Internally, the relations and non-relation files that are managed in
+the versioned store are together called "relishes".
 
-A file that covers the whole key range is called a L0 file (Level 0), while a
-file that covers only part of the key range is called a L1 file. The "level" of
-a file is not explicitly stored anywhere, you can only distinguish them by
-looking at the key range that a file covers. The read-path doesn't need to
-treat L0 and L1 files any differently.
+If a file has been dropped, the last layer file for it is created
+with the _DROPPED suffix, e.g.
+
+    rel_1663_13990_2609_0_10_000000000169C348_0000000001702000_DROPPED
 
 
 ## Notation used in this document
 
-FIXME: This is somewhat obsolete, the layer files cover a key-range rather than
-a particular relation nowadays. However, the description on how you find a page
-version, and how branching and GC works is still valid.
-
 The full path of a delta file looks like this:
 
-```
     .zenith/tenants/941ddc8604413b88b3d208bddf90396c/timelines/4af489b06af8eed9e27a841775616962/rel_1663_13990_2609_0_10_000000000169C348_0000000001702000
-```
 
 For simplicity, the examples below use a simplified notation for the
 paths.  The tenant ID is left out, the timeline ID is replaced with
@@ -188,10 +185,8 @@ with a human-readable table name. The LSNs are also shorter. For
 example, a base image file at LSN 100 and a delta file between 100-200
 for 'orders' table on 'main' branch is represented like this:
 
-```
     main/orders_100
     main/orders_100_200
-```
 
 
 # Creating layer files
@@ -201,14 +196,12 @@ branch called 'main' and two tables, 'orders' and 'customers'. The end
 of WAL is currently at LSN 250. In this starting situation, you would
 have these files on disk:
 
-```
 	main/orders_100
 	main/orders_100_200
 	main/orders_200
 	main/customers_100
 	main/customers_100_200
 	main/customers_200
-```
 
 In addition to those files, the recent changes between LSN 200 and the
 end of WAL at 250 are kept in memory. If the page server crashes, the
@@ -239,7 +232,6 @@ If the customers table is modified later, a new file is created for it
 at the next checkpoint. The new file will cover the "gap" from the
 last layer file, so the LSN ranges are always contiguous:
 
-```
 	main/orders_100
 	main/orders_100_200
 	main/orders_200
@@ -252,7 +244,6 @@ last layer file, so the LSN ranges are always contiguous:
 	main/customers_200
 	main/customers_200_500
 	main/customers_500
-```
 
 ## Reading page versions
 
@@ -276,18 +267,15 @@ involves replaying any WAL records applicable to the page between LSNs
 
 Imagine that a child branch is created at LSN 250:
 
-```
             @250
     ----main--+-------------------------->
                \
                 +---child-------------->
-```
 
 
 Then, the 'orders' table is updated differently on the 'main' and
 'child' branches. You now have this situation on disk:
 
-```
     main/orders_100
     main/orders_100_200
     main/orders_200
@@ -302,7 +290,6 @@ Then, the 'orders' table is updated differently on the 'main' and
     child/orders_300
     child/orders_300_400
     child/orders_400
-```
 
 Because the 'customers' table hasn't been modified on the child
 branch, there is no file for it there. If you request a page for it on
@@ -315,7 +302,6 @@ is linear, and the request's LSN identifies unambiguously which file
 you need to look at. For example, the history for the 'orders' table
 on the 'main' branch consists of these files:
 
-```
     main/orders_100
     main/orders_100_200
     main/orders_200
@@ -323,12 +309,10 @@ on the 'main' branch consists of these files:
     main/orders_300
     main/orders_300_400
     main/orders_400
-```
 
 And from the 'child' branch's point of view, it consists of these
 files:
 
-```
     main/orders_100
     main/orders_100_200
     main/orders_200
@@ -337,7 +321,6 @@ files:
     child/orders_300
     child/orders_300_400
     child/orders_400
-```
 
 The branch metadata includes the point where the child branch was
 created, LSN 250. If a page request comes with LSN 275, we read the
@@ -370,7 +353,6 @@ Let's look at the single branch scenario again. Imagine that the end
 of the branch is LSN 525, so that the GC horizon is currently at
 525-150 = 375
 
-```
 	main/orders_100
 	main/orders_100_200
 	main/orders_200
@@ -383,13 +365,11 @@ of the branch is LSN 525, so that the GC horizon is currently at
 	main/customers_100
 	main/customers_100_200
 	main/customers_200
-```
 
 We can remove the following files because the end LSNs of those files are
 older than GC horizon 375, and there are more recent layer files for the
 table:
 
-```
 	main/orders_100       DELETE
 	main/orders_100_200   DELETE
 	main/orders_200       DELETE
@@ -402,9 +382,8 @@ table:
 	main/customers_100      DELETE
 	main/customers_100_200  DELETE
 	main/customers_200      KEEP, NO NEWER VERSION
-```
 
-'main/customers_200' is old enough, but it cannot be
+'main/customers_100_200' is old enough, but it cannot be
 removed because there is no newer layer file for the table.
 
 Things get slightly more complicated with multiple branches. All of
@@ -413,7 +392,6 @@ retain older shapshot files that are still needed by child branches.
 For example, if child branch is created at LSN 150, and the 'customers'
 table is updated on the branch, you would have these files:
 
-```
 	main/orders_100        KEEP, NEEDED BY child BRANCH
 	main/orders_100_200    KEEP, NEEDED BY child BRANCH
 	main/orders_200        DELETE
@@ -428,7 +406,6 @@ table is updated on the branch, you would have these files:
 	main/customers_200       KEEP, NO NEWER VERSION
 	child/customers_150_300  DELETE
 	child/customers_300      KEEP, NO NEWER VERSION
-```
 
 In this situation, 'main/orders_100' and 'main/orders_100_200' cannot
 be removed, even though they are older than the GC horizon, because
@@ -438,7 +415,6 @@ and 'main/orders_200_300' can still be removed.
 If 'orders' is modified later on the 'child' branch, we will create a
 new base image and delta file for it on the child:
 
-```
 	main/orders_100
 	main/orders_100_200
 
@@ -451,7 +427,6 @@ new base image and delta file for it on the child:
 	child/customers_300
 	child/orders_150_400
 	child/orders_400
-```
 
 After this, the 'main/orders_100' and 'main/orders_100_200' file could
 be removed. It is no longer needed by the child branch, because there
@@ -467,7 +442,6 @@ Describe GC and checkpoint interval settings.
 In principle, each relation can be checkpointed separately, i.e. the
 LSN ranges of the files don't need to line up. So this would be legal:
 
-```
 	main/orders_100
 	main/orders_100_200
 	main/orders_200
@@ -480,7 +454,6 @@ LSN ranges of the files don't need to line up. So this would be legal:
 	main/customers_250
 	main/customers_250_500
 	main/customers_500
-```
 
 However, the code currently always checkpoints all relations together.
 So that situation doesn't arise in practice.
@@ -503,13 +476,11 @@ does that.  It could be useful, however, as a transient state when
 garbage collecting around branch points, or explicit recovery
 points. For example, if we start with this:
 
-```
 	main/orders_100
 	main/orders_100_200
 	main/orders_200
 	main/orders_200_300
 	main/orders_300
-```
 
 And there is a branch or explicit recovery point at LSN 150, we could
 replace 'main/orders_100_200' with 'main/orders_150' to keep a

pageserver/src/layered_repository/blob_io.rs

@@ -1,173 +0,0 @@
-//!
-//! Functions for reading and writing variable-sized "blobs".
-//!
-//! Each blob begins with a 1- or 4-byte length field, followed by the
-//! actual data. If the length is smaller than 128 bytes, the length
-//! is written as a one byte. If it's larger than that, the length
-//! is written as a four-byte integer, in big-endian, with the high
-//! bit set. This way, we can detect whether it's 1- or 4-byte header
-//! by peeking at the first byte.
-//!
-//! len <  128: 0XXXXXXX
-//! len >= 128: 1XXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX
-//!
-use crate::layered_repository::block_io::{BlockCursor, BlockReader};
-use crate::page_cache::PAGE_SZ;
-use std::cmp::min;
-use std::io::{Error, ErrorKind};
-
-/// For reading
-pub trait BlobCursor {
-    /// Read a blob into a new buffer.
-    fn read_blob(&mut self, offset: u64) -> Result<Vec<u8>, std::io::Error> {
-        let mut buf = Vec::new();
-        self.read_blob_into_buf(offset, &mut buf)?;
-        Ok(buf)
-    }
-
-    /// Read blob into the given buffer. Any previous contents in the buffer
-    /// are overwritten.
-    fn read_blob_into_buf(
-        &mut self,
-        offset: u64,
-        dstbuf: &mut Vec<u8>,
-    ) -> Result<(), std::io::Error>;
-}
-
-impl<'a, R> BlobCursor for BlockCursor<R>
-where
-    R: BlockReader,
-{
-    fn read_blob_into_buf(
-        &mut self,
-        offset: u64,
-        dstbuf: &mut Vec<u8>,
-    ) -> Result<(), std::io::Error> {
-        let mut blknum = (offset / PAGE_SZ as u64) as u32;
-        let mut off = (offset % PAGE_SZ as u64) as usize;
-
-        let mut buf = self.read_blk(blknum)?;
-
-        // peek at the first byte, to determine if it's a 1- or 4-byte length
-        let first_len_byte = buf[off];
-        let len: usize = if first_len_byte < 0x80 {
-            // 1-byte length header
-            off += 1;
-            first_len_byte as usize
-        } else {
-            // 4-byte length header
-            let mut len_buf = [0u8; 4];
-            let thislen = PAGE_SZ - off;
-            if thislen < 4 {
-                // it is split across two pages
-                len_buf[..thislen].copy_from_slice(&buf[off..PAGE_SZ]);
-                blknum += 1;
-                buf = self.read_blk(blknum)?;
-                len_buf[thislen..].copy_from_slice(&buf[0..4 - thislen]);
-                off = 4 - thislen;
-            } else {
-                len_buf.copy_from_slice(&buf[off..off + 4]);
-                off += 4;
-            }
-            len_buf[0] &= 0x7f;
-            u32::from_be_bytes(len_buf) as usize
-        };
-
-        dstbuf.clear();
-
-        // Read the payload
-        let mut remain = len;
-        while remain > 0 {
-            let mut page_remain = PAGE_SZ - off;
-            if page_remain == 0 {
-                // continue on next page
-                blknum += 1;
-                buf = self.read_blk(blknum)?;
-                off = 0;
-                page_remain = PAGE_SZ;
-            }
-            let this_blk_len = min(remain, page_remain);
-            dstbuf.extend_from_slice(&buf[off..off + this_blk_len]);
-            remain -= this_blk_len;
-            off += this_blk_len;
-        }
-        Ok(())
-    }
-}
-
-///
-/// Abstract trait for a data sink that you can write blobs to.
-///
-pub trait BlobWriter {
-    /// Write a blob of data. Returns the offset that it was written to,
-    /// which can be used to retrieve the data later.
-    fn write_blob(&mut self, srcbuf: &[u8]) -> Result<u64, Error>;
-}
-
-///
-/// An implementation of BlobWriter to write blobs to anything that
-/// implements std::io::Write.
-///
-pub struct WriteBlobWriter<W>
-where
-    W: std::io::Write,
-{
-    inner: W,
-    offset: u64,
-}
-
-impl<W> WriteBlobWriter<W>
-where
-    W: std::io::Write,
-{
-    pub fn new(inner: W, start_offset: u64) -> Self {
-        WriteBlobWriter {
-            inner,
-            offset: start_offset,
-        }
-    }
-
-    pub fn size(&self) -> u64 {
-        self.offset
-    }
-
-    /// Access the underlying Write object.
-    ///
-    /// NOTE: WriteBlobWriter keeps track of the current write offset. If
-    /// you write something directly to the inner Write object, it makes the
-    /// internally tracked 'offset' to go out of sync. So don't do that.
-    pub fn into_inner(self) -> W {
-        self.inner
-    }
-}
-
-impl<W> BlobWriter for WriteBlobWriter<W>
-where
-    W: std::io::Write,
-{
-    fn write_blob(&mut self, srcbuf: &[u8]) -> Result<u64, Error> {
-        let offset = self.offset;
-
-        if srcbuf.len() < 128 {
-            // Short blob. Write a 1-byte length header
-            let len_buf = srcbuf.len() as u8;
-            self.inner.write_all(&[len_buf])?;
-            self.offset += 1;
-        } else {
-            // Write a 4-byte length header
-            if srcbuf.len() > 0x7fff_ffff {
-                return Err(Error::new(
-                    ErrorKind::Other,
-                    format!("blob too large ({} bytes)", srcbuf.len()),
-                ));
-            }
-            let mut len_buf = ((srcbuf.len()) as u32).to_be_bytes();
-            len_buf[0] |= 0x80;
-            self.inner.write_all(&len_buf)?;
-            self.offset += 4;
-        }
-        self.inner.write_all(srcbuf)?;
-        self.offset += srcbuf.len() as u64;
-        Ok(offset)
-    }
-}

pageserver/src/layered_repository/block_io.rs

@@ -1,218 +0,0 @@
-//!
-//! Low-level Block-oriented I/O functions
-//!
-
-use crate::page_cache;
-use crate::page_cache::{ReadBufResult, PAGE_SZ};
-use bytes::Bytes;
-use lazy_static::lazy_static;
-use std::ops::{Deref, DerefMut};
-use std::os::unix::fs::FileExt;
-use std::sync::atomic::AtomicU64;
-
-/// This is implemented by anything that can read 8 kB (PAGE_SZ)
-/// blocks, using the page cache
-///
-/// There are currently two implementations: EphemeralFile, and FileBlockReader
-/// below.
-pub trait BlockReader {
-    type BlockLease: Deref<Target = [u8; PAGE_SZ]> + 'static;
-
-    ///
-    /// Read a block. Returns a "lease" object that can be used to
-    /// access to the contents of the page. (For the page cache, the
-    /// lease object represents a lock on the buffer.)
-    ///
-    fn read_blk(&self, blknum: u32) -> Result<Self::BlockLease, std::io::Error>;
-
-    ///
-    /// Create a new "cursor" for reading from this reader.
-    ///
-    /// A cursor caches the last accessed page, allowing for faster
-    /// access if the same block is accessed repeatedly.
-    fn block_cursor(&self) -> BlockCursor<&Self>
-    where
-        Self: Sized,
-    {
-        BlockCursor::new(self)
-    }
-}
-
-impl<B> BlockReader for &B
-where
-    B: BlockReader,
-{
-    type BlockLease = B::BlockLease;
-
-    fn read_blk(&self, blknum: u32) -> Result<Self::BlockLease, std::io::Error> {
-        (*self).read_blk(blknum)
-    }
-}
-
-///
-/// A "cursor" for efficiently reading multiple pages from a BlockReader
-///
-/// A cursor caches the last accessed page, allowing for faster access if the
-/// same block is accessed repeatedly.
-///
-/// You can access the last page with `*cursor`. 'read_blk' returns 'self', so
-/// that in many cases you can use a BlockCursor as a drop-in replacement for
-/// the underlying BlockReader. For example:
-///
-/// ```no_run
-/// # use pageserver::layered_repository::block_io::{BlockReader, FileBlockReader};
-/// # let reader: FileBlockReader<std::fs::File> = todo!();
-/// let cursor = reader.block_cursor();
-/// let buf = cursor.read_blk(1);
-/// // do stuff with 'buf'
-/// let buf = cursor.read_blk(2);
-/// // do stuff with 'buf'
-/// ```
-///
-pub struct BlockCursor<R>
-where
-    R: BlockReader,
-{
-    reader: R,
-    /// last accessed page
-    cache: Option<(u32, R::BlockLease)>,
-}
-
-impl<R> BlockCursor<R>
-where
-    R: BlockReader,
-{
-    pub fn new(reader: R) -> Self {
-        BlockCursor {
-            reader,
-            cache: None,
-        }
-    }
-
-    pub fn read_blk(&mut self, blknum: u32) -> Result<&Self, std::io::Error> {
-        // Fast return if this is the same block as before
-        if let Some((cached_blk, _buf)) = &self.cache {
-            if *cached_blk == blknum {
-                return Ok(self);
-            }
-        }
-
-        // Read the block from the underlying reader, and cache it
-        self.cache = None;
-        let buf = self.reader.read_blk(blknum)?;
-        self.cache = Some((blknum, buf));
-
-        Ok(self)
-    }
-}
-
-impl<R> Deref for BlockCursor<R>
-where
-    R: BlockReader,
-{
-    type Target = [u8; PAGE_SZ];
-
-    fn deref(&self) -> &<Self as Deref>::Target {
-        &self.cache.as_ref().unwrap().1
-    }
-}
-
-lazy_static! {
-    static ref NEXT_ID: AtomicU64 = AtomicU64::new(1);
-}
-
-/// An adapter for reading a (virtual) file using the page cache.
-///
-/// The file is assumed to be immutable. This doesn't provide any functions
-/// for modifying the file, nor for invalidating the cache if it is modified.
-pub struct FileBlockReader<F> {
-    pub file: F,
-
-    /// Unique ID of this file, used as key in the page cache.
-    file_id: u64,
-}
-
-impl<F> FileBlockReader<F>
-where
-    F: FileExt,
-{
-    pub fn new(file: F) -> Self {
-        let file_id = NEXT_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-
-        FileBlockReader { file_id, file }
-    }
-
-    /// Read a page from the underlying file into given buffer.
-    fn fill_buffer(&self, buf: &mut [u8], blkno: u32) -> Result<(), std::io::Error> {
-        assert!(buf.len() == PAGE_SZ);
-        self.file.read_exact_at(buf, blkno as u64 * PAGE_SZ as u64)
-    }
-}
-
-impl<F> BlockReader for FileBlockReader<F>
-where
-    F: FileExt,
-{
-    type BlockLease = page_cache::PageReadGuard<'static>;
-
-    fn read_blk(&self, blknum: u32) -> Result<Self::BlockLease, std::io::Error> {
-        // Look up the right page
-        let cache = page_cache::get();
-        loop {
-            match cache.read_immutable_buf(self.file_id, blknum) {
-                ReadBufResult::Found(guard) => break Ok(guard),
-                ReadBufResult::NotFound(mut write_guard) => {
-                    // Read the page from disk into the buffer
-                    self.fill_buffer(write_guard.deref_mut(), blknum)?;
-                    write_guard.mark_valid();
-
-                    // Swap for read lock
-                    continue;
-                }
-            };
-        }
-    }
-}
-
-///
-/// Trait for block-oriented output
-///
-pub trait BlockWriter {
-    ///
-    /// Write a page to the underlying storage.
-    ///
-    /// 'buf' must be of size PAGE_SZ. Returns the block number the page was
-    /// written to.
-    ///
-    fn write_blk(&mut self, buf: Bytes) -> Result<u32, std::io::Error>;
-}
-
-///
-/// A simple in-memory buffer of blocks.
-///
-pub struct BlockBuf {
-    pub blocks: Vec<Bytes>,
-}
-impl BlockWriter for BlockBuf {
-    fn write_blk(&mut self, buf: Bytes) -> Result<u32, std::io::Error> {
-        assert!(buf.len() == PAGE_SZ);
-        let blknum = self.blocks.len();
-        self.blocks.push(buf);
-        Ok(blknum as u32)
-    }
-}
-
-impl BlockBuf {
-    pub fn new() -> Self {
-        BlockBuf { blocks: Vec::new() }
-    }
-
-    pub fn size(&self) -> u64 {
-        (self.blocks.len() * PAGE_SZ) as u64
-    }
-}
-impl Default for BlockBuf {
-    fn default() -> Self {
-        Self::new()
-    }
-}

pageserver/src/layered_repository/ephemeral_file.rs

@@ -2,8 +2,6 @@
 //! used to keep in-memory layers spilled on disk.
 
 use crate::config::PageServerConf;
-use crate::layered_repository::blob_io::BlobWriter;
-use crate::layered_repository::block_io::BlockReader;
 use crate::page_cache;
 use crate::page_cache::PAGE_SZ;
 use crate::page_cache::{ReadBufResult, WriteBufResult};
@@ -12,12 +10,12 @@ use lazy_static::lazy_static;
 use std::cmp::min;
 use std::collections::HashMap;
 use std::fs::OpenOptions;
-use std::io::{Error, ErrorKind};
+use std::io::{Error, ErrorKind, Seek, SeekFrom, Write};
 use std::ops::DerefMut;
 use std::path::PathBuf;
 use std::sync::{Arc, RwLock};
-use tracing::*;
-use utils::zid::{ZTenantId, ZTimelineId};
+use zenith_utils::zid::ZTenantId;
+use zenith_utils::zid::ZTimelineId;
 
 use std::os::unix::fs::FileExt;
 
@@ -43,7 +41,7 @@ pub struct EphemeralFile {
     _timelineid: ZTimelineId,
     file: Arc<VirtualFile>,
 
-    size: u64,
+    pos: u64,
 }
 
 impl EphemeralFile {
@@ -72,11 +70,11 @@ impl EphemeralFile {
             _tenantid: tenantid,
             _timelineid: timelineid,
             file: file_rc,
-            size: 0,
+            pos: 0,
         })
     }
 
-    fn fill_buffer(&self, buf: &mut [u8], blkno: u32) -> Result<(), Error> {
+    pub fn fill_buffer(&self, buf: &mut [u8], blkno: u32) -> Result<(), Error> {
         let mut off = 0;
         while off < PAGE_SZ {
             let n = self
@@ -95,26 +93,6 @@ impl EphemeralFile {
         }
         Ok(())
     }
-
-    fn get_buf_for_write(&self, blkno: u32) -> Result<page_cache::PageWriteGuard, Error> {
-        // Look up the right page
-        let cache = page_cache::get();
-        let mut write_guard = match cache.write_ephemeral_buf(self.file_id, blkno) {
-            WriteBufResult::Found(guard) => guard,
-            WriteBufResult::NotFound(mut guard) => {
-                // Read the page from disk into the buffer
-                // TODO: if we're overwriting the whole page, no need to read it in first
-                self.fill_buffer(guard.deref_mut(), blkno)?;
-                guard.mark_valid();
-
-                // And then fall through to modify it.
-                guard
-            }
-        };
-        write_guard.mark_dirty();
-
-        Ok(write_guard)
-    }
 }
 
 /// Does the given filename look like an ephemeral file?
@@ -189,61 +167,48 @@ impl FileExt for EphemeralFile {
     }
 }
 
-impl BlobWriter for EphemeralFile {
-    fn write_blob(&mut self, srcbuf: &[u8]) -> Result<u64, Error> {
-        let pos = self.size;
+impl Write for EphemeralFile {
+    fn write(&mut self, buf: &[u8]) -> Result<usize, Error> {
+        let n = self.write_at(buf, self.pos)?;
+        self.pos += n as u64;
+        Ok(n)
+    }
 
-        let mut blknum = (self.size / PAGE_SZ as u64) as u32;
-        let mut off = (pos % PAGE_SZ as u64) as usize;
+    fn flush(&mut self) -> Result<(), std::io::Error> {
+        // we don't need to flush data:
+        // * we either write input bytes or not, not keeping any intermediate data buffered
+        // * rust unix file `flush` impl does not flush things either, returning `Ok(())`
+        Ok(())
+    }
+}
 
-        let mut buf = self.get_buf_for_write(blknum)?;
-
-        // Write the length field
-        if srcbuf.len() < 0x80 {
-            buf[off] = srcbuf.len() as u8;
-            off += 1;
-        } else {
-            let mut len_buf = u32::to_be_bytes(srcbuf.len() as u32);
-            len_buf[0] |= 0x80;
-            let thislen = PAGE_SZ - off;
-            if thislen < 4 {
-                // it needs to be split across pages
-                buf[off..(off + thislen)].copy_from_slice(&len_buf[..thislen]);
-                blknum += 1;
-                buf = self.get_buf_for_write(blknum)?;
-                buf[0..4 - thislen].copy_from_slice(&len_buf[thislen..]);
-                off = 4 - thislen;
-            } else {
-                buf[off..off + 4].copy_from_slice(&len_buf);
-                off += 4;
+impl Seek for EphemeralFile {
+    fn seek(&mut self, pos: SeekFrom) -> Result<u64, Error> {
+        match pos {
+            SeekFrom::Start(offset) => {
+                self.pos = offset;
+            }
+            SeekFrom::End(_offset) => {
+                return Err(Error::new(
+                    ErrorKind::Other,
+                    "SeekFrom::End not supported by EphemeralFile",
+                ));
+            }
+            SeekFrom::Current(offset) => {
+                let pos = self.pos as i128 + offset as i128;
+                if pos < 0 {
+                    return Err(Error::new(
+                        ErrorKind::InvalidInput,
+                        "offset would be negative",
+                    ));
+                }
+                if pos > u64::MAX as i128 {
+                    return Err(Error::new(ErrorKind::InvalidInput, "offset overflow"));
+                }
+                self.pos = pos as u64;
             }
         }
-
-        // Write the payload
-        let mut buf_remain = srcbuf;
-        while !buf_remain.is_empty() {
-            let mut page_remain = PAGE_SZ - off;
-            if page_remain == 0 {
-                blknum += 1;
-                buf = self.get_buf_for_write(blknum)?;
-                off = 0;
-                page_remain = PAGE_SZ;
-            }
-            let this_blk_len = min(page_remain, buf_remain.len());
-            buf[off..(off + this_blk_len)].copy_from_slice(&buf_remain[..this_blk_len]);
-            off += this_blk_len;
-            buf_remain = &buf_remain[this_blk_len..];
-        }
-        drop(buf);
-
-        if srcbuf.len() < 0x80 {
-            self.size += 1;
-        } else {
-            self.size += 4;
-        }
-        self.size += srcbuf.len() as u64;
-
-        Ok(pos)
+        Ok(self.pos)
     }
 }
 
@@ -256,31 +221,16 @@ impl Drop for EphemeralFile {
         // remove entry from the hash map
         EPHEMERAL_FILES.write().unwrap().files.remove(&self.file_id);
 
-        // unlink the file
-        let res = std::fs::remove_file(&self.file.path);
-        if let Err(e) = res {
-            warn!(
-                "could not remove ephemeral file '{}': {}",
-                self.file.path.display(),
-                e
-            );
-        }
+        // unlink file
+        // FIXME: print error
+        let _ = std::fs::remove_file(&self.file.path);
     }
 }
 
 pub fn writeback(file_id: u64, blkno: u32, buf: &[u8]) -> Result<(), std::io::Error> {
     if let Some(file) = EPHEMERAL_FILES.read().unwrap().files.get(&file_id) {
-        match file.write_all_at(buf, blkno as u64 * PAGE_SZ as u64) {
-            Ok(_) => Ok(()),
-            Err(e) => Err(std::io::Error::new(
-                ErrorKind::Other,
-                format!(
-                    "failed to write back to ephemeral file at {} error: {}",
-                    file.path.display(),
-                    e
-                ),
-            )),
-        }
+        file.write_all_at(buf, blkno as u64 * PAGE_SZ as u64)?;
+        Ok(())
     } else {
         Err(std::io::Error::new(
             ErrorKind::Other,
@@ -289,34 +239,11 @@ pub fn writeback(file_id: u64, blkno: u32, buf: &[u8]) -> Result<(), std::io::Er
     }
 }
 
-impl BlockReader for EphemeralFile {
-    type BlockLease = page_cache::PageReadGuard<'static>;
-
-    fn read_blk(&self, blknum: u32) -> Result<Self::BlockLease, std::io::Error> {
-        // Look up the right page
-        let cache = page_cache::get();
-        loop {
-            match cache.read_ephemeral_buf(self.file_id, blknum) {
-                ReadBufResult::Found(guard) => return Ok(guard),
-                ReadBufResult::NotFound(mut write_guard) => {
-                    // Read the page from disk into the buffer
-                    self.fill_buffer(write_guard.deref_mut(), blknum)?;
-                    write_guard.mark_valid();
-
-                    // Swap for read lock
-                    continue;
-                }
-            };
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::layered_repository::blob_io::{BlobCursor, BlobWriter};
-    use crate::layered_repository::block_io::BlockCursor;
-    use rand::{seq::SliceRandom, thread_rng, RngCore};
+    use rand::seq::SliceRandom;
+    use rand::thread_rng;
     use std::fs;
     use std::str::FromStr;
 
@@ -354,19 +281,19 @@ mod tests {
     fn test_ephemeral_files() -> Result<(), Error> {
         let (conf, tenantid, timelineid) = repo_harness("ephemeral_files")?;
 
-        let file_a = EphemeralFile::create(conf, tenantid, timelineid)?;
+        let mut file_a = EphemeralFile::create(conf, tenantid, timelineid)?;
 
-        file_a.write_all_at(b"foo", 0)?;
+        file_a.write_all(b"foo")?;
         assert_eq!("foo", read_string(&file_a, 0, 20)?);
 
-        file_a.write_all_at(b"bar", 3)?;
+        file_a.write_all(b"bar")?;
         assert_eq!("foobar", read_string(&file_a, 0, 20)?);
 
         // Open a lot of files, enough to cause some page evictions.
         let mut efiles = Vec::new();
         for fileno in 0..100 {
-            let efile = EphemeralFile::create(conf, tenantid, timelineid)?;
-            efile.write_all_at(format!("file {}", fileno).as_bytes(), 0)?;
+            let mut efile = EphemeralFile::create(conf, tenantid, timelineid)?;
+            efile.write_all(format!("file {}", fileno).as_bytes())?;
             assert_eq!(format!("file {}", fileno), read_string(&efile, 0, 10)?);
             efiles.push((fileno, efile));
         }
@@ -380,47 +307,4 @@ mod tests {
 
         Ok(())
     }
-
-    #[test]
-    fn test_ephemeral_blobs() -> Result<(), Error> {
-        let (conf, tenantid, timelineid) = repo_harness("ephemeral_blobs")?;
-
-        let mut file = EphemeralFile::create(conf, tenantid, timelineid)?;
-
-        let pos_foo = file.write_blob(b"foo")?;
-        assert_eq!(b"foo", file.block_cursor().read_blob(pos_foo)?.as_slice());
-        let pos_bar = file.write_blob(b"bar")?;
-        assert_eq!(b"foo", file.block_cursor().read_blob(pos_foo)?.as_slice());
-        assert_eq!(b"bar", file.block_cursor().read_blob(pos_bar)?.as_slice());
-
-        let mut blobs = Vec::new();
-        for i in 0..10000 {
-            let data = Vec::from(format!("blob{}", i).as_bytes());
-            let pos = file.write_blob(&data)?;
-            blobs.push((pos, data));
-        }
-        // also test with a large blobs
-        for i in 0..100 {
-            let data = format!("blob{}", i).as_bytes().repeat(100);
-            let pos = file.write_blob(&data)?;
-            blobs.push((pos, data));
-        }
-
-        let mut cursor = BlockCursor::new(&file);
-        for (pos, expected) in blobs {
-            let actual = cursor.read_blob(pos)?;
-            assert_eq!(actual, expected);
-        }
-        drop(cursor);
-
-        // Test a large blob that spans multiple pages
-        let mut large_data = Vec::new();
-        large_data.resize(20000, 0);
-        thread_rng().fill_bytes(&mut large_data);
-        let pos_large = file.write_blob(&large_data)?;
-        let result = file.block_cursor().read_blob(pos_large)?;
-        assert_eq!(result, large_data);
-
-        Ok(())
-    }
 }

pageserver/src/layered_repository/filename.rs

@@ -2,50 +2,29 @@
 //! Helper functions for dealing with filenames of the image and delta layer files.
 //!
 use crate::config::PageServerConf;
-use crate::repository::Key;
-use std::cmp::Ordering;
+use crate::layered_repository::storage_layer::SegmentTag;
+use crate::relish::*;
 use std::fmt;
-use std::ops::Range;
 use std::path::PathBuf;
 
-use utils::lsn::Lsn;
+use zenith_utils::lsn::Lsn;
 
 // Note: LayeredTimeline::load_layer_map() relies on this sort order
-#[derive(Debug, PartialEq, Eq, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
 pub struct DeltaFileName {
-    pub key_range: Range<Key>,
-    pub lsn_range: Range<Lsn>,
-}
-
-impl PartialOrd for DeltaFileName {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Ord for DeltaFileName {
-    fn cmp(&self, other: &Self) -> Ordering {
-        let mut cmp = self.key_range.start.cmp(&other.key_range.start);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.key_range.end.cmp(&other.key_range.end);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.lsn_range.start.cmp(&other.lsn_range.start);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.lsn_range.end.cmp(&other.lsn_range.end);
-
-        cmp
-    }
+    pub seg: SegmentTag,
+    pub start_lsn: Lsn,
+    pub end_lsn: Lsn,
+    pub dropped: bool,
 }
 
 /// Represents the filename of a DeltaLayer
 ///
-///    <key start>-<key end>__<LSN start>-<LSN end>
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>
+///
+/// or if it was dropped:
+///
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>_DROPPED
 ///
 impl DeltaFileName {
     ///
@@ -53,121 +32,234 @@ impl DeltaFileName {
     /// match the expected pattern.
     ///
     pub fn parse_str(fname: &str) -> Option<Self> {
-        let mut parts = fname.split("__");
-        let mut key_parts = parts.next()?.split('-');
-        let mut lsn_parts = parts.next()?.split('-');
-
-        let key_start_str = key_parts.next()?;
-        let key_end_str = key_parts.next()?;
-        let lsn_start_str = lsn_parts.next()?;
-        let lsn_end_str = lsn_parts.next()?;
-        if parts.next().is_some() || key_parts.next().is_some() || key_parts.next().is_some() {
+        let rel;
+        let mut parts;
+        if let Some(rest) = fname.strip_prefix("rel_") {
+            parts = rest.split('_');
+            rel = RelishTag::Relation(RelTag {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+                relnode: parts.next()?.parse::<u32>().ok()?,
+                forknum: parts.next()?.parse::<u8>().ok()?,
+            });
+        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
+            parts = rest.split('_');
+            rel = RelishTag::FileNodeMap {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
+            parts = rest.split('_');
+            rel = RelishTag::TwoPhase {
+                xid: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
+            parts = rest.split('_');
+            rel = RelishTag::Checkpoint;
+        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
+            parts = rest.split('_');
+            rel = RelishTag::ControlFile;
+        } else {
             return None;
         }
 
-        let key_start = Key::from_hex(key_start_str).ok()?;
-        let key_end = Key::from_hex(key_end_str).ok()?;
+        let segno = parts.next()?.parse::<u32>().ok()?;
 
-        let start_lsn = Lsn::from_hex(lsn_start_str).ok()?;
-        let end_lsn = Lsn::from_hex(lsn_end_str).ok()?;
+        let seg = SegmentTag { rel, segno };
 
-        if start_lsn >= end_lsn {
-            return None;
-            // or panic?
+        let start_lsn = Lsn::from_hex(parts.next()?).ok()?;
+        let end_lsn = Lsn::from_hex(parts.next()?).ok()?;
+
+        let mut dropped = false;
+        if let Some(suffix) = parts.next() {
+            if suffix == "DROPPED" {
+                dropped = true;
+            } else {
+                return None;
+            }
         }
-
-        if key_start >= key_end {
+        if parts.next().is_some() {
             return None;
-            // or panic?
         }
 
         Some(DeltaFileName {
-            key_range: key_start..key_end,
-            lsn_range: start_lsn..end_lsn,
+            seg,
+            start_lsn,
+            end_lsn,
+            dropped,
         })
     }
 }
 
 impl fmt::Display for DeltaFileName {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let basename = match self.seg.rel {
+            RelishTag::Relation(reltag) => format!(
+                "rel_{}_{}_{}_{}",
+                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
+            ),
+            RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno,
+            } => format!("pg_xact_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno,
+            } => format!("pg_multixact_members_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno,
+            } => format!("pg_multixact_offsets_{:04X}", segno),
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
+            RelishTag::Checkpoint => "pg_control_checkpoint".to_string(),
+            RelishTag::ControlFile => "pg_control".to_string(),
+        };
+
         write!(
             f,
-            "{}-{}__{:016X}-{:016X}",
-            self.key_range.start,
-            self.key_range.end,
-            u64::from(self.lsn_range.start),
-            u64::from(self.lsn_range.end),
+            "{}_{}_{:016X}_{:016X}{}",
+            basename,
+            self.seg.segno,
+            u64::from(self.start_lsn),
+            u64::from(self.end_lsn),
+            if self.dropped { "_DROPPED" } else { "" }
         )
     }
 }
 
-#[derive(Debug, PartialEq, Eq, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
 pub struct ImageFileName {
-    pub key_range: Range<Key>,
+    pub seg: SegmentTag,
     pub lsn: Lsn,
 }
 
-impl PartialOrd for ImageFileName {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Ord for ImageFileName {
-    fn cmp(&self, other: &Self) -> Ordering {
-        let mut cmp = self.key_range.start.cmp(&other.key_range.start);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.key_range.end.cmp(&other.key_range.end);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.lsn.cmp(&other.lsn);
-
-        cmp
-    }
-}
-
 ///
 /// Represents the filename of an ImageLayer
 ///
-///    <key start>-<key end>__<LSN>
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<LSN>
+///
 impl ImageFileName {
     ///
     /// Parse a string as an image file name. Returns None if the filename does not
     /// match the expected pattern.
     ///
     pub fn parse_str(fname: &str) -> Option<Self> {
-        let mut parts = fname.split("__");
-        let mut key_parts = parts.next()?.split('-');
-
-        let key_start_str = key_parts.next()?;
-        let key_end_str = key_parts.next()?;
-        let lsn_str = parts.next()?;
-        if parts.next().is_some() || key_parts.next().is_some() {
+        let rel;
+        let mut parts;
+        if let Some(rest) = fname.strip_prefix("rel_") {
+            parts = rest.split('_');
+            rel = RelishTag::Relation(RelTag {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+                relnode: parts.next()?.parse::<u32>().ok()?,
+                forknum: parts.next()?.parse::<u8>().ok()?,
+            });
+        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
+            parts = rest.split('_');
+            rel = RelishTag::FileNodeMap {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
+            parts = rest.split('_');
+            rel = RelishTag::TwoPhase {
+                xid: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
+            parts = rest.split('_');
+            rel = RelishTag::Checkpoint;
+        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
+            parts = rest.split('_');
+            rel = RelishTag::ControlFile;
+        } else {
             return None;
         }
 
-        let key_start = Key::from_hex(key_start_str).ok()?;
-        let key_end = Key::from_hex(key_end_str).ok()?;
+        let segno = parts.next()?.parse::<u32>().ok()?;
 
-        let lsn = Lsn::from_hex(lsn_str).ok()?;
+        let seg = SegmentTag { rel, segno };
 
-        Some(ImageFileName {
-            key_range: key_start..key_end,
-            lsn,
-        })
+        let lsn = Lsn::from_hex(parts.next()?).ok()?;
+
+        if parts.next().is_some() {
+            return None;
+        }
+
+        Some(ImageFileName { seg, lsn })
     }
 }
 
 impl fmt::Display for ImageFileName {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let basename = match self.seg.rel {
+            RelishTag::Relation(reltag) => format!(
+                "rel_{}_{}_{}_{}",
+                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
+            ),
+            RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno,
+            } => format!("pg_xact_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno,
+            } => format!("pg_multixact_members_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno,
+            } => format!("pg_multixact_offsets_{:04X}", segno),
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
+            RelishTag::Checkpoint => "pg_control_checkpoint".to_string(),
+            RelishTag::ControlFile => "pg_control".to_string(),
+        };
+
         write!(
             f,
-            "{}-{}__{:016X}",
-            self.key_range.start,
-            self.key_range.end,
+            "{}_{}_{:016X}",
+            basename,
+            self.seg.segno,
             u64::from(self.lsn),
         )
     }

pageserver/src/layered_repository/global_layer_map.rs

@@ -0,0 +1,142 @@
+//!
+//! Global registry of open layers.
+//!
+//! Whenever a new in-memory layer is created to hold incoming WAL, it is registered
+//! in [`GLOBAL_LAYER_MAP`], so that we can keep track of the total number of
+//! in-memory layers in the system, and know when we need to evict some to release
+//! memory.
+//!
+//! Each layer is assigned a unique ID when it's registered in the global registry.
+//! The ID can be used to relocate the layer later, without having to hold locks.
+//!
+
+use std::sync::atomic::{AtomicU8, Ordering};
+use std::sync::{Arc, RwLock};
+
+use super::inmemory_layer::InMemoryLayer;
+
+use lazy_static::lazy_static;
+
+const MAX_USAGE_COUNT: u8 = 5;
+
+lazy_static! {
+    pub static ref GLOBAL_LAYER_MAP: RwLock<InMemoryLayers> =
+        RwLock::new(InMemoryLayers::default());
+}
+
+// TODO these types can probably be smaller
+#[derive(PartialEq, Eq, Clone, Copy)]
+pub struct LayerId {
+    index: usize,
+    tag: u64, // to avoid ABA problem
+}
+
+enum SlotData {
+    Occupied(Arc<InMemoryLayer>),
+    /// Vacant slots form a linked list, the value is the index
+    /// of the next vacant slot in the list.
+    Vacant(Option<usize>),
+}
+
+struct Slot {
+    tag: u64,
+    data: SlotData,
+    usage_count: AtomicU8, // for clock algorithm
+}
+
+#[derive(Default)]
+pub struct InMemoryLayers {
+    slots: Vec<Slot>,
+    num_occupied: usize,
+
+    // Head of free-slot list.
+    next_empty_slot_idx: Option<usize>,
+}
+
+impl InMemoryLayers {
+    pub fn insert(&mut self, layer: Arc<InMemoryLayer>) -> LayerId {
+        let slot_idx = match self.next_empty_slot_idx {
+            Some(slot_idx) => slot_idx,
+            None => {
+                let idx = self.slots.len();
+                self.slots.push(Slot {
+                    tag: 0,
+                    data: SlotData::Vacant(None),
+                    usage_count: AtomicU8::new(0),
+                });
+                idx
+            }
+        };
+        let slots_len = self.slots.len();
+
+        let slot = &mut self.slots[slot_idx];
+
+        match slot.data {
+            SlotData::Occupied(_) => {
+                panic!("an occupied slot was in the free list");
+            }
+            SlotData::Vacant(next_empty_slot_idx) => {
+                self.next_empty_slot_idx = next_empty_slot_idx;
+            }
+        }
+
+        slot.data = SlotData::Occupied(layer);
+        slot.usage_count.store(1, Ordering::Relaxed);
+
+        self.num_occupied += 1;
+        assert!(self.num_occupied <= slots_len);
+
+        LayerId {
+            index: slot_idx,
+            tag: slot.tag,
+        }
+    }
+
+    pub fn get(&self, layer_id: &LayerId) -> Option<Arc<InMemoryLayer>> {
+        let slot = self.slots.get(layer_id.index)?; // TODO should out of bounds indexes just panic?
+        if slot.tag != layer_id.tag {
+            return None;
+        }
+
+        if let SlotData::Occupied(layer) = &slot.data {
+            let _ = slot.usage_count.fetch_update(
+                Ordering::Relaxed,
+                Ordering::Relaxed,
+                |old_usage_count| {
+                    if old_usage_count < MAX_USAGE_COUNT {
+                        Some(old_usage_count + 1)
+                    } else {
+                        None
+                    }
+                },
+            );
+            Some(Arc::clone(layer))
+        } else {
+            None
+        }
+    }
+
+    // TODO this won't be a public API in the future
+    pub fn remove(&mut self, layer_id: &LayerId) {
+        let slot = &mut self.slots[layer_id.index];
+
+        if slot.tag != layer_id.tag {
+            return;
+        }
+
+        match &slot.data {
+            SlotData::Occupied(_layer) => {
+                // TODO evict the layer
+            }
+            SlotData::Vacant(_) => unimplemented!(),
+        }
+
+        slot.data = SlotData::Vacant(self.next_empty_slot_idx);
+        self.next_empty_slot_idx = Some(layer_id.index);
+
+        assert!(self.num_occupied > 0);
+        self.num_occupied -= 1;
+
+        slot.tag = slot.tag.wrapping_add(1);
+    }
+}

pageserver/src/layered_repository/image_layer.rs

@@ -1,98 +1,86 @@
-//! An ImageLayer represents an image or a snapshot of a key-range at
-//! one particular LSN. It contains an image of all key-value pairs
-//! in its key-range. Any key that falls into the image layer's range
-//! but does not exist in the layer, does not exist.
+//! An ImageLayer represents an image or a snapshot of a segment at one particular LSN.
+//! It is stored in a file on disk.
 //!
-//! An image layer is stored in a file on disk. The file is stored in
-//! timelines/<timelineid> directory.  Currently, there are no
-//! subdirectories, and each image layer file is named like this:
+//! On disk, the image files are stored in timelines/<timelineid> directory.
+//! Currently, there are no subdirectories, and each image layer file is named like this:
 //!
-//!    <key start>-<key end>__<LSN>
+//! Note that segno is
+//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<LSN>
 //!
 //! For example:
 //!
-//!    000000067F000032BE0000400000000070B6-000000067F000032BE0000400000000080B6__00000000346BC568
+//!    1663_13990_2609_0_5_000000000169C348
+//!
+//! An image file is constructed using the 'bookfile' crate.
+//!
+//! Only metadata is loaded into memory by the load function.
+//! When images are needed, they are read directly from disk.
+//!
+//! For blocky relishes, the images are stored in BLOCKY_IMAGES_CHAPTER.
+//! All the images are required to be BLOCK_SIZE, which allows for random access.
+//!
+//! For non-blocky relishes, the image can be found in NONBLOCKY_IMAGE_CHAPTER.
 //!
-//! Every image layer file consists of three parts: "summary",
-//! "index", and "values".  The summary is a fixed size header at the
-//! beginning of the file, and it contains basic information about the
-//! layer, and offsets to the other parts. The "index" is a B-tree,
-//! mapping from Key to an offset in the "values" part.  The
-//! actual page images are stored in the "values" part.
 use crate::config::PageServerConf;
-use crate::layered_repository::blob_io::{BlobCursor, BlobWriter, WriteBlobWriter};
-use crate::layered_repository::block_io::{BlockBuf, BlockReader, FileBlockReader};
-use crate::layered_repository::disk_btree::{DiskBtreeBuilder, DiskBtreeReader, VisitDirection};
 use crate::layered_repository::filename::{ImageFileName, PathOrConf};
 use crate::layered_repository::storage_layer::{
-    Layer, ValueReconstructResult, ValueReconstructState,
+    Layer, PageReconstructData, PageReconstructResult, SegmentBlk, SegmentTag,
 };
-use crate::page_cache::PAGE_SZ;
-use crate::repository::{Key, Value, KEY_SIZE};
+use crate::layered_repository::RELISH_SEG_SIZE;
 use crate::virtual_file::VirtualFile;
-use crate::{IMAGE_FILE_MAGIC, STORAGE_FORMAT_VERSION};
-use anyhow::{bail, ensure, Context, Result};
+use crate::{ZTenantId, ZTimelineId};
+use anyhow::{anyhow, bail, ensure, Context, Result};
 use bytes::Bytes;
-use hex;
+use log::*;
 use serde::{Deserialize, Serialize};
+use std::convert::TryInto;
 use std::fs;
-use std::io::Write;
-use std::io::{Seek, SeekFrom};
-use std::ops::Range;
+use std::io::{BufWriter, Write};
 use std::path::{Path, PathBuf};
-use std::sync::{RwLock, RwLockReadGuard};
-use tracing::*;
+use std::sync::{Mutex, MutexGuard};
 
-use utils::{
-    bin_ser::BeSer,
-    lsn::Lsn,
-    zid::{ZTenantId, ZTimelineId},
-};
+use bookfile::{Book, BookWriter, ChapterWriter};
+
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
+
+// Magic constant to identify a Zenith segment image file
+pub const IMAGE_FILE_MAGIC: u32 = 0x5A616E01 + 1;
+
+/// Contains each block in block # order
+const BLOCKY_IMAGES_CHAPTER: u64 = 1;
+const NONBLOCKY_IMAGE_CHAPTER: u64 = 2;
+
+/// Contains the [`Summary`] struct
+const SUMMARY_CHAPTER: u64 = 3;
 
-///
-/// Header stored in the beginning of the file
-///
-/// After this comes the 'values' part, starting on block 1. After that,
-/// the 'index' starts at the block indicated by 'index_start_blk'
-///
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
 struct Summary {
-    /// Magic value to identify this as a zenith image file. Always IMAGE_FILE_MAGIC.
-    magic: u16,
-    format_version: u16,
-
     tenantid: ZTenantId,
     timelineid: ZTimelineId,
-    key_range: Range<Key>,
-    lsn: Lsn,
+    seg: SegmentTag,
 
-    /// Block number where the 'index' part of the file begins.
-    index_start_blk: u32,
-    /// Block within the 'index', where the B-tree root page is stored
-    index_root_blk: u32,
-    // the 'values' part starts after the summary header, on block 1.
+    lsn: Lsn,
 }
 
 impl From<&ImageLayer> for Summary {
     fn from(layer: &ImageLayer) -> Self {
         Self {
-            magic: IMAGE_FILE_MAGIC,
-            format_version: STORAGE_FORMAT_VERSION,
             tenantid: layer.tenantid,
             timelineid: layer.timelineid,
-            key_range: layer.key_range.clone(),
-            lsn: layer.lsn,
+            seg: layer.seg,
 
-            index_start_blk: 0,
-            index_root_blk: 0,
+            lsn: layer.lsn,
         }
     }
 }
 
+const BLOCK_SIZE: usize = 8192;
+
 ///
 /// ImageLayer is the in-memory data structure associated with an on-disk image
 /// file.  We keep an ImageLayer in memory for each file, in the LayerMap. If a
-/// layer is in "loaded" state, we have a copy of the index in memory, in 'inner'.
+/// layer is in "loaded" state, we have a copy of the file in memory, in 'inner'.
 /// Otherwise the struct is just a placeholder for a file that exists on disk,
 /// and it needs to be loaded before using it in queries.
 ///
@@ -100,24 +88,26 @@ pub struct ImageLayer {
     path_or_conf: PathOrConf,
     pub tenantid: ZTenantId,
     pub timelineid: ZTimelineId,
-    pub key_range: Range<Key>,
+    pub seg: SegmentTag,
 
     // This entry contains an image of all pages as of this LSN
     pub lsn: Lsn,
 
-    inner: RwLock<ImageLayerInner>,
+    inner: Mutex<ImageLayerInner>,
+}
+
+#[derive(Clone)]
+enum ImageType {
+    Blocky { num_blocks: SegmentBlk },
+    NonBlocky,
 }
 
 pub struct ImageLayerInner {
-    /// If false, the 'index' has not been loaded into memory yet.
-    loaded: bool,
+    /// If None, the 'image_type' has not been loaded into memory yet.
+    book: Option<Book<VirtualFile>>,
 
-    // values copied from summary
-    index_start_blk: u32,
-    index_root_blk: u32,
-
-    /// Reader object for reading blocks from the file. (None if not loaded yet)
-    file: Option<FileBlockReader<VirtualFile>>,
+    /// Derived from filename and bookfile chapter metadata
+    image_type: ImageType,
 }
 
 impl Layer for ImageLayer {
@@ -125,10 +115,6 @@ impl Layer for ImageLayer {
         PathBuf::from(self.layer_name().to_string())
     }
 
-    fn local_path(&self) -> Option<PathBuf> {
-        Some(self.path())
-    }
-
     fn get_tenant_id(&self) -> ZTenantId {
         self.tenantid
     }
@@ -137,52 +123,99 @@ impl Layer for ImageLayer {
         self.timelineid
     }
 
-    fn get_key_range(&self) -> Range<Key> {
-        self.key_range.clone()
+    fn get_seg_tag(&self) -> SegmentTag {
+        self.seg
     }
 
-    fn get_lsn_range(&self) -> Range<Lsn> {
+    fn is_dropped(&self) -> bool {
+        false
+    }
+
+    fn get_start_lsn(&self) -> Lsn {
+        self.lsn
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
         // End-bound is exclusive
-        self.lsn..(self.lsn + 1)
+        self.lsn + 1
     }
 
     /// Look up given page in the file
-    fn get_value_reconstruct_data(
+    fn get_page_reconstruct_data(
         &self,
-        key: Key,
-        lsn_range: Range<Lsn>,
-        reconstruct_state: &mut ValueReconstructState,
-    ) -> anyhow::Result<ValueReconstructResult> {
-        assert!(self.key_range.contains(&key));
-        assert!(lsn_range.start >= self.lsn);
-        assert!(lsn_range.end >= self.lsn);
+        blknum: SegmentBlk,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<PageReconstructResult> {
+        assert!((0..RELISH_SEG_SIZE).contains(&blknum));
+        assert!(lsn >= self.lsn);
+
+        match reconstruct_data.page_img {
+            Some((cached_lsn, _)) if self.lsn <= cached_lsn => {
+                return Ok(PageReconstructResult::Complete)
+            }
+            _ => {}
+        }
 
         let inner = self.load()?;
 
-        let file = inner.file.as_ref().unwrap();
-        let tree_reader = DiskBtreeReader::new(inner.index_start_blk, inner.index_root_blk, file);
+        let buf = match &inner.image_type {
+            ImageType::Blocky { num_blocks } => {
+                // Check if the request is beyond EOF
+                if blknum >= *num_blocks {
+                    return Ok(PageReconstructResult::Missing(lsn));
+                }
 
-        let mut keybuf: [u8; KEY_SIZE] = [0u8; KEY_SIZE];
-        key.write_to_byte_slice(&mut keybuf);
-        if let Some(offset) = tree_reader.get(&keybuf)? {
-            let blob = file.block_cursor().read_blob(offset).with_context(|| {
-                format!(
-                    "failed to read value from data file {} at offset {}",
-                    self.filename().display(),
-                    offset
-                )
-            })?;
-            let value = Bytes::from(blob);
+                let mut buf = vec![0u8; BLOCK_SIZE];
+                let offset = BLOCK_SIZE as u64 * blknum as u64;
 
-            reconstruct_state.img = Some((self.lsn, value));
-            Ok(ValueReconstructResult::Complete)
-        } else {
-            Ok(ValueReconstructResult::Missing)
+                let chapter = inner
+                    .book
+                    .as_ref()
+                    .unwrap()
+                    .chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
+
+                chapter.read_exact_at(&mut buf, offset).with_context(|| {
+                    format!(
+                        "failed to read page from data file {} at offset {}",
+                        self.filename().display(),
+                        offset
+                    )
+                })?;
+
+                buf
+            }
+            ImageType::NonBlocky => {
+                ensure!(blknum == 0);
+                inner
+                    .book
+                    .as_ref()
+                    .unwrap()
+                    .read_chapter(NONBLOCKY_IMAGE_CHAPTER)?
+                    .into_vec()
+            }
+        };
+
+        reconstruct_data.page_img = Some((self.lsn, Bytes::from(buf)));
+        Ok(PageReconstructResult::Complete)
+    }
+
+    /// Get size of the segment
+    fn get_seg_size(&self, _lsn: Lsn) -> Result<SegmentBlk> {
+        let inner = self.load()?;
+        match inner.image_type {
+            ImageType::Blocky { num_blocks } => Ok(num_blocks),
+            ImageType::NonBlocky => Err(anyhow!("get_seg_size called for non-blocky segment")),
         }
     }
 
-    fn iter(&self) -> Box<dyn Iterator<Item = Result<(Key, Lsn, Value)>>> {
-        todo!();
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, _lsn: Lsn) -> Result<bool> {
+        Ok(true)
+    }
+
+    fn unload(&self) -> Result<()> {
+        Ok(())
     }
 
     fn delete(&self) -> Result<()> {
@@ -200,27 +233,25 @@ impl Layer for ImageLayer {
     }
 
     /// debugging function to print out the contents of the layer
-    fn dump(&self, verbose: bool) -> Result<()> {
+    fn dump(&self) -> Result<()> {
         println!(
-            "----- image layer for ten {} tli {} key {}-{} at {} ----",
-            self.tenantid, self.timelineid, self.key_range.start, self.key_range.end, self.lsn
+            "----- image layer for ten {} tli {} seg {} at {} ----",
+            self.tenantid, self.timelineid, self.seg, self.lsn
         );
 
-        if !verbose {
-            return Ok(());
-        }
-
         let inner = self.load()?;
-        let file = inner.file.as_ref().unwrap();
-        let tree_reader =
-            DiskBtreeReader::<_, KEY_SIZE>::new(inner.index_start_blk, inner.index_root_blk, file);
 
-        tree_reader.dump()?;
-
-        tree_reader.visit(&[0u8; KEY_SIZE], VisitDirection::Forwards, |key, value| {
-            println!("key: {} offset {}", hex::encode(key), value);
-            true
-        })?;
+        match inner.image_type {
+            ImageType::Blocky { num_blocks } => println!("({}) blocks ", num_blocks),
+            ImageType::NonBlocky => {
+                let chapter = inner
+                    .book
+                    .as_ref()
+                    .unwrap()
+                    .read_chapter(NONBLOCKY_IMAGE_CHAPTER)?;
+                println!("non-blocky ({} bytes)", chapter.len());
+            }
+        }
 
         Ok(())
     }
@@ -242,57 +273,32 @@ impl ImageLayer {
     }
 
     ///
-    /// Open the underlying file and read the metadata into memory, if it's
-    /// not loaded already.
+    /// Load the contents of the file into memory
     ///
-    fn load(&self) -> Result<RwLockReadGuard<ImageLayerInner>> {
-        loop {
-            // Quick exit if already loaded
-            let inner = self.inner.read().unwrap();
-            if inner.loaded {
-                return Ok(inner);
-            }
+    fn load(&self) -> Result<MutexGuard<ImageLayerInner>> {
+        // quick exit if already loaded
+        let mut inner = self.inner.lock().unwrap();
 
-            // Need to open the file and load the metadata. Upgrade our lock to
-            // a write lock. (Or rather, release and re-lock in write mode.)
-            drop(inner);
-            let mut inner = self.inner.write().unwrap();
-            if !inner.loaded {
-                self.load_inner(&mut inner).with_context(|| {
-                    format!("Failed to load image layer {}", self.path().display())
-                })?
-            } else {
-                // Another thread loaded it while we were not holding the lock.
-            }
-
-            // We now have the file open and loaded. There's no function to do
-            // that in the std library RwLock, so we have to release and re-lock
-            // in read mode. (To be precise, the lock guard was moved in the
-            // above call to `load_inner`, so it's already been released). And
-            // while we do that, another thread could unload again, so we have
-            // to re-check and retry if that happens.
-            drop(inner);
+        if inner.book.is_some() {
+            return Ok(inner);
         }
-    }
 
-    fn load_inner(&self, inner: &mut ImageLayerInner) -> Result<()> {
         let path = self.path();
-
-        // Open the file if it's not open already.
-        if inner.file.is_none() {
-            let file = VirtualFile::open(&path)
-                .with_context(|| format!("Failed to open file '{}'", path.display()))?;
-            inner.file = Some(FileBlockReader::new(file));
-        }
-        let file = inner.file.as_mut().unwrap();
-        let summary_blk = file.read_blk(0)?;
-        let actual_summary = Summary::des_prefix(summary_blk.as_ref())?;
+        let file = VirtualFile::open(&path)
+            .with_context(|| format!("Failed to open virtual file '{}'", path.display()))?;
+        let book = Book::new(file).with_context(|| {
+            format!(
+                "Failed to open virtual file '{}' as a bookfile",
+                path.display()
+            )
+        })?;
 
         match &self.path_or_conf {
             PathOrConf::Conf(_) => {
-                let mut expected_summary = Summary::from(self);
-                expected_summary.index_start_blk = actual_summary.index_start_blk;
-                expected_summary.index_root_blk = actual_summary.index_root_blk;
+                let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
+                let actual_summary = Summary::des(&chapter)?;
+
+                let expected_summary = Summary::from(self);
 
                 if actual_summary != expected_summary {
                     bail!("in-file summary does not match expected summary. actual = {:?} expected = {:?}", actual_summary, expected_summary);
@@ -312,10 +318,25 @@ impl ImageLayer {
             }
         }
 
-        inner.index_start_blk = actual_summary.index_start_blk;
-        inner.index_root_blk = actual_summary.index_root_blk;
-        inner.loaded = true;
-        Ok(())
+        let image_type = if self.seg.rel.is_blocky() {
+            let chapter = book.chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
+            let images_len = chapter.len();
+            ensure!(images_len % BLOCK_SIZE as u64 == 0);
+            let num_blocks: SegmentBlk = (images_len / BLOCK_SIZE as u64).try_into()?;
+            ImageType::Blocky { num_blocks }
+        } else {
+            let _chapter = book.chapter_reader(NONBLOCKY_IMAGE_CHAPTER)?;
+            ImageType::NonBlocky
+        };
+
+        debug!("loaded from {}", &path.display());
+
+        *inner = ImageLayerInner {
+            book: Some(book),
+            image_type,
+        };
+
+        Ok(inner)
     }
 
     /// Create an ImageLayer struct representing an existing file on disk
@@ -329,13 +350,11 @@ impl ImageLayer {
             path_or_conf: PathOrConf::Conf(conf),
             timelineid,
             tenantid,
-            key_range: filename.key_range.clone(),
+            seg: filename.seg,
             lsn: filename.lsn,
-            inner: RwLock::new(ImageLayerInner {
-                loaded: false,
-                file: None,
-                index_start_blk: 0,
-                index_root_blk: 0,
+            inner: Mutex::new(ImageLayerInner {
+                book: None,
+                image_type: ImageType::Blocky { num_blocks: 0 },
             }),
         }
     }
@@ -343,33 +362,29 @@ impl ImageLayer {
     /// Create an ImageLayer struct representing an existing file on disk.
     ///
     /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary.
-    pub fn new_for_path<F>(path: &Path, file: F) -> Result<ImageLayer>
+    pub fn new_for_path<F>(path: &Path, book: &Book<F>) -> Result<ImageLayer>
     where
         F: std::os::unix::prelude::FileExt,
     {
-        let mut summary_buf = Vec::new();
-        summary_buf.resize(PAGE_SZ, 0);
-        file.read_exact_at(&mut summary_buf, 0)?;
-        let summary = Summary::des_prefix(&summary_buf)?;
+        let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
+        let summary = Summary::des(&chapter)?;
 
         Ok(ImageLayer {
             path_or_conf: PathOrConf::Path(path.to_path_buf()),
             timelineid: summary.timelineid,
             tenantid: summary.tenantid,
-            key_range: summary.key_range,
+            seg: summary.seg,
             lsn: summary.lsn,
-            inner: RwLock::new(ImageLayerInner {
-                file: None,
-                loaded: false,
-                index_start_blk: 0,
-                index_root_blk: 0,
+            inner: Mutex::new(ImageLayerInner {
+                book: None,
+                image_type: ImageType::Blocky { num_blocks: 0 },
             }),
         })
     }
 
     fn layer_name(&self) -> ImageFileName {
         ImageFileName {
-            key_range: self.key_range.clone(),
+            seg: self.seg,
             lsn: self.lsn,
         }
     }
@@ -391,21 +406,22 @@ impl ImageLayer {
 ///
 /// 1. Create the ImageLayerWriter by calling ImageLayerWriter::new(...)
 ///
-/// 2. Write the contents by calling `put_page_image` for every key-value
-///    pair in the key range.
+/// 2. Write the contents by calling `put_page_image` for every page
+///    in the segment.
 ///
 /// 3. Call `finish`.
 ///
 pub struct ImageLayerWriter {
     conf: &'static PageServerConf,
-    _path: PathBuf,
     timelineid: ZTimelineId,
     tenantid: ZTenantId,
-    key_range: Range<Key>,
+    seg: SegmentTag,
     lsn: Lsn,
 
-    blob_writer: WriteBlobWriter<VirtualFile>,
-    tree: DiskBtreeBuilder<BlockBuf, KEY_SIZE>,
+    num_blocks: SegmentBlk,
+
+    page_image_writer: ChapterWriter<BufWriter<VirtualFile>>,
+    num_blocks_written: SegmentBlk,
 }
 
 impl ImageLayerWriter {
@@ -413,9 +429,10 @@ impl ImageLayerWriter {
         conf: &'static PageServerConf,
         timelineid: ZTimelineId,
         tenantid: ZTenantId,
-        key_range: &Range<Key>,
+        seg: SegmentTag,
         lsn: Lsn,
-    ) -> anyhow::Result<ImageLayerWriter> {
+        num_blocks: SegmentBlk,
+    ) -> Result<ImageLayerWriter> {
         // Create the file
         //
         // Note: This overwrites any existing file. There shouldn't be any.
@@ -424,92 +441,90 @@ impl ImageLayerWriter {
             &PathOrConf::Conf(conf),
             timelineid,
             tenantid,
-            &ImageFileName {
-                key_range: key_range.clone(),
-                lsn,
-            },
+            &ImageFileName { seg, lsn },
         );
-        info!("new image layer {}", path.display());
-        let mut file = VirtualFile::create(&path)?;
-        // make room for the header block
-        file.seek(SeekFrom::Start(PAGE_SZ as u64))?;
-        let blob_writer = WriteBlobWriter::new(file, PAGE_SZ as u64);
+        let file = VirtualFile::create(&path)?;
+        let buf_writer = BufWriter::new(file);
+        let book = BookWriter::new(buf_writer, IMAGE_FILE_MAGIC)?;
 
-        // Initialize the b-tree index builder
-        let block_buf = BlockBuf::new();
-        let tree_builder = DiskBtreeBuilder::new(block_buf);
+        // Open the page-images chapter for writing. The calls to
+        // `put_page_image` will use this to write the contents.
+        let chapter = if seg.rel.is_blocky() {
+            book.new_chapter(BLOCKY_IMAGES_CHAPTER)
+        } else {
+            assert_eq!(num_blocks, 1);
+            book.new_chapter(NONBLOCKY_IMAGE_CHAPTER)
+        };
 
         let writer = ImageLayerWriter {
             conf,
-            _path: path,
             timelineid,
             tenantid,
-            key_range: key_range.clone(),
+            seg,
             lsn,
-            tree: tree_builder,
-            blob_writer,
+            num_blocks,
+            page_image_writer: chapter,
+            num_blocks_written: 0,
         };
 
         Ok(writer)
     }
 
     ///
-    /// Write next value to the file.
+    /// Write next page image to the file.
     ///
     /// The page versions must be appended in blknum order.
     ///
-    pub fn put_image(&mut self, key: Key, img: &[u8]) -> Result<()> {
-        ensure!(self.key_range.contains(&key));
-        let off = self.blob_writer.write_blob(img)?;
-
-        let mut keybuf: [u8; KEY_SIZE] = [0u8; KEY_SIZE];
-        key.write_to_byte_slice(&mut keybuf);
-        self.tree.append(&keybuf, off)?;
-
+    pub fn put_page_image(&mut self, block_bytes: &[u8]) -> Result<()> {
+        assert!(self.num_blocks_written < self.num_blocks);
+        if self.seg.rel.is_blocky() {
+            assert_eq!(block_bytes.len(), BLOCK_SIZE);
+        }
+        self.page_image_writer.write_all(block_bytes)?;
+        self.num_blocks_written += 1;
         Ok(())
     }
 
-    pub fn finish(self) -> anyhow::Result<ImageLayer> {
-        let index_start_blk =
-            ((self.blob_writer.size() + PAGE_SZ as u64 - 1) / PAGE_SZ as u64) as u32;
+    pub fn finish(self) -> Result<ImageLayer> {
+        // Check that the `put_page_image' was called for every block.
+        assert!(self.num_blocks_written == self.num_blocks);
 
-        let mut file = self.blob_writer.into_inner();
+        // Close the page-images chapter
+        let book = self.page_image_writer.close()?;
 
-        // Write out the index
-        file.seek(SeekFrom::Start(index_start_blk as u64 * PAGE_SZ as u64))?;
-        let (index_root_blk, block_buf) = self.tree.finish()?;
-        for buf in block_buf.blocks {
-            file.write_all(buf.as_ref())?;
-        }
-
-        // Fill in the summary on blk 0
+        // Write out the summary chapter
+        let image_type = if self.seg.rel.is_blocky() {
+            ImageType::Blocky {
+                num_blocks: self.num_blocks,
+            }
+        } else {
+            ImageType::NonBlocky
+        };
+        let mut chapter = book.new_chapter(SUMMARY_CHAPTER);
         let summary = Summary {
-            magic: IMAGE_FILE_MAGIC,
-            format_version: STORAGE_FORMAT_VERSION,
             tenantid: self.tenantid,
             timelineid: self.timelineid,
-            key_range: self.key_range.clone(),
+            seg: self.seg,
             lsn: self.lsn,
-            index_start_blk,
-            index_root_blk,
         };
-        file.seek(SeekFrom::Start(0))?;
-        Summary::ser_into(&summary, &mut file)?;
+        Summary::ser_into(&summary, &mut chapter)?;
+        let book = chapter.close()?;
+
+        // This flushes the underlying 'buf_writer'.
+        book.close()?;
 
         // Note: Because we open the file in write-only mode, we cannot
         // reuse the same VirtualFile for reading later. That's why we don't
-        // set inner.file here. The first read will have to re-open it.
+        // set inner.book here. The first read will have to re-open it.
         let layer = ImageLayer {
             path_or_conf: PathOrConf::Conf(self.conf),
             timelineid: self.timelineid,
             tenantid: self.tenantid,
-            key_range: self.key_range.clone(),
+            seg: self.seg,
             lsn: self.lsn,
-            inner: RwLock::new(ImageLayerInner {
-                loaded: false,
-                file: None,
-                index_start_blk,
-                index_root_blk,
+            inner: Mutex::new(ImageLayerInner {
+                book: None,
+                image_type,
             }),
         };
         trace!("created image layer {}", layer.path().display());

pageserver/src/layered_repository/inmemory_layer.rs

@@ -1,39 +1,39 @@
-//! An in-memory layer stores recently received key-value pairs.
+//! An in-memory layer stores recently received PageVersions.
+//! The page versions are held in a BTreeMap. To avoid OOM errors, the map size is limited
+//! and layers can be spilled to disk into ephemeral files.
 //!
-//! The "in-memory" part of the name is a bit misleading: the actual page versions are
-//! held in an ephemeral file, not in memory. The metadata for each page version, i.e.
-//! its position in the file, is kept in memory, though.
+//! And there's another BTreeMap to track the size of the relation.
 //!
 use crate::config::PageServerConf;
-use crate::layered_repository::blob_io::{BlobCursor, BlobWriter};
-use crate::layered_repository::block_io::BlockReader;
 use crate::layered_repository::delta_layer::{DeltaLayer, DeltaLayerWriter};
 use crate::layered_repository::ephemeral_file::EphemeralFile;
+use crate::layered_repository::filename::DeltaFileName;
+use crate::layered_repository::image_layer::{ImageLayer, ImageLayerWriter};
 use crate::layered_repository::storage_layer::{
-    Layer, ValueReconstructResult, ValueReconstructState,
+    Layer, PageReconstructData, PageReconstructResult, PageVersion, SegmentBlk, SegmentTag,
+    RELISH_SEG_SIZE,
 };
-use crate::repository::{Key, Value};
-use crate::walrecord;
-use anyhow::{bail, ensure, Result};
+use crate::layered_repository::LayeredTimeline;
+use crate::layered_repository::ZERO_PAGE;
+use crate::repository::ZenithWalRecord;
+use crate::{ZTenantId, ZTimelineId};
+use anyhow::{ensure, Result};
+use bytes::Bytes;
+use log::*;
 use std::collections::HashMap;
-use tracing::*;
-use utils::{
-    bin_ser::BeSer,
-    lsn::Lsn,
-    vec_map::VecMap,
-    zid::{ZTenantId, ZTimelineId},
-};
-// avoid binding to Write (conflicts with std::io::Write)
-// while being able to use std::fmt::Write's methods
-use std::fmt::Write as _;
-use std::ops::Range;
+use std::io::Seek;
+use std::os::unix::fs::FileExt;
 use std::path::PathBuf;
-use std::sync::RwLock;
+use std::sync::{Arc, RwLock};
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
+use zenith_utils::vec_map::VecMap;
 
 pub struct InMemoryLayer {
     conf: &'static PageServerConf,
     tenantid: ZTenantId,
     timelineid: ZTimelineId,
+    seg: SegmentTag,
 
     ///
     /// This layer contains all the changes from 'start_lsn'. The
@@ -41,9 +41,27 @@ pub struct InMemoryLayer {
     ///
     start_lsn: Lsn,
 
+    ///
+    /// LSN of the oldest page version stored in this layer.
+    ///
+    /// This is different from 'start_lsn' in that we enforce that the 'start_lsn'
+    /// of a layer always matches the 'end_lsn' of its predecessor, even if there
+    /// are no page versions until at a later LSN. That way you can detect any
+    /// missing layer files more easily. 'oldest_lsn' is the first page version
+    /// actually stored in this layer. In the range between 'start_lsn' and
+    /// 'oldest_lsn', there are no changes to the segment.
+    /// 'oldest_lsn' is used to adjust 'disk_consistent_lsn' and that is why it should
+    /// point to the beginning of WAL record. This is the other difference with 'start_lsn'
+    /// which points to end of WAL record. This is why 'oldest_lsn' can be smaller than 'start_lsn'.
+    ///
+    oldest_lsn: Lsn,
+
     /// The above fields never change. The parts that do change are in 'inner',
     /// and protected by mutex.
     inner: RwLock<InMemoryLayerInner>,
+
+    /// Predecessor layer might be needed?
+    incremental: bool,
 }
 
 pub struct InMemoryLayerInner {
@@ -51,23 +69,98 @@ pub struct InMemoryLayerInner {
     /// Writes are only allowed when this is None
     end_lsn: Option<Lsn>,
 
-    ///
-    /// All versions of all pages in the layer are kept here.  Indexed
-    /// by block number and LSN. The value is an offset into the
-    /// ephemeral file where the page version is stored.
-    ///
-    index: HashMap<Key, VecMap<Lsn, u64>>,
+    /// If this relation was dropped, remember when that happened.
+    /// The drop LSN is recorded in [`end_lsn`].
+    dropped: bool,
 
-    /// The values are stored in a serialized format in this file.
-    /// Each serialized Value is preceded by a 'u32' length field.
-    /// PerSeg::page_versions map stores offsets into this file.
+    /// The PageVersion structs are stored in a serialized format in this file.
+    /// Each serialized PageVersion is preceded by a 'u32' length field.
+    /// 'page_versions' map stores offsets into this file.
     file: EphemeralFile,
+
+    /// Metadata about all versions of all pages in the layer is kept
+    /// here.  Indexed by block number and LSN. The value is an offset
+    /// into the ephemeral file where the page version is stored.
+    page_versions: HashMap<SegmentBlk, VecMap<Lsn, u64>>,
+
+    ///
+    /// `seg_sizes` tracks the size of the segment at different points in time.
+    ///
+    /// For a blocky rel, there is always one entry, at the layer's start_lsn,
+    /// so that determining the size never depends on the predecessor layer. For
+    /// a non-blocky rel, 'seg_sizes' is not used and is always empty.
+    ///
+    seg_sizes: VecMap<Lsn, SegmentBlk>,
+
+    ///
+    /// LSN of the newest page version stored in this layer.
+    ///
+    /// The difference between 'end_lsn' and 'latest_lsn' is the same as between
+    /// 'start_lsn' and 'oldest_lsn'. See comments in 'oldest_lsn'.
+    ///
+    latest_lsn: Lsn,
 }
 
 impl InMemoryLayerInner {
     fn assert_writeable(&self) {
         assert!(self.end_lsn.is_none());
     }
+
+    fn get_seg_size(&self, lsn: Lsn) -> SegmentBlk {
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let slice = self.seg_sizes.slice_range(..=lsn);
+
+        // We make sure there is always at least one entry
+        if let Some((_entry_lsn, entry)) = slice.last() {
+            *entry
+        } else {
+            panic!("could not find seg size in in-memory layer");
+        }
+    }
+
+    ///
+    /// Read a page version from the ephemeral file.
+    ///
+    fn read_pv(&self, off: u64) -> Result<PageVersion> {
+        let mut buf = Vec::new();
+        self.read_pv_bytes(off, &mut buf)?;
+        Ok(PageVersion::des(&buf)?)
+    }
+
+    ///
+    /// Read a page version from the ephemeral file, as raw bytes, at
+    /// the given offset.  The bytes are read into 'buf', which is
+    /// expanded if necessary. Returns the size of the page version.
+    ///
+    fn read_pv_bytes(&self, off: u64, buf: &mut Vec<u8>) -> Result<usize> {
+        // read length
+        let mut lenbuf = [0u8; 4];
+        self.file.read_exact_at(&mut lenbuf, off)?;
+        let len = u32::from_ne_bytes(lenbuf) as usize;
+
+        if buf.len() < len {
+            buf.resize(len, 0);
+        }
+        self.file.read_exact_at(&mut buf[0..len], off + 4)?;
+        Ok(len)
+    }
+
+    fn write_pv(&mut self, pv: &PageVersion) -> Result<u64> {
+        // remember starting position
+        let pos = self.file.stream_position()?;
+
+        // make room for the 'length' field by writing zeros as a placeholder.
+        self.file.seek(std::io::SeekFrom::Start(pos + 4)).unwrap();
+
+        pv.ser_into(&mut self.file).unwrap();
+
+        // write the 'length' field.
+        let len = self.file.stream_position()? - pos - 4;
+        let lenbuf = u32::to_ne_bytes(len as u32);
+        self.file.write_all_at(&lenbuf, pos)?;
+
+        Ok(pos)
+    }
 }
 
 impl Layer for InMemoryLayer {
@@ -77,16 +170,21 @@ impl Layer for InMemoryLayer {
     fn filename(&self) -> PathBuf {
         let inner = self.inner.read().unwrap();
 
-        let end_lsn = inner.end_lsn.unwrap_or(Lsn(u64::MAX));
+        let end_lsn = if let Some(drop_lsn) = inner.end_lsn {
+            drop_lsn
+        } else {
+            Lsn(u64::MAX)
+        };
 
-        PathBuf::from(format!(
-            "inmem-{:016X}-{:016X}",
-            self.start_lsn.0, end_lsn.0
-        ))
-    }
+        let delta_filename = DeltaFileName {
+            seg: self.seg,
+            start_lsn: self.start_lsn,
+            end_lsn,
+            dropped: inner.dropped,
+        }
+        .to_string();
 
-    fn local_path(&self) -> Option<PathBuf> {
-        None
+        PathBuf::from(format!("inmem-{}", delta_filename))
     }
 
     fn get_tenant_id(&self) -> ZTenantId {
@@ -97,83 +195,149 @@ impl Layer for InMemoryLayer {
         self.timelineid
     }
 
-    fn get_key_range(&self) -> Range<Key> {
-        Key::MIN..Key::MAX
+    fn get_seg_tag(&self) -> SegmentTag {
+        self.seg
     }
 
-    fn get_lsn_range(&self) -> Range<Lsn> {
+    fn get_start_lsn(&self) -> Lsn {
+        self.start_lsn
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
         let inner = self.inner.read().unwrap();
 
-        let end_lsn = if let Some(end_lsn) = inner.end_lsn {
+        if let Some(end_lsn) = inner.end_lsn {
             end_lsn
         } else {
             Lsn(u64::MAX)
-        };
-        self.start_lsn..end_lsn
+        }
     }
 
-    /// Look up given value in the layer.
-    fn get_value_reconstruct_data(
+    fn is_dropped(&self) -> bool {
+        let inner = self.inner.read().unwrap();
+        inner.dropped
+    }
+
+    /// Look up given page in the cache.
+    fn get_page_reconstruct_data(
         &self,
-        key: Key,
-        lsn_range: Range<Lsn>,
-        reconstruct_state: &mut ValueReconstructState,
-    ) -> anyhow::Result<ValueReconstructResult> {
-        ensure!(lsn_range.start >= self.start_lsn);
+        blknum: SegmentBlk,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<PageReconstructResult> {
         let mut need_image = true;
 
-        let inner = self.inner.read().unwrap();
+        assert!((0..RELISH_SEG_SIZE).contains(&blknum));
 
-        let mut reader = inner.file.block_cursor();
+        {
+            let inner = self.inner.read().unwrap();
 
-        // Scan the page versions backwards, starting from `lsn`.
-        if let Some(vec_map) = inner.index.get(&key) {
-            let slice = vec_map.slice_range(lsn_range);
-            for (entry_lsn, pos) in slice.iter().rev() {
-                let buf = reader.read_blob(*pos)?;
-                let value = Value::des(&buf)?;
-                match value {
-                    Value::Image(img) => {
-                        reconstruct_state.img = Some((*entry_lsn, img));
-                        return Ok(ValueReconstructResult::Complete);
+            // Scan the page versions backwards, starting from `lsn`.
+            if let Some(vec_map) = inner.page_versions.get(&blknum) {
+                let slice = vec_map.slice_range(..=lsn);
+                for (entry_lsn, pos) in slice.iter().rev() {
+                    match &reconstruct_data.page_img {
+                        Some((cached_lsn, _)) if entry_lsn <= cached_lsn => {
+                            return Ok(PageReconstructResult::Complete)
+                        }
+                        _ => {}
                     }
-                    Value::WalRecord(rec) => {
-                        let will_init = rec.will_init();
-                        reconstruct_state.records.push((*entry_lsn, rec));
-                        if will_init {
-                            // This WAL record initializes the page, so no need to go further back
+
+                    let pv = inner.read_pv(*pos)?;
+                    match pv {
+                        PageVersion::Page(img) => {
+                            reconstruct_data.page_img = Some((*entry_lsn, img));
                             need_image = false;
                             break;
                         }
+                        PageVersion::Wal(rec) => {
+                            reconstruct_data.records.push((*entry_lsn, rec.clone()));
+                            if rec.will_init() {
+                                // This WAL record initializes the page, so no need to go further back
+                                need_image = false;
+                                break;
+                            }
+                        }
                     }
                 }
             }
+
+            // If we didn't find any records for this, check if the request is beyond EOF
+            if need_image
+                && reconstruct_data.records.is_empty()
+                && self.seg.rel.is_blocky()
+                && blknum >= self.get_seg_size(lsn)?
+            {
+                return Ok(PageReconstructResult::Missing(self.start_lsn));
+            }
+
+            // release lock on 'inner'
         }
 
-        // release lock on 'inner'
-
         // If an older page image is needed to reconstruct the page, let the
-        // caller know.
+        // caller know
         if need_image {
-            Ok(ValueReconstructResult::Continue)
+            if self.incremental {
+                Ok(PageReconstructResult::Continue(Lsn(self.start_lsn.0 - 1)))
+            } else {
+                Ok(PageReconstructResult::Missing(self.start_lsn))
+            }
         } else {
-            Ok(ValueReconstructResult::Complete)
+            Ok(PageReconstructResult::Complete)
         }
     }
 
-    fn iter(&self) -> Box<dyn Iterator<Item = Result<(Key, Lsn, Value)>>> {
-        todo!();
+    /// Get size of the relation at given LSN
+    fn get_seg_size(&self, lsn: Lsn) -> Result<SegmentBlk> {
+        assert!(lsn >= self.start_lsn);
+        ensure!(
+            self.seg.rel.is_blocky(),
+            "get_seg_size() called on a non-blocky rel"
+        );
+
+        let inner = self.inner.read().unwrap();
+        Ok(inner.get_seg_size(lsn))
+    }
+
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
+        let inner = self.inner.read().unwrap();
+
+        // If the segment created after requested LSN,
+        // it doesn't exist in the layer. But we shouldn't
+        // have requested it in the first place.
+        assert!(lsn >= self.start_lsn);
+
+        // Is the requested LSN after the segment was dropped?
+        if inner.dropped {
+            if let Some(end_lsn) = inner.end_lsn {
+                if lsn >= end_lsn {
+                    return Ok(false);
+                }
+            } else {
+                panic!("dropped in-memory layer with no end LSN");
+            }
+        }
+
+        // Otherwise, it exists
+        Ok(true)
+    }
+
+    /// Cannot unload anything in an in-memory layer, since there's no backing
+    /// store. To release memory used by an in-memory layer, use 'freeze' to turn
+    /// it into an on-disk layer.
+    fn unload(&self) -> Result<()> {
+        Ok(())
     }
 
     /// Nothing to do here. When you drop the last reference to the layer, it will
     /// be deallocated.
     fn delete(&self) -> Result<()> {
-        bail!("can't delete an InMemoryLayer")
+        panic!("can't delete an InMemoryLayer")
     }
 
     fn is_incremental(&self) -> bool {
-        // in-memory layer is always considered incremental.
-        true
+        self.incremental
     }
 
     fn is_in_memory(&self) -> bool {
@@ -181,7 +345,7 @@ impl Layer for InMemoryLayer {
     }
 
     /// debugging function to print out the contents of the layer
-    fn dump(&self, verbose: bool) -> Result<()> {
+    fn dump(&self) -> Result<()> {
         let inner = self.inner.read().unwrap();
 
         let end_str = inner
@@ -191,40 +355,29 @@ impl Layer for InMemoryLayer {
             .unwrap_or_default();
 
         println!(
-            "----- in-memory layer for tli {} LSNs {}-{} ----",
-            self.timelineid, self.start_lsn, end_str,
+            "----- in-memory layer for tli {} seg {} {}-{} {} ----",
+            self.timelineid, self.seg, self.start_lsn, end_str, inner.dropped,
         );
 
-        if !verbose {
-            return Ok(());
+        for (k, v) in inner.seg_sizes.as_slice() {
+            println!("seg_sizes {}: {}", k, v);
         }
 
-        let mut cursor = inner.file.block_cursor();
-        let mut buf = Vec::new();
-        for (key, vec_map) in inner.index.iter() {
-            for (lsn, pos) in vec_map.as_slice() {
-                let mut desc = String::new();
-                cursor.read_blob_into_buf(*pos, &mut buf)?;
-                let val = Value::des(&buf);
-                match val {
-                    Ok(Value::Image(img)) => {
-                        write!(&mut desc, " img {} bytes", img.len())?;
-                    }
-                    Ok(Value::WalRecord(rec)) => {
-                        let wal_desc = walrecord::describe_wal_record(&rec).unwrap();
-                        write!(
-                            &mut desc,
-                            " rec {} bytes will_init: {} {}",
-                            buf.len(),
-                            rec.will_init(),
-                            wal_desc
-                        )?;
-                    }
-                    Err(err) => {
-                        write!(&mut desc, " DESERIALIZATION ERROR: {}", err)?;
-                    }
-                }
-                println!("  key {} at {}: {}", key, lsn, desc);
+        // List the blocks in order
+        let mut page_versions: Vec<(&SegmentBlk, &VecMap<Lsn, u64>)> =
+            inner.page_versions.iter().collect();
+        page_versions.sort_by_key(|k| k.0);
+
+        for (blknum, versions) in page_versions {
+            for (lsn, off) in versions.as_slice() {
+                let pv = inner.read_pv(*off);
+                let pv_description = match pv {
+                    Ok(PageVersion::Page(_img)) => "page",
+                    Ok(PageVersion::Wal(_rec)) => "wal",
+                    Err(_err) => "INVALID",
+                };
+
+                println!("blk {} at {}: {}\n", blknum, lsn, pv_description);
             }
         }
 
@@ -232,7 +385,23 @@ impl Layer for InMemoryLayer {
     }
 }
 
+/// A result of an inmemory layer data being written to disk.
+pub struct LayersOnDisk {
+    pub delta_layers: Vec<DeltaLayer>,
+    pub image_layers: Vec<ImageLayer>,
+}
+
 impl InMemoryLayer {
+    /// Return the oldest page version that's stored in this layer
+    pub fn get_oldest_lsn(&self) -> Lsn {
+        self.oldest_lsn
+    }
+
+    pub fn get_latest_lsn(&self) -> Lsn {
+        let inner = self.inner.read().unwrap();
+        inner.latest_lsn
+    }
+
     ///
     /// Create a new, empty, in-memory layer
     ///
@@ -240,77 +409,286 @@ impl InMemoryLayer {
         conf: &'static PageServerConf,
         timelineid: ZTimelineId,
         tenantid: ZTenantId,
+        seg: SegmentTag,
         start_lsn: Lsn,
+        oldest_lsn: Lsn,
     ) -> Result<InMemoryLayer> {
         trace!(
-            "initializing new empty InMemoryLayer for writing on timeline {} at {}",
+            "initializing new empty InMemoryLayer for writing {} on timeline {} at {}",
+            seg,
             timelineid,
             start_lsn
         );
 
+        // The segment is initially empty, so initialize 'seg_sizes' with 0.
+        let mut seg_sizes = VecMap::default();
+        if seg.rel.is_blocky() {
+            seg_sizes.append(start_lsn, 0).unwrap();
+        }
+
         let file = EphemeralFile::create(conf, tenantid, timelineid)?;
 
         Ok(InMemoryLayer {
             conf,
             timelineid,
             tenantid,
+            seg,
             start_lsn,
+            oldest_lsn,
+            incremental: false,
             inner: RwLock::new(InMemoryLayerInner {
                 end_lsn: None,
-                index: HashMap::new(),
+                dropped: false,
                 file,
+                page_versions: HashMap::new(),
+                seg_sizes,
+                latest_lsn: oldest_lsn,
             }),
         })
     }
 
     // Write operations
 
+    /// Remember new page version, as a WAL record over previous version
+    pub fn put_wal_record(
+        &self,
+        lsn: Lsn,
+        blknum: SegmentBlk,
+        rec: ZenithWalRecord,
+    ) -> Result<u32> {
+        self.put_page_version(blknum, lsn, PageVersion::Wal(rec))
+    }
+
+    /// Remember new page version, as a full page image
+    pub fn put_page_image(&self, blknum: SegmentBlk, lsn: Lsn, img: Bytes) -> Result<u32> {
+        self.put_page_version(blknum, lsn, PageVersion::Page(img))
+    }
+
     /// Common subroutine of the public put_wal_record() and put_page_image() functions.
     /// Adds the page version to the in-memory tree
-    pub fn put_value(&self, key: Key, lsn: Lsn, val: Value) -> Result<()> {
-        trace!("put_value key {} at {}/{}", key, self.timelineid, lsn);
+    pub fn put_page_version(&self, blknum: SegmentBlk, lsn: Lsn, pv: PageVersion) -> Result<u32> {
+        assert!((0..RELISH_SEG_SIZE).contains(&blknum));
+
+        trace!(
+            "put_page_version blk {} of {} at {}/{}",
+            blknum,
+            self.seg.rel,
+            self.timelineid,
+            lsn
+        );
         let mut inner = self.inner.write().unwrap();
 
         inner.assert_writeable();
+        assert!(lsn >= inner.latest_lsn);
+        inner.latest_lsn = lsn;
 
-        let off = inner.file.write_blob(&Value::ser(&val)?)?;
-
-        let vec_map = inner.index.entry(key).or_default();
-        let old = vec_map.append_or_update_last(lsn, off).unwrap().0;
-        if old.is_some() {
-            // We already had an entry for this LSN. That's odd..
-            warn!("Key {} at {} already exists", key, lsn);
+        // Write the page version to the file, and remember its offset in 'page_versions'
+        {
+            let off = inner.write_pv(&pv)?;
+            let vec_map = inner.page_versions.entry(blknum).or_default();
+            let old = vec_map.append_or_update_last(lsn, off).unwrap().0;
+            if old.is_some() {
+                // We already had an entry for this LSN. That's odd..
+                warn!(
+                    "Page version of rel {} blk {} at {} already exists",
+                    self.seg.rel, blknum, lsn
+                );
+            }
         }
 
-        Ok(())
+        // Also update the relation size, if this extended the relation.
+        if self.seg.rel.is_blocky() {
+            let newsize = blknum + 1;
+
+            // use inner get_seg_size, since calling self.get_seg_size will try to acquire the lock,
+            // which we've just acquired above
+            let oldsize = inner.get_seg_size(lsn);
+            if newsize > oldsize {
+                trace!(
+                    "enlarging segment {} from {} to {} blocks at {}",
+                    self.seg,
+                    oldsize,
+                    newsize,
+                    lsn
+                );
+
+                // If we are extending the relation by more than one page, initialize the "gap"
+                // with zeros
+                //
+                // XXX: What if the caller initializes the gap with subsequent call with same LSN?
+                // I don't think that can happen currently, but that is highly dependent on how
+                // PostgreSQL writes its WAL records and there's no guarantee of it. If it does
+                // happen, we would hit the "page version already exists" warning above on the
+                // subsequent call to initialize the gap page.
+                for gapblknum in oldsize..blknum {
+                    let zeropv = PageVersion::Page(ZERO_PAGE.clone());
+                    trace!(
+                        "filling gap blk {} with zeros for write of {}",
+                        gapblknum,
+                        blknum
+                    );
+
+                    // Write the page version to the file, and remember its offset in
+                    // 'page_versions'
+                    {
+                        let off = inner.write_pv(&zeropv)?;
+                        let vec_map = inner.page_versions.entry(gapblknum).or_default();
+                        let old = vec_map.append_or_update_last(lsn, off).unwrap().0;
+                        if old.is_some() {
+                            warn!(
+                                "Page version of seg {} blk {} at {} already exists",
+                                self.seg, gapblknum, lsn
+                            );
+                        }
+                    }
+                }
+
+                inner.seg_sizes.append_or_update_last(lsn, newsize).unwrap();
+                return Ok(newsize - oldsize);
+            }
+        }
+
+        Ok(0)
     }
 
-    pub fn put_tombstone(&self, _key_range: Range<Key>, _lsn: Lsn) -> Result<()> {
-        // TODO: Currently, we just leak the storage for any deleted keys
+    /// Remember that the relation was truncated at given LSN
+    pub fn put_truncation(&self, lsn: Lsn, new_size: SegmentBlk) {
+        assert!(
+            self.seg.rel.is_blocky(),
+            "put_truncation() called on a non-blocky rel"
+        );
 
-        Ok(())
+        let mut inner = self.inner.write().unwrap();
+        inner.assert_writeable();
+
+        // check that this we truncate to a smaller size than segment was before the truncation
+        let old_size = inner.get_seg_size(lsn);
+        assert!(new_size < old_size);
+
+        let (old, _delta_size) = inner
+            .seg_sizes
+            .append_or_update_last(lsn, new_size)
+            .unwrap();
+
+        if old.is_some() {
+            // We already had an entry for this LSN. That's odd..
+            warn!("Inserting truncation, but had an entry for the LSN already");
+        }
+    }
+
+    /// Remember that the segment was dropped at given LSN
+    pub fn drop_segment(&self, lsn: Lsn) {
+        let mut inner = self.inner.write().unwrap();
+
+        assert!(inner.end_lsn.is_none());
+        assert!(!inner.dropped);
+        inner.dropped = true;
+        assert!(self.start_lsn < lsn);
+        inner.end_lsn = Some(lsn);
+
+        trace!("dropped segment {} at {}", self.seg, lsn);
+    }
+
+    ///
+    /// Initialize a new InMemoryLayer for, by copying the state at the given
+    /// point in time from given existing layer.
+    ///
+    pub fn create_successor_layer(
+        conf: &'static PageServerConf,
+        src: Arc<dyn Layer>,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        start_lsn: Lsn,
+        oldest_lsn: Lsn,
+    ) -> Result<InMemoryLayer> {
+        let seg = src.get_seg_tag();
+
+        assert!(oldest_lsn.is_aligned());
+
+        trace!(
+            "initializing new InMemoryLayer for writing {} on timeline {} at {}",
+            seg,
+            timelineid,
+            start_lsn,
+        );
+
+        // Copy the segment size at the start LSN from the predecessor layer.
+        let mut seg_sizes = VecMap::default();
+        if seg.rel.is_blocky() {
+            let size = src.get_seg_size(start_lsn)?;
+            seg_sizes.append(start_lsn, size).unwrap();
+        }
+
+        let file = EphemeralFile::create(conf, tenantid, timelineid)?;
+
+        Ok(InMemoryLayer {
+            conf,
+            timelineid,
+            tenantid,
+            seg,
+            start_lsn,
+            oldest_lsn,
+            incremental: true,
+            inner: RwLock::new(InMemoryLayerInner {
+                end_lsn: None,
+                dropped: false,
+                file,
+                page_versions: HashMap::new(),
+                seg_sizes,
+                latest_lsn: oldest_lsn,
+            }),
+        })
+    }
+
+    pub fn is_writeable(&self) -> bool {
+        let inner = self.inner.read().unwrap();
+        inner.end_lsn.is_none()
     }
 
     /// Make the layer non-writeable. Only call once.
     /// Records the end_lsn for non-dropped layers.
-    /// `end_lsn` is exclusive
+    /// `end_lsn` is inclusive
     pub fn freeze(&self, end_lsn: Lsn) {
         let mut inner = self.inner.write().unwrap();
 
-        assert!(self.start_lsn < end_lsn);
-        inner.end_lsn = Some(end_lsn);
+        if inner.end_lsn.is_some() {
+            assert!(inner.dropped);
+        } else {
+            assert!(!inner.dropped);
+            assert!(self.start_lsn < end_lsn + 1);
+            inner.end_lsn = Some(Lsn(end_lsn.0 + 1));
 
-        for vec_map in inner.index.values() {
-            for (lsn, _pos) in vec_map.as_slice() {
-                assert!(*lsn < end_lsn);
+            if let Some((lsn, _)) = inner.seg_sizes.as_slice().last() {
+                assert!(lsn <= &end_lsn, "{:?} {:?}", lsn, end_lsn);
+            }
+
+            for (_blk, vec_map) in inner.page_versions.iter() {
+                for (lsn, _pos) in vec_map.as_slice() {
+                    assert!(*lsn <= end_lsn);
+                }
             }
         }
     }
 
-    /// Write this frozen in-memory layer to disk.
+    /// Write the this frozen in-memory layer to disk.
     ///
-    /// Returns a new delta layer with all the same data as this in-memory layer
-    pub fn write_to_disk(&self) -> Result<DeltaLayer> {
+    /// Returns new layers that replace this one.
+    /// If not dropped and reconstruct_pages is true, returns a new image layer containing the page versions
+    /// at the `end_lsn`. Can also return a DeltaLayer that includes all the
+    /// WAL records between start and end LSN. (The delta layer is not needed
+    /// when a new relish is created with a single LSN, so that the start and
+    /// end LSN are the same.)
+    pub fn write_to_disk(
+        &self,
+        timeline: &LayeredTimeline,
+        reconstruct_pages: bool,
+    ) -> Result<LayersOnDisk> {
+        trace!(
+            "write_to_disk {} get_end_lsn is {}",
+            self.filename().display(),
+            self.get_end_lsn()
+        );
+
         // Grab the lock in read-mode. We hold it over the I/O, but because this
         // layer is not writeable anymore, no one should be trying to acquire the
         // write lock on it, so we shouldn't block anyone. There's one exception
@@ -322,32 +700,105 @@ impl InMemoryLayer {
         // rare though, so we just accept the potential latency hit for now.
         let inner = self.inner.read().unwrap();
 
-        let mut delta_layer_writer = DeltaLayerWriter::new(
-            self.conf,
-            self.timelineid,
-            self.tenantid,
-            Key::MIN,
-            self.start_lsn..inner.end_lsn.unwrap(),
-        )?;
+        // Since `end_lsn` is exclusive, subtract 1 to calculate the last LSN
+        // that is included.
+        let end_lsn_exclusive = inner.end_lsn.unwrap();
+        let end_lsn_inclusive = Lsn(end_lsn_exclusive.0 - 1);
 
-        let mut buf = Vec::new();
-
-        let mut cursor = inner.file.block_cursor();
-
-        let mut keys: Vec<(&Key, &VecMap<Lsn, u64>)> = inner.index.iter().collect();
-        keys.sort_by_key(|k| k.0);
-
-        for (key, vec_map) in keys.iter() {
-            let key = **key;
-            // Write all page versions
-            for (lsn, pos) in vec_map.as_slice() {
-                cursor.read_blob_into_buf(*pos, &mut buf)?;
-                let val = Value::des(&buf)?;
-                delta_layer_writer.put_value(key, *lsn, val)?;
-            }
+        // Figure out if we should create a delta layer, image layer, or both.
+        let image_lsn: Option<Lsn>;
+        let delta_end_lsn: Option<Lsn>;
+        if self.is_dropped() || !reconstruct_pages {
+            // The segment was dropped. Create just a delta layer containing all the
+            // changes up to and including the drop.
+            delta_end_lsn = Some(end_lsn_exclusive);
+            image_lsn = None;
+        } else if self.start_lsn == end_lsn_inclusive {
+            // The layer contains exactly one LSN. It's enough to write an image
+            // layer at that LSN.
+            delta_end_lsn = None;
+            image_lsn = Some(end_lsn_inclusive);
+        } else {
+            // Create a delta layer with all the changes up to the end LSN,
+            // and an image layer at the end LSN.
+            //
+            // Note that we the delta layer does *not* include the page versions
+            // at the end LSN. They are included in the image layer, and there's
+            // no need to store them twice.
+            delta_end_lsn = Some(end_lsn_inclusive);
+            image_lsn = Some(end_lsn_inclusive);
         }
 
-        let delta_layer = delta_layer_writer.finish(Key::MAX)?;
-        Ok(delta_layer)
+        let mut delta_layers = Vec::new();
+        let mut image_layers = Vec::new();
+
+        if let Some(delta_end_lsn) = delta_end_lsn {
+            let mut delta_layer_writer = DeltaLayerWriter::new(
+                self.conf,
+                self.timelineid,
+                self.tenantid,
+                self.seg,
+                self.start_lsn,
+                delta_end_lsn,
+                self.is_dropped(),
+            )?;
+
+            // Write all page versions, in block + LSN order
+            let mut buf: Vec<u8> = Vec::new();
+
+            let pv_iter = inner.page_versions.iter();
+            let mut pages: Vec<(&SegmentBlk, &VecMap<Lsn, u64>)> = pv_iter.collect();
+            pages.sort_by_key(|(blknum, _vec_map)| *blknum);
+            for (blknum, vec_map) in pages {
+                for (lsn, pos) in vec_map.as_slice() {
+                    if *lsn < delta_end_lsn {
+                        let len = inner.read_pv_bytes(*pos, &mut buf)?;
+                        delta_layer_writer.put_page_version(*blknum, *lsn, &buf[..len])?;
+                    }
+                }
+            }
+
+            // Create seg_sizes
+            let seg_sizes = if delta_end_lsn == end_lsn_exclusive {
+                inner.seg_sizes.clone()
+            } else {
+                inner.seg_sizes.split_at(&end_lsn_exclusive).0
+            };
+
+            let delta_layer = delta_layer_writer.finish(seg_sizes)?;
+            delta_layers.push(delta_layer);
+        }
+
+        drop(inner);
+
+        // Write a new base image layer at the cutoff point
+        if let Some(image_lsn) = image_lsn {
+            let size = if self.seg.rel.is_blocky() {
+                self.get_seg_size(image_lsn)?
+            } else {
+                1
+            };
+            let mut image_layer_writer = ImageLayerWriter::new(
+                self.conf,
+                self.timelineid,
+                self.tenantid,
+                self.seg,
+                image_lsn,
+                size,
+            )?;
+
+            for blknum in 0..size {
+                let img = timeline.materialize_page(self.seg, blknum, image_lsn, &*self)?;
+
+                image_layer_writer.put_page_image(&img)?;
+            }
+            let image_layer = image_layer_writer.finish()?;
+            image_layers.push(image_layer);
+        }
+
+        Ok(LayersOnDisk {
+            delta_layers,
+            image_layers,
+        })
     }
 }

pageserver/src/layered_repository/interval_tree.rs

@@ -0,0 +1,468 @@
+///
+/// IntervalTree is data structure for holding intervals. It is generic
+/// to make unit testing possible, but the only real user of it is the layer map,
+///
+/// It's inspired by the "segment tree" or a "statistic tree" as described in
+/// https://en.wikipedia.org/wiki/Segment_tree. However, we use a B-tree to hold
+/// the points instead of a binary tree. This is called an "interval tree" instead
+/// of "segment tree" because the term "segment" is already using Zenith to mean
+/// something else. To add to the confusion, there is another data structure known
+/// as "interval tree" out there (see https://en.wikipedia.org/wiki/Interval_tree),
+/// for storing intervals, but this isn't that.
+///
+/// The basic idea is to have a B-tree of "interesting Points". At each Point,
+/// there is a list of intervals that contain the point. The Points are formed
+/// from the start bounds of each interval; there is a Point for each distinct
+/// start bound.
+///
+/// Operations:
+///
+/// To find intervals that contain a given point, you search the b-tree to find
+/// the nearest Point <= search key. Then you just return the list of intervals.
+///
+/// To insert an interval, find the Point with start key equal to the inserted item.
+/// If the Point doesn't exist yet, create it, by copying all the items from the
+/// previous Point that cover the new Point. Then walk right, inserting the new
+/// interval to all the Points that are contained by the new interval (including the
+/// newly created Point).
+///
+/// To remove an interval, you scan the tree for all the Points that are contained by
+/// the removed interval, and remove it from the list in each Point.
+///
+/// Requirements and assumptions:
+///
+/// - Can store overlapping items
+/// - But there are not many overlapping items
+/// - The interval bounds don't change after it is added to the tree
+/// - Intervals are uniquely identified by pointer equality. You must not be insert the
+///   same interval object twice, and `remove` uses pointer equality to remove the right
+///   interval. It is OK to have two intervals with the same bounds, however.
+///
+use std::collections::BTreeMap;
+use std::fmt::Debug;
+use std::ops::Range;
+use std::sync::Arc;
+
+pub struct IntervalTree<I: ?Sized>
+where
+    I: IntervalItem,
+{
+    points: BTreeMap<I::Key, Point<I>>,
+}
+
+struct Point<I: ?Sized> {
+    /// All intervals that contain this point, in no particular order.
+    ///
+    /// We assume that there aren't a lot of overlappingg intervals, so that this vector
+    /// never grows very large. If that assumption doesn't hold, we could keep this ordered
+    /// by the end bound, to speed up `search`. But as long as there are only a few elements,
+    /// a linear search is OK.
+    elements: Vec<Arc<I>>,
+}
+
+/// Abstraction for an interval that can be stored in the tree
+///
+/// The start bound is inclusive and the end bound is exclusive. End must be greater
+/// than start.
+pub trait IntervalItem {
+    type Key: Ord + Copy + Debug + Sized;
+
+    fn start_key(&self) -> Self::Key;
+    fn end_key(&self) -> Self::Key;
+
+    fn bounds(&self) -> Range<Self::Key> {
+        self.start_key()..self.end_key()
+    }
+}
+
+impl<I: ?Sized> IntervalTree<I>
+where
+    I: IntervalItem,
+{
+    /// Return an element that contains 'key', or precedes it.
+    ///
+    /// If there are multiple candidates, returns the one with the highest 'end' key.
+    pub fn search(&self, key: I::Key) -> Option<Arc<I>> {
+        // Find the greatest point that precedes or is equal to the search key. If there is
+        // none, returns None.
+        let (_, p) = self.points.range(..=key).next_back()?;
+
+        // Find the element with the highest end key at this point
+        let highest_item = p
+            .elements
+            .iter()
+            .reduce(|a, b| {
+                // starting with Rust 1.53, could use `std::cmp::min_by_key` here
+                if a.end_key() > b.end_key() {
+                    a
+                } else {
+                    b
+                }
+            })
+            .unwrap();
+        Some(Arc::clone(highest_item))
+    }
+
+    /// Iterate over all items with start bound >= 'key'
+    pub fn iter_newer(&self, key: I::Key) -> IntervalIter<I> {
+        IntervalIter {
+            point_iter: self.points.range(key..),
+            elem_iter: None,
+        }
+    }
+
+    /// Iterate over all items
+    pub fn iter(&self) -> IntervalIter<I> {
+        IntervalIter {
+            point_iter: self.points.range(..),
+            elem_iter: None,
+        }
+    }
+
+    pub fn insert(&mut self, item: Arc<I>) {
+        let start_key = item.start_key();
+        let end_key = item.end_key();
+        assert!(start_key < end_key);
+        let bounds = start_key..end_key;
+
+        // Find the starting point and walk forward from there
+        let mut found_start_point = false;
+        let iter = self.points.range_mut(bounds);
+        for (point_key, point) in iter {
+            if *point_key == start_key {
+                found_start_point = true;
+                // It is an error to insert the same item to the tree twice.
+                assert!(
+                    !point.elements.iter().any(|x| Arc::ptr_eq(x, &item)),
+                    "interval is already in the tree"
+                );
+            }
+            point.elements.push(Arc::clone(&item));
+        }
+        if !found_start_point {
+            // Create a new Point for the starting point
+
+            // Look at the previous point, and copy over elements that overlap with this
+            // new point
+            let mut new_elements: Vec<Arc<I>> = Vec::new();
+            if let Some((_, prev_point)) = self.points.range(..start_key).next_back() {
+                let overlapping_prev_elements = prev_point
+                    .elements
+                    .iter()
+                    .filter(|x| x.bounds().contains(&start_key))
+                    .cloned();
+
+                new_elements.extend(overlapping_prev_elements);
+            }
+            new_elements.push(item);
+
+            let new_point = Point {
+                elements: new_elements,
+            };
+            self.points.insert(start_key, new_point);
+        }
+    }
+
+    pub fn remove(&mut self, item: &Arc<I>) {
+        // range search points
+        let start_key = item.start_key();
+        let end_key = item.end_key();
+        let bounds = start_key..end_key;
+
+        let mut points_to_remove: Vec<I::Key> = Vec::new();
+        let mut found_start_point = false;
+        for (point_key, point) in self.points.range_mut(bounds) {
+            if *point_key == start_key {
+                found_start_point = true;
+            }
+            let len_before = point.elements.len();
+            point.elements.retain(|other| !Arc::ptr_eq(other, item));
+            let len_after = point.elements.len();
+            assert_eq!(len_after + 1, len_before);
+            if len_after == 0 {
+                points_to_remove.push(*point_key);
+            }
+        }
+        assert!(found_start_point);
+
+        for k in points_to_remove {
+            self.points.remove(&k).unwrap();
+        }
+    }
+}
+
+pub struct IntervalIter<'a, I: ?Sized>
+where
+    I: IntervalItem,
+{
+    point_iter: std::collections::btree_map::Range<'a, I::Key, Point<I>>,
+    elem_iter: Option<(I::Key, std::slice::Iter<'a, Arc<I>>)>,
+}
+
+impl<'a, I> Iterator for IntervalIter<'a, I>
+where
+    I: IntervalItem + ?Sized,
+{
+    type Item = Arc<I>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // Iterate over all elements in all the points in 'point_iter'. To avoid
+        // returning the same element twice, we only return each element at its
+        // starting point.
+        loop {
+            // Return next remaining element from the current point
+            if let Some((point_key, elem_iter)) = &mut self.elem_iter {
+                for elem in elem_iter {
+                    if elem.start_key() == *point_key {
+                        return Some(Arc::clone(elem));
+                    }
+                }
+            }
+            // No more elements at this point. Move to next point.
+            if let Some((point_key, point)) = self.point_iter.next() {
+                self.elem_iter = Some((*point_key, point.elements.iter()));
+                continue;
+            } else {
+                // No more points, all done
+                return None;
+            }
+        }
+    }
+}
+
+impl<I: ?Sized> Default for IntervalTree<I>
+where
+    I: IntervalItem,
+{
+    fn default() -> Self {
+        IntervalTree {
+            points: BTreeMap::new(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fmt;
+
+    #[derive(Debug)]
+    struct MockItem {
+        start_key: u32,
+        end_key: u32,
+        val: String,
+    }
+    impl IntervalItem for MockItem {
+        type Key = u32;
+
+        fn start_key(&self) -> u32 {
+            self.start_key
+        }
+        fn end_key(&self) -> u32 {
+            self.end_key
+        }
+    }
+    impl MockItem {
+        fn new(start_key: u32, end_key: u32) -> Self {
+            MockItem {
+                start_key,
+                end_key,
+                val: format!("{}-{}", start_key, end_key),
+            }
+        }
+        fn new_str(start_key: u32, end_key: u32, val: &str) -> Self {
+            MockItem {
+                start_key,
+                end_key,
+                val: format!("{}-{}: {}", start_key, end_key, val),
+            }
+        }
+    }
+    impl fmt::Display for MockItem {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            write!(f, "{}", self.val)
+        }
+    }
+    #[rustfmt::skip]
+    fn assert_search(
+        tree: &IntervalTree<MockItem>,
+        key: u32,
+        expected: &[&str],
+    ) -> Option<Arc<MockItem>> {
+        if let Some(v) = tree.search(key) {
+            let vstr = v.to_string();
+
+            assert!(!expected.is_empty(), "search with {} returned {}, expected None", key, v);
+            assert!(
+                expected.contains(&vstr.as_str()),
+                "search with {} returned {}, expected one of: {:?}",
+                key, v, expected,
+            );
+
+            Some(v)
+        } else {
+            assert!(
+                expected.is_empty(),
+                "search with {} returned None, expected one of {:?}",
+                key, expected
+            );
+            None
+        }
+    }
+
+    fn assert_contents(tree: &IntervalTree<MockItem>, expected: &[&str]) {
+        let mut contents: Vec<String> = tree.iter().map(|e| e.to_string()).collect();
+        contents.sort();
+        assert_eq!(contents, expected);
+    }
+
+    fn dump_tree(tree: &IntervalTree<MockItem>) {
+        for (point_key, point) in tree.points.iter() {
+            print!("{}:", point_key);
+            for e in point.elements.iter() {
+                print!(" {}", e);
+            }
+            println!();
+        }
+    }
+
+    #[test]
+    fn test_interval_tree_simple() {
+        let mut tree: IntervalTree<MockItem> = IntervalTree::default();
+
+        // Simple, non-overlapping ranges.
+        tree.insert(Arc::new(MockItem::new(10, 11)));
+        tree.insert(Arc::new(MockItem::new(11, 12)));
+        tree.insert(Arc::new(MockItem::new(12, 13)));
+        tree.insert(Arc::new(MockItem::new(18, 19)));
+        tree.insert(Arc::new(MockItem::new(17, 18)));
+        tree.insert(Arc::new(MockItem::new(15, 16)));
+
+        assert_search(&tree, 9, &[]);
+        assert_search(&tree, 10, &["10-11"]);
+        assert_search(&tree, 11, &["11-12"]);
+        assert_search(&tree, 12, &["12-13"]);
+        assert_search(&tree, 13, &["12-13"]);
+        assert_search(&tree, 14, &["12-13"]);
+        assert_search(&tree, 15, &["15-16"]);
+        assert_search(&tree, 16, &["15-16"]);
+        assert_search(&tree, 17, &["17-18"]);
+        assert_search(&tree, 18, &["18-19"]);
+        assert_search(&tree, 19, &["18-19"]);
+        assert_search(&tree, 20, &["18-19"]);
+
+        // remove a few entries and search around them again
+        tree.remove(&assert_search(&tree, 10, &["10-11"]).unwrap()); // first entry
+        tree.remove(&assert_search(&tree, 12, &["12-13"]).unwrap()); // entry in the middle
+        tree.remove(&assert_search(&tree, 18, &["18-19"]).unwrap()); // last entry
+        assert_search(&tree, 9, &[]);
+        assert_search(&tree, 10, &[]);
+        assert_search(&tree, 11, &["11-12"]);
+        assert_search(&tree, 12, &["11-12"]);
+        assert_search(&tree, 14, &["11-12"]);
+        assert_search(&tree, 15, &["15-16"]);
+        assert_search(&tree, 17, &["17-18"]);
+        assert_search(&tree, 18, &["17-18"]);
+    }
+
+    #[test]
+    fn test_interval_tree_overlap() {
+        let mut tree: IntervalTree<MockItem> = IntervalTree::default();
+
+        // Overlapping items
+        tree.insert(Arc::new(MockItem::new(22, 24)));
+        tree.insert(Arc::new(MockItem::new(23, 25)));
+        let x24_26 = Arc::new(MockItem::new(24, 26));
+        tree.insert(Arc::clone(&x24_26));
+        let x26_28 = Arc::new(MockItem::new(26, 28));
+        tree.insert(Arc::clone(&x26_28));
+        tree.insert(Arc::new(MockItem::new(25, 27)));
+
+        assert_search(&tree, 22, &["22-24"]);
+        assert_search(&tree, 23, &["22-24", "23-25"]);
+        assert_search(&tree, 24, &["23-25", "24-26"]);
+        assert_search(&tree, 25, &["24-26", "25-27"]);
+        assert_search(&tree, 26, &["25-27", "26-28"]);
+        assert_search(&tree, 27, &["26-28"]);
+        assert_search(&tree, 28, &["26-28"]);
+        assert_search(&tree, 29, &["26-28"]);
+
+        tree.remove(&x24_26);
+        tree.remove(&x26_28);
+        assert_search(&tree, 23, &["22-24", "23-25"]);
+        assert_search(&tree, 24, &["23-25"]);
+        assert_search(&tree, 25, &["25-27"]);
+        assert_search(&tree, 26, &["25-27"]);
+        assert_search(&tree, 27, &["25-27"]);
+        assert_search(&tree, 28, &["25-27"]);
+        assert_search(&tree, 29, &["25-27"]);
+    }
+
+    #[test]
+    fn test_interval_tree_nested() {
+        let mut tree: IntervalTree<MockItem> = IntervalTree::default();
+
+        // Items containing other items
+        tree.insert(Arc::new(MockItem::new(31, 39)));
+        tree.insert(Arc::new(MockItem::new(32, 34)));
+        tree.insert(Arc::new(MockItem::new(33, 35)));
+        tree.insert(Arc::new(MockItem::new(30, 40)));
+
+        assert_search(&tree, 30, &["30-40"]);
+        assert_search(&tree, 31, &["30-40", "31-39"]);
+        assert_search(&tree, 32, &["30-40", "32-34", "31-39"]);
+        assert_search(&tree, 33, &["30-40", "32-34", "33-35", "31-39"]);
+        assert_search(&tree, 34, &["30-40", "33-35", "31-39"]);
+        assert_search(&tree, 35, &["30-40", "31-39"]);
+        assert_search(&tree, 36, &["30-40", "31-39"]);
+        assert_search(&tree, 37, &["30-40", "31-39"]);
+        assert_search(&tree, 38, &["30-40", "31-39"]);
+        assert_search(&tree, 39, &["30-40"]);
+        assert_search(&tree, 40, &["30-40"]);
+        assert_search(&tree, 41, &["30-40"]);
+    }
+
+    #[test]
+    fn test_interval_tree_duplicates() {
+        let mut tree: IntervalTree<MockItem> = IntervalTree::default();
+
+        // Duplicate keys
+        let item_a = Arc::new(MockItem::new_str(55, 56, "a"));
+        tree.insert(Arc::clone(&item_a));
+        let item_b = Arc::new(MockItem::new_str(55, 56, "b"));
+        tree.insert(Arc::clone(&item_b));
+        let item_c = Arc::new(MockItem::new_str(55, 56, "c"));
+        tree.insert(Arc::clone(&item_c));
+        let item_d = Arc::new(MockItem::new_str(54, 56, "d"));
+        tree.insert(Arc::clone(&item_d));
+        let item_e = Arc::new(MockItem::new_str(55, 57, "e"));
+        tree.insert(Arc::clone(&item_e));
+
+        dump_tree(&tree);
+
+        assert_search(
+            &tree,
+            55,
+            &["55-56: a", "55-56: b", "55-56: c", "54-56: d", "55-57: e"],
+        );
+        tree.remove(&item_b);
+        dump_tree(&tree);
+
+        assert_contents(&tree, &["54-56: d", "55-56: a", "55-56: c", "55-57: e"]);
+
+        tree.remove(&item_d);
+        dump_tree(&tree);
+        assert_contents(&tree, &["55-56: a", "55-56: c", "55-57: e"]);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_interval_tree_insert_twice() {
+        let mut tree: IntervalTree<MockItem> = IntervalTree::default();
+
+        // Inserting the same item twice is not cool
+        let item = Arc::new(MockItem::new(1, 2));
+        tree.insert(Arc::clone(&item));
+        tree.insert(Arc::clone(&item)); // fails assertion
+    }
+}

pageserver/src/layered_repository/layer_map.rs

@@ -1,29 +1,32 @@
 //!
-//! The layer map tracks what layers exist in a timeline.
+//! The layer map tracks what layers exist for all the relishes in a timeline.
 //!
 //! When the timeline is first accessed, the server lists of all layer files
 //! in the timelines/<timelineid> directory, and populates this map with
-//! ImageLayer and DeltaLayer structs corresponding to each file. When the first
-//! new WAL record is received, we create an InMemoryLayer to hold the incoming
-//! records. Now and then, in the checkpoint() function, the in-memory layer is
-//! are frozen, and it is split up into new image and delta layers and the
-//! corresponding files are written to disk.
+//! ImageLayer and DeltaLayer structs corresponding to each file. When new WAL
+//! is received, we create InMemoryLayers to hold the incoming records. Now and
+//! then, in the checkpoint() function, the in-memory layers are frozen, forming
+//! new image and delta layers and corresponding files are written to disk.
 //!
 
-use crate::layered_repository::storage_layer::Layer;
-use crate::layered_repository::storage_layer::{range_eq, range_overlaps};
+use crate::layered_repository::interval_tree::{IntervalItem, IntervalIter, IntervalTree};
+use crate::layered_repository::storage_layer::{Layer, SegmentTag};
 use crate::layered_repository::InMemoryLayer;
-use crate::repository::Key;
+use crate::relish::*;
 use anyhow::Result;
 use lazy_static::lazy_static;
-use metrics::{register_int_gauge, IntGauge};
-use std::collections::VecDeque;
-use std::ops::Range;
+use std::cmp::Ordering;
+use std::collections::{BinaryHeap, HashMap};
 use std::sync::Arc;
-use tracing::*;
-use utils::lsn::Lsn;
+use zenith_metrics::{register_int_gauge, IntGauge};
+use zenith_utils::lsn::Lsn;
+
+use super::global_layer_map::{LayerId, GLOBAL_LAYER_MAP};
 
 lazy_static! {
+    static ref NUM_INMEMORY_LAYERS: IntGauge =
+        register_int_gauge!("pageserver_inmemory_layers", "Number of layers in memory")
+            .expect("failed to define a metric");
     static ref NUM_ONDISK_LAYERS: IntGauge =
         register_int_gauge!("pageserver_ondisk_layers", "Number of layers on-disk")
             .expect("failed to define a metric");
@@ -34,142 +37,98 @@ lazy_static! {
 ///
 #[derive(Default)]
 pub struct LayerMap {
-    //
-    // 'open_layer' holds the current InMemoryLayer that is accepting new
-    // records. If it is None, 'next_open_layer_at' will be set instead, indicating
-    // where the start LSN of the next InMemoryLayer that is to be created.
-    //
-    pub open_layer: Option<Arc<InMemoryLayer>>,
-    pub next_open_layer_at: Option<Lsn>,
+    /// All the layers keyed by segment tag
+    segs: HashMap<SegmentTag, SegEntry>,
 
-    ///
-    /// Frozen layers, if any. Frozen layers are in-memory layers that
-    /// are no longer added to, but haven't been written out to disk
-    /// yet. They contain WAL older than the current 'open_layer' or
-    /// 'next_open_layer_at', but newer than any historic layer.
-    /// The frozen layers are in order from oldest to newest, so that
-    /// the newest one is in the 'back' of the VecDeque, and the oldest
-    /// in the 'front'.
-    ///
-    pub frozen_layers: VecDeque<Arc<InMemoryLayer>>,
+    /// All in-memory layers, ordered by 'oldest_lsn' and generation
+    /// of each layer. This allows easy access to the in-memory layer that
+    /// contains the oldest WAL record.
+    open_layers: BinaryHeap<OpenLayerEntry>,
 
-    /// All the historic layers are kept here
-
-    /// TODO: This is a placeholder implementation of a data structure
-    /// to hold information about all the layer files on disk and in
-    /// S3. Currently, it's just a vector and all operations perform a
-    /// linear scan over it.  That obviously becomes slow as the
-    /// number of layers grows. I'm imagining that an R-tree or some
-    /// other 2D data structure would be the long-term solution here.
-    historic_layers: Vec<Arc<dyn Layer>>,
-}
-
-/// Return value of LayerMap::search
-pub struct SearchResult {
-    pub layer: Arc<dyn Layer>,
-    pub lsn_floor: Lsn,
+    /// Generation number, used to distinguish newly inserted entries in the
+    /// binary heap from older entries during checkpoint.
+    current_generation: u64,
 }
 
 impl LayerMap {
     ///
-    /// Find the latest layer that covers the given 'key', with lsn <
-    /// 'end_lsn'.
+    /// Look up a layer using the given segment tag and LSN. This differs from a
+    /// plain key-value lookup in that if there is any layer that covers the
+    /// given LSN, or precedes the given LSN, it is returned. In other words,
+    /// you don't need to know the exact start LSN of the layer.
     ///
-    /// Returns the layer, if any, and an 'lsn_floor' value that
-    /// indicates which portion of the layer the caller should
-    /// check. 'lsn_floor' is normally the start-LSN of the layer, but
-    /// can be greater if there is an overlapping layer that might
-    /// contain the version, even if it's missing from the returned
-    /// layer.
+    pub fn get(&self, tag: &SegmentTag, lsn: Lsn) -> Option<Arc<dyn Layer>> {
+        let segentry = self.segs.get(tag)?;
+
+        segentry.get(lsn)
+    }
+
     ///
-    pub fn search(&self, key: Key, end_lsn: Lsn) -> Result<Option<SearchResult>> {
-        // linear search
-        // Find the latest image layer that covers the given key
-        let mut latest_img: Option<Arc<dyn Layer>> = None;
-        let mut latest_img_lsn: Option<Lsn> = None;
-        for l in self.historic_layers.iter() {
-            if l.is_incremental() {
-                continue;
-            }
-            if !l.get_key_range().contains(&key) {
-                continue;
-            }
-            let img_lsn = l.get_lsn_range().start;
+    /// Get the open layer for given segment for writing. Or None if no open
+    /// layer exists.
+    ///
+    pub fn get_open(&self, tag: &SegmentTag) -> Option<Arc<InMemoryLayer>> {
+        let segentry = self.segs.get(tag)?;
 
-            if img_lsn >= end_lsn {
-                // too new
-                continue;
-            }
-            if Lsn(img_lsn.0 + 1) == end_lsn {
-                // found exact match
-                return Ok(Some(SearchResult {
-                    layer: Arc::clone(l),
-                    lsn_floor: img_lsn,
-                }));
-            }
-            if img_lsn > latest_img_lsn.unwrap_or(Lsn(0)) {
-                latest_img = Some(Arc::clone(l));
-                latest_img_lsn = Some(img_lsn);
-            }
-        }
+        segentry
+            .open_layer_id
+            .and_then(|layer_id| GLOBAL_LAYER_MAP.read().unwrap().get(&layer_id))
+    }
 
-        // Search the delta layers
-        let mut latest_delta: Option<Arc<dyn Layer>> = None;
-        for l in self.historic_layers.iter() {
-            if !l.is_incremental() {
-                continue;
-            }
-            if !l.get_key_range().contains(&key) {
-                continue;
-            }
+    ///
+    /// Insert an open in-memory layer
+    ///
+    pub fn insert_open(&mut self, layer: Arc<InMemoryLayer>) {
+        let segentry = self.segs.entry(layer.get_seg_tag()).or_default();
 
-            if l.get_lsn_range().start >= end_lsn {
-                // too new
-                continue;
-            }
+        let layer_id = segentry.update_open(Arc::clone(&layer));
 
-            if l.get_lsn_range().end >= end_lsn {
-                // this layer contains the requested point in the key/lsn space.
-                // No need to search any further
-                trace!(
-                    "found layer {} for request on {key} at {end_lsn}",
-                    l.filename().display(),
-                );
-                latest_delta.replace(Arc::clone(l));
-                break;
-            }
-            // this layer's end LSN is smaller than the requested point. If there's
-            // nothing newer, this is what we need to return. Remember this.
-            if let Some(old_candidate) = &latest_delta {
-                if l.get_lsn_range().end > old_candidate.get_lsn_range().end {
-                    latest_delta.replace(Arc::clone(l));
-                }
+        let oldest_lsn = layer.get_oldest_lsn();
+
+        // After a crash and restart, 'oldest_lsn' of the oldest in-memory
+        // layer becomes the WAL streaming starting point, so it better not point
+        // in the middle of a WAL record.
+        assert!(oldest_lsn.is_aligned());
+
+        // Also add it to the binary heap
+        let open_layer_entry = OpenLayerEntry {
+            oldest_lsn: layer.get_oldest_lsn(),
+            layer_id,
+            generation: self.current_generation,
+        };
+        self.open_layers.push(open_layer_entry);
+
+        NUM_INMEMORY_LAYERS.inc();
+    }
+
+    /// Remove an open in-memory layer
+    pub fn remove_open(&mut self, layer_id: LayerId) {
+        // Note: we don't try to remove the entry from the binary heap.
+        // It will be removed lazily by peek_oldest_open() when it's made it to
+        // the top of the heap.
+
+        let layer_opt = {
+            let mut global_map = GLOBAL_LAYER_MAP.write().unwrap();
+            let layer_opt = global_map.get(&layer_id);
+            global_map.remove(&layer_id);
+            // TODO it's bad that a ref can still exist after being evicted from cache
+            layer_opt
+        };
+
+        if let Some(layer) = layer_opt {
+            let mut segentry = self.segs.get_mut(&layer.get_seg_tag()).unwrap();
+
+            if segentry.open_layer_id == Some(layer_id) {
+                // Also remove it from the SegEntry of this segment
+                segentry.open_layer_id = None;
             } else {
-                latest_delta.replace(Arc::clone(l));
+                // We could have already updated segentry.open for
+                // dropped (non-writeable) layer. This is fine.
+                assert!(!layer.is_writeable());
+                assert!(layer.is_dropped());
             }
-        }
-        if let Some(l) = latest_delta {
-            trace!(
-                "found (old) layer {} for request on {key} at {end_lsn}",
-                l.filename().display(),
-            );
-            let lsn_floor = std::cmp::max(
-                Lsn(latest_img_lsn.unwrap_or(Lsn(0)).0 + 1),
-                l.get_lsn_range().start,
-            );
-            Ok(Some(SearchResult {
-                lsn_floor,
-                layer: l,
-            }))
-        } else if let Some(l) = latest_img {
-            trace!("found img layer and no deltas for request on {key} at {end_lsn}");
-            Ok(Some(SearchResult {
-                lsn_floor: latest_img_lsn.unwrap(),
-                layer: l,
-            }))
-        } else {
-            trace!("no layer found for request on {key} at {end_lsn}");
-            Ok(None)
+
+            NUM_INMEMORY_LAYERS.dec();
         }
     }
 
@@ -177,7 +136,9 @@ impl LayerMap {
     /// Insert an on-disk layer
     ///
     pub fn insert_historic(&mut self, layer: Arc<dyn Layer>) {
-        self.historic_layers.push(layer);
+        let segentry = self.segs.entry(layer.get_seg_tag()).or_default();
+        segentry.insert_historic(layer);
+
         NUM_ONDISK_LAYERS.inc();
     }
 
@@ -187,205 +148,347 @@ impl LayerMap {
     /// This should be called when the corresponding file on disk has been deleted.
     ///
     pub fn remove_historic(&mut self, layer: Arc<dyn Layer>) {
-        let len_before = self.historic_layers.len();
+        let tag = layer.get_seg_tag();
 
-        // FIXME: ptr_eq might fail to return true for 'dyn'
-        // references.  Clippy complains about this. In practice it
-        // seems to work, the assertion below would be triggered
-        // otherwise but this ought to be fixed.
-        #[allow(clippy::vtable_address_comparisons)]
-        self.historic_layers
-            .retain(|other| !Arc::ptr_eq(other, &layer));
-
-        assert_eq!(self.historic_layers.len(), len_before - 1);
+        if let Some(segentry) = self.segs.get_mut(&tag) {
+            segentry.historic.remove(&layer);
+        }
         NUM_ONDISK_LAYERS.dec();
     }
 
-    /// Is there a newer image layer for given key-range?
+    // List relations along with a flag that marks if they exist at the given lsn.
+    // spcnode 0 and dbnode 0 have special meanings and mean all tabespaces/databases.
+    // Pass Tag if we're only interested in some relations.
+    pub fn list_relishes(&self, tag: Option<RelTag>, lsn: Lsn) -> Result<HashMap<RelishTag, bool>> {
+        let mut rels: HashMap<RelishTag, bool> = HashMap::new();
+
+        for (seg, segentry) in self.segs.iter() {
+            match seg.rel {
+                RelishTag::Relation(reltag) => {
+                    if let Some(request_rel) = tag {
+                        if (request_rel.spcnode == 0 || reltag.spcnode == request_rel.spcnode)
+                            && (request_rel.dbnode == 0 || reltag.dbnode == request_rel.dbnode)
+                        {
+                            if let Some(exists) = segentry.exists_at_lsn(lsn)? {
+                                rels.insert(seg.rel, exists);
+                            }
+                        }
+                    }
+                }
+                _ => {
+                    if tag == None {
+                        if let Some(exists) = segentry.exists_at_lsn(lsn)? {
+                            rels.insert(seg.rel, exists);
+                        }
+                    }
+                }
+            }
+        }
+        Ok(rels)
+    }
+
+    /// Is there a newer image layer for given segment?
     ///
     /// This is used for garbage collection, to determine if an old layer can
     /// be deleted.
-    /// We ignore layers newer than disk_consistent_lsn because they will be removed at restart
-    /// We also only look at historic layers
-    //#[allow(dead_code)]
+    /// We ignore segments newer than disk_consistent_lsn because they will be removed at restart
     pub fn newer_image_layer_exists(
         &self,
-        key_range: &Range<Key>,
+        seg: SegmentTag,
         lsn: Lsn,
         disk_consistent_lsn: Lsn,
-    ) -> Result<bool> {
-        let mut range_remain = key_range.clone();
-
-        loop {
-            let mut made_progress = false;
-            for l in self.historic_layers.iter() {
-                if l.is_incremental() {
-                    continue;
-                }
-                let img_lsn = l.get_lsn_range().start;
-                if !l.is_incremental()
-                    && l.get_key_range().contains(&range_remain.start)
-                    && img_lsn > lsn
-                    && img_lsn < disk_consistent_lsn
-                {
-                    made_progress = true;
-                    let img_key_end = l.get_key_range().end;
-
-                    if img_key_end >= range_remain.end {
-                        return Ok(true);
-                    }
-                    range_remain.start = img_key_end;
-                }
-            }
-
-            if !made_progress {
-                return Ok(false);
-            }
+    ) -> bool {
+        if let Some(segentry) = self.segs.get(&seg) {
+            segentry.newer_image_layer_exists(lsn, disk_consistent_lsn)
+        } else {
+            false
         }
     }
 
-    pub fn iter_historic_layers(&self) -> impl Iterator<Item = &Arc<dyn Layer>> {
-        self.historic_layers.iter()
-    }
-
-    /// Find the last image layer that covers 'key', ignoring any image layers
-    /// newer than 'lsn'.
-    fn find_latest_image(&self, key: Key, lsn: Lsn) -> Option<Arc<dyn Layer>> {
-        let mut candidate_lsn = Lsn(0);
-        let mut candidate = None;
-        for l in self.historic_layers.iter() {
-            if l.is_incremental() {
-                continue;
-            }
-
-            if !l.get_key_range().contains(&key) {
-                continue;
-            }
-
-            let this_lsn = l.get_lsn_range().start;
-            if this_lsn > lsn {
-                continue;
-            }
-            if this_lsn < candidate_lsn {
-                // our previous candidate was better
-                continue;
-            }
-            candidate_lsn = this_lsn;
-            candidate = Some(Arc::clone(l));
-        }
-
-        candidate
-    }
-
+    /// Is there any layer for given segment that is alive at the lsn?
     ///
-    /// Divide the whole given range of keys into sub-ranges based on the latest
-    /// image layer that covers each range. (This is used when creating  new
-    /// image layers)
-    ///
-    // FIXME: clippy complains that the result type is very complex. She's probably
-    // right...
-    #[allow(clippy::type_complexity)]
-    pub fn image_coverage(
-        &self,
-        key_range: &Range<Key>,
-        lsn: Lsn,
-    ) -> Result<Vec<(Range<Key>, Option<Arc<dyn Layer>>)>> {
-        let mut points = vec![key_range.start];
-        for l in self.historic_layers.iter() {
-            if l.get_lsn_range().start > lsn {
-                continue;
-            }
-            let range = l.get_key_range();
-            if key_range.contains(&range.start) {
-                points.push(l.get_key_range().start);
-            }
-            if key_range.contains(&range.end) {
-                points.push(l.get_key_range().end);
-            }
-        }
-        points.push(key_range.end);
-
-        points.sort();
-        points.dedup();
-
-        // Ok, we now have a list of "interesting" points in the key space
-
-        // For each range between the points, find the latest image
-        let mut start = *points.first().unwrap();
-        let mut ranges = Vec::new();
-        for end in points[1..].iter() {
-            let img = self.find_latest_image(start, lsn);
-
-            ranges.push((start..*end, img));
-
-            start = *end;
-        }
-        Ok(ranges)
+    /// This is a public wrapper for SegEntry fucntion,
+    /// used for garbage collection, to determine if some alive layer
+    /// exists at the lsn. If so, we shouldn't delete a newer dropped layer
+    /// to avoid incorrectly making it visible.
+    pub fn layer_exists_at_lsn(&self, seg: SegmentTag, lsn: Lsn) -> Result<bool> {
+        Ok(if let Some(segentry) = self.segs.get(&seg) {
+            segentry.exists_at_lsn(lsn)?.unwrap_or(false)
+        } else {
+            false
+        })
     }
 
-    /// Count how many L1 delta layers there are that overlap with the
-    /// given key and LSN range.
-    pub fn count_deltas(&self, key_range: &Range<Key>, lsn_range: &Range<Lsn>) -> Result<usize> {
-        let mut result = 0;
-        for l in self.historic_layers.iter() {
-            if !l.is_incremental() {
-                continue;
-            }
-            if !range_overlaps(&l.get_lsn_range(), lsn_range) {
-                continue;
-            }
-            if !range_overlaps(&l.get_key_range(), key_range) {
-                continue;
-            }
+    /// Return the oldest in-memory layer, along with its generation number.
+    pub fn peek_oldest_open(&mut self) -> Option<(LayerId, Arc<InMemoryLayer>, u64)> {
+        let global_map = GLOBAL_LAYER_MAP.read().unwrap();
 
-            // We ignore level0 delta layers. Unless the whole keyspace fits
-            // into one partition
-            if !range_eq(key_range, &(Key::MIN..Key::MAX))
-                && range_eq(&l.get_key_range(), &(Key::MIN..Key::MAX))
-            {
-                continue;
+        while let Some(oldest_entry) = self.open_layers.peek() {
+            if let Some(layer) = global_map.get(&oldest_entry.layer_id) {
+                return Some((oldest_entry.layer_id, layer, oldest_entry.generation));
+            } else {
+                self.open_layers.pop();
             }
-
-            result += 1;
         }
-        Ok(result)
+        None
     }
 
-    /// Return all L0 delta layers
-    pub fn get_level0_deltas(&self) -> Result<Vec<Arc<dyn Layer>>> {
-        let mut deltas = Vec::new();
-        for l in self.historic_layers.iter() {
-            if !l.is_incremental() {
-                continue;
-            }
-            if l.get_key_range() != (Key::MIN..Key::MAX) {
-                continue;
-            }
-            deltas.push(Arc::clone(l));
+    /// Increment the generation number used to stamp open in-memory layers. Layers
+    /// added with `insert_open` after this call will be associated with the new
+    /// generation. Returns the new generation number.
+    pub fn increment_generation(&mut self) -> u64 {
+        self.current_generation += 1;
+        self.current_generation
+    }
+
+    pub fn iter_historic_layers(&self) -> HistoricLayerIter {
+        HistoricLayerIter {
+            seg_iter: self.segs.iter(),
+            iter: None,
         }
-        Ok(deltas)
     }
 
     /// debugging function to print out the contents of the layer map
     #[allow(unused)]
-    pub fn dump(&self, verbose: bool) -> Result<()> {
+    pub fn dump(&self) -> Result<()> {
         println!("Begin dump LayerMap");
+        for (seg, segentry) in self.segs.iter() {
+            if let Some(open) = &segentry.open_layer_id {
+                if let Some(layer) = GLOBAL_LAYER_MAP.read().unwrap().get(open) {
+                    layer.dump()?;
+                } else {
+                    println!("layer not found in global map");
+                }
+            }
 
-        println!("open_layer:");
-        if let Some(open_layer) = &self.open_layer {
-            open_layer.dump(verbose)?;
-        }
-
-        println!("frozen_layers:");
-        for frozen_layer in self.frozen_layers.iter() {
-            frozen_layer.dump(verbose)?;
-        }
-
-        println!("historic_layers:");
-        for layer in self.historic_layers.iter() {
-            layer.dump(verbose)?;
+            for layer in segentry.historic.iter() {
+                layer.dump()?;
+            }
         }
         println!("End dump LayerMap");
         Ok(())
     }
 }
+
+impl IntervalItem for dyn Layer {
+    type Key = Lsn;
+
+    fn start_key(&self) -> Lsn {
+        self.get_start_lsn()
+    }
+    fn end_key(&self) -> Lsn {
+        self.get_end_lsn()
+    }
+}
+
+///
+/// Per-segment entry in the LayerMap::segs hash map. Holds all the layers
+/// associated with the segment.
+///
+/// The last layer that is open for writes is always an InMemoryLayer,
+/// and is kept in a separate field, because there can be only one for
+/// each segment. The older layers, stored on disk, are kept in an
+/// IntervalTree.
+#[derive(Default)]
+struct SegEntry {
+    open_layer_id: Option<LayerId>,
+    historic: IntervalTree<dyn Layer>,
+}
+
+impl SegEntry {
+    /// Does the segment exist at given LSN?
+    /// Return None if object is not found in this SegEntry.
+    fn exists_at_lsn(&self, lsn: Lsn) -> Result<Option<bool>> {
+        if let Some(layer) = self.get(lsn) {
+            Ok(Some(layer.get_seg_exists(lsn)?))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub fn get(&self, lsn: Lsn) -> Option<Arc<dyn Layer>> {
+        if let Some(open_layer_id) = &self.open_layer_id {
+            let open_layer = GLOBAL_LAYER_MAP.read().unwrap().get(open_layer_id)?;
+            if open_layer.get_start_lsn() <= lsn {
+                return Some(open_layer);
+            }
+        }
+
+        self.historic.search(lsn)
+    }
+
+    pub fn newer_image_layer_exists(&self, lsn: Lsn, disk_consistent_lsn: Lsn) -> bool {
+        // We only check on-disk layers, because
+        // in-memory layers are not durable
+
+        // The end-LSN is exclusive, while disk_consistent_lsn is
+        // inclusive. For example, if disk_consistent_lsn is 100, it is
+        // OK for a delta layer to have end LSN 101, but if the end LSN
+        // is 102, then it might not have been fully flushed to disk
+        // before crash.
+        self.historic
+            .iter_newer(lsn)
+            .any(|layer| !layer.is_incremental() && layer.get_end_lsn() <= disk_consistent_lsn + 1)
+    }
+
+    // Set new open layer for a SegEntry.
+    // It's ok to rewrite previous open layer,
+    // but only if it is not writeable anymore.
+    pub fn update_open(&mut self, layer: Arc<InMemoryLayer>) -> LayerId {
+        if let Some(prev_open_layer_id) = &self.open_layer_id {
+            if let Some(prev_open_layer) = GLOBAL_LAYER_MAP.read().unwrap().get(prev_open_layer_id)
+            {
+                assert!(!prev_open_layer.is_writeable());
+            }
+        }
+        let open_layer_id = GLOBAL_LAYER_MAP.write().unwrap().insert(layer);
+        self.open_layer_id = Some(open_layer_id);
+        open_layer_id
+    }
+
+    pub fn insert_historic(&mut self, layer: Arc<dyn Layer>) {
+        self.historic.insert(layer);
+    }
+}
+
+/// Entry held in LayerMap::open_layers, with boilerplate comparison routines
+/// to implement a min-heap ordered by 'oldest_lsn' and 'generation'
+///
+/// The generation number associated with each entry can be used to distinguish
+/// recently-added entries (i.e after last call to increment_generation()) from older
+/// entries with the same 'oldest_lsn'.
+struct OpenLayerEntry {
+    oldest_lsn: Lsn, // copy of layer.get_oldest_lsn()
+    generation: u64,
+    layer_id: LayerId,
+}
+impl Ord for OpenLayerEntry {
+    fn cmp(&self, other: &Self) -> Ordering {
+        // BinaryHeap is a max-heap, and we want a min-heap. Reverse the ordering here
+        // to get that. Entries with identical oldest_lsn are ordered by generation
+        other
+            .oldest_lsn
+            .cmp(&self.oldest_lsn)
+            .then_with(|| other.generation.cmp(&self.generation))
+    }
+}
+impl PartialOrd for OpenLayerEntry {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+impl PartialEq for OpenLayerEntry {
+    fn eq(&self, other: &Self) -> bool {
+        self.cmp(other) == Ordering::Equal
+    }
+}
+impl Eq for OpenLayerEntry {}
+
+/// Iterator returned by LayerMap::iter_historic_layers()
+pub struct HistoricLayerIter<'a> {
+    seg_iter: std::collections::hash_map::Iter<'a, SegmentTag, SegEntry>,
+    iter: Option<IntervalIter<'a, dyn Layer>>,
+}
+
+impl<'a> Iterator for HistoricLayerIter<'a> {
+    type Item = Arc<dyn Layer>;
+
+    fn next(&mut self) -> std::option::Option<<Self as std::iter::Iterator>::Item> {
+        loop {
+            if let Some(x) = &mut self.iter {
+                if let Some(x) = x.next() {
+                    return Some(Arc::clone(&x));
+                }
+            }
+            if let Some((_tag, segentry)) = self.seg_iter.next() {
+                self.iter = Some(segentry.historic.iter());
+                continue;
+            } else {
+                return None;
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::config::PageServerConf;
+    use std::str::FromStr;
+    use zenith_utils::zid::{ZTenantId, ZTimelineId};
+
+    /// Arbitrary relation tag, for testing.
+    const TESTREL_A: RelishTag = RelishTag::Relation(RelTag {
+        spcnode: 0,
+        dbnode: 111,
+        relnode: 1000,
+        forknum: 0,
+    });
+
+    lazy_static! {
+        static ref DUMMY_TIMELINEID: ZTimelineId =
+            ZTimelineId::from_str("00000000000000000000000000000000").unwrap();
+        static ref DUMMY_TENANTID: ZTenantId =
+            ZTenantId::from_str("00000000000000000000000000000000").unwrap();
+    }
+
+    /// Construct a dummy InMemoryLayer for testing
+    fn dummy_inmem_layer(
+        conf: &'static PageServerConf,
+        segno: u32,
+        start_lsn: Lsn,
+        oldest_lsn: Lsn,
+    ) -> Arc<InMemoryLayer> {
+        Arc::new(
+            InMemoryLayer::create(
+                conf,
+                *DUMMY_TIMELINEID,
+                *DUMMY_TENANTID,
+                SegmentTag {
+                    rel: TESTREL_A,
+                    segno,
+                },
+                start_lsn,
+                oldest_lsn,
+            )
+            .unwrap(),
+        )
+    }
+
+    #[test]
+    fn test_open_layers() -> Result<()> {
+        let conf = PageServerConf::dummy_conf(PageServerConf::test_repo_dir("dummy_inmem_layer"));
+        let conf = Box::leak(Box::new(conf));
+        std::fs::create_dir_all(conf.timeline_path(&DUMMY_TIMELINEID, &DUMMY_TENANTID))?;
+
+        let mut layers = LayerMap::default();
+
+        let gen1 = layers.increment_generation();
+        layers.insert_open(dummy_inmem_layer(conf, 0, Lsn(0x100), Lsn(0x100)));
+        layers.insert_open(dummy_inmem_layer(conf, 1, Lsn(0x100), Lsn(0x200)));
+        layers.insert_open(dummy_inmem_layer(conf, 2, Lsn(0x100), Lsn(0x120)));
+        layers.insert_open(dummy_inmem_layer(conf, 3, Lsn(0x100), Lsn(0x110)));
+
+        let gen2 = layers.increment_generation();
+        layers.insert_open(dummy_inmem_layer(conf, 4, Lsn(0x100), Lsn(0x110)));
+        layers.insert_open(dummy_inmem_layer(conf, 5, Lsn(0x100), Lsn(0x100)));
+
+        // A helper function (closure) to pop the next oldest open entry from the layer map,
+        // and assert that it is what we'd expect
+        let mut assert_pop_layer = |expected_segno: u32, expected_generation: u64| {
+            let (layer_id, l, generation) = layers.peek_oldest_open().unwrap();
+            assert!(l.get_seg_tag().segno == expected_segno);
+            assert!(generation == expected_generation);
+            layers.remove_open(layer_id);
+        };
+
+        assert_pop_layer(0, gen1); // 0x100
+        assert_pop_layer(5, gen2); // 0x100
+        assert_pop_layer(3, gen1); // 0x110
+        assert_pop_layer(4, gen2); // 0x110
+        assert_pop_layer(2, gen1); // 0x120
+        assert_pop_layer(1, gen1); // 0x200
+
+        Ok(())
+    }
+}

pageserver/src/layered_repository/metadata.rs

@@ -6,24 +6,21 @@
 //!
 //! The module contains all structs and related helper methods related to timeline metadata.
 
-use std::path::PathBuf;
+use std::{convert::TryInto, path::PathBuf};
 
 use anyhow::ensure;
-use serde::{Deserialize, Serialize};
-use utils::{
+use zenith_utils::{
     bin_ser::BeSer,
     lsn::Lsn,
     zid::{ZTenantId, ZTimelineId},
 };
 
 use crate::config::PageServerConf;
-use crate::STORAGE_FORMAT_VERSION;
 
-/// We assume that a write of up to METADATA_MAX_SIZE bytes is atomic.
-///
-/// This is the same assumption that PostgreSQL makes with the control file,
-/// see PG_CONTROL_MAX_SAFE_SIZE
-const METADATA_MAX_SIZE: usize = 512;
+// Taken from PG_CONTROL_MAX_SAFE_SIZE
+const METADATA_MAX_SAFE_SIZE: usize = 512;
+const METADATA_CHECKSUM_SIZE: usize = std::mem::size_of::<u32>();
+const METADATA_MAX_DATA_SIZE: usize = METADATA_MAX_SAFE_SIZE - METADATA_CHECKSUM_SIZE;
 
 /// The name of the metadata file pageserver creates per timeline.
 pub const METADATA_FILE_NAME: &str = "metadata";
@@ -31,22 +28,8 @@ pub const METADATA_FILE_NAME: &str = "metadata";
 /// Metadata stored on disk for each timeline
 ///
 /// The fields correspond to the values we hold in memory, in LayeredTimeline.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
 pub struct TimelineMetadata {
-    hdr: TimelineMetadataHeader,
-    body: TimelineMetadataBody,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-struct TimelineMetadataHeader {
-    checksum: u32,       // CRC of serialized metadata body
-    size: u16,           // size of serialized metadata
-    format_version: u16, // storage format version (used for compatibility checks)
-}
-const METADATA_HDR_SIZE: usize = std::mem::size_of::<TimelineMetadataHeader>();
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-struct TimelineMetadataBody {
     disk_consistent_lsn: Lsn,
     // This is only set if we know it. We track it in memory when the page
     // server is running, but we only track the value corresponding to
@@ -86,90 +69,130 @@ impl TimelineMetadata {
         initdb_lsn: Lsn,
     ) -> Self {
         Self {
-            hdr: TimelineMetadataHeader {
-                checksum: 0,
-                size: 0,
-                format_version: STORAGE_FORMAT_VERSION,
-            },
-            body: TimelineMetadataBody {
-                disk_consistent_lsn,
-                prev_record_lsn,
-                ancestor_timeline,
-                ancestor_lsn,
-                latest_gc_cutoff_lsn,
-                initdb_lsn,
-            },
+            disk_consistent_lsn,
+            prev_record_lsn,
+            ancestor_timeline,
+            ancestor_lsn,
+            latest_gc_cutoff_lsn,
+            initdb_lsn,
         }
     }
 
     pub fn from_bytes(metadata_bytes: &[u8]) -> anyhow::Result<Self> {
         ensure!(
-            metadata_bytes.len() == METADATA_MAX_SIZE,
+            metadata_bytes.len() == METADATA_MAX_SAFE_SIZE,
             "metadata bytes size is wrong"
         );
-        let hdr = TimelineMetadataHeader::des(&metadata_bytes[0..METADATA_HDR_SIZE])?;
+
+        let data = &metadata_bytes[..METADATA_MAX_DATA_SIZE];
+        let calculated_checksum = crc32c::crc32c(data);
+
+        let checksum_bytes: &[u8; METADATA_CHECKSUM_SIZE] =
+            metadata_bytes[METADATA_MAX_DATA_SIZE..].try_into()?;
+        let expected_checksum = u32::from_le_bytes(*checksum_bytes);
         ensure!(
-            hdr.format_version == STORAGE_FORMAT_VERSION,
-            "format version mismatch"
-        );
-        let metadata_size = hdr.size as usize;
-        ensure!(
-            metadata_size <= METADATA_MAX_SIZE,
-            "corrupted metadata file"
-        );
-        let calculated_checksum = crc32c::crc32c(&metadata_bytes[METADATA_HDR_SIZE..metadata_size]);
-        ensure!(
-            hdr.checksum == calculated_checksum,
+            calculated_checksum == expected_checksum,
             "metadata checksum mismatch"
         );
-        let body = TimelineMetadataBody::des(&metadata_bytes[METADATA_HDR_SIZE..metadata_size])?;
-        ensure!(
-            body.disk_consistent_lsn.is_aligned(),
-            "disk_consistent_lsn is not aligned"
-        );
 
-        Ok(TimelineMetadata { hdr, body })
+        let data = TimelineMetadata::from(serialize::DeTimelineMetadata::des_prefix(data)?);
+        assert!(data.disk_consistent_lsn.is_aligned());
+
+        Ok(data)
     }
 
     pub fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
-        let body_bytes = self.body.ser()?;
-        let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
-        let hdr = TimelineMetadataHeader {
-            size: metadata_size as u16,
-            format_version: STORAGE_FORMAT_VERSION,
-            checksum: crc32c::crc32c(&body_bytes),
-        };
-        let hdr_bytes = hdr.ser()?;
-        let mut metadata_bytes = vec![0u8; METADATA_MAX_SIZE];
-        metadata_bytes[0..METADATA_HDR_SIZE].copy_from_slice(&hdr_bytes);
-        metadata_bytes[METADATA_HDR_SIZE..metadata_size].copy_from_slice(&body_bytes);
+        let serializeable_metadata = serialize::SeTimelineMetadata::from(self);
+        let mut metadata_bytes = serialize::SeTimelineMetadata::ser(&serializeable_metadata)?;
+        assert!(metadata_bytes.len() <= METADATA_MAX_DATA_SIZE);
+        metadata_bytes.resize(METADATA_MAX_SAFE_SIZE, 0u8);
+
+        let checksum = crc32c::crc32c(&metadata_bytes[..METADATA_MAX_DATA_SIZE]);
+        metadata_bytes[METADATA_MAX_DATA_SIZE..].copy_from_slice(&u32::to_le_bytes(checksum));
         Ok(metadata_bytes)
     }
 
     /// [`Lsn`] that corresponds to the corresponding timeline directory
     /// contents, stored locally in the pageserver workdir.
     pub fn disk_consistent_lsn(&self) -> Lsn {
-        self.body.disk_consistent_lsn
+        self.disk_consistent_lsn
     }
 
     pub fn prev_record_lsn(&self) -> Option<Lsn> {
-        self.body.prev_record_lsn
+        self.prev_record_lsn
     }
 
     pub fn ancestor_timeline(&self) -> Option<ZTimelineId> {
-        self.body.ancestor_timeline
+        self.ancestor_timeline
     }
 
     pub fn ancestor_lsn(&self) -> Lsn {
-        self.body.ancestor_lsn
+        self.ancestor_lsn
     }
 
     pub fn latest_gc_cutoff_lsn(&self) -> Lsn {
-        self.body.latest_gc_cutoff_lsn
+        self.latest_gc_cutoff_lsn
     }
 
     pub fn initdb_lsn(&self) -> Lsn {
-        self.body.initdb_lsn
+        self.initdb_lsn
+    }
+}
+
+/// This module is for direct conversion of metadata to bytes and back.
+/// For a certain metadata, besides the conversion a few verification steps has to
+/// be done, so all serde derives are hidden from the user, to avoid accidental
+/// verification-less metadata creation.
+mod serialize {
+    use serde::{Deserialize, Serialize};
+    use zenith_utils::{lsn::Lsn, zid::ZTimelineId};
+
+    use super::TimelineMetadata;
+
+    #[derive(Serialize)]
+    pub(super) struct SeTimelineMetadata<'a> {
+        disk_consistent_lsn: &'a Lsn,
+        prev_record_lsn: &'a Option<Lsn>,
+        ancestor_timeline: &'a Option<ZTimelineId>,
+        ancestor_lsn: &'a Lsn,
+        latest_gc_cutoff_lsn: &'a Lsn,
+        initdb_lsn: &'a Lsn,
+    }
+
+    impl<'a> From<&'a TimelineMetadata> for SeTimelineMetadata<'a> {
+        fn from(other: &'a TimelineMetadata) -> Self {
+            Self {
+                disk_consistent_lsn: &other.disk_consistent_lsn,
+                prev_record_lsn: &other.prev_record_lsn,
+                ancestor_timeline: &other.ancestor_timeline,
+                ancestor_lsn: &other.ancestor_lsn,
+                latest_gc_cutoff_lsn: &other.latest_gc_cutoff_lsn,
+                initdb_lsn: &other.initdb_lsn,
+            }
+        }
+    }
+
+    #[derive(Deserialize)]
+    pub(super) struct DeTimelineMetadata {
+        disk_consistent_lsn: Lsn,
+        prev_record_lsn: Option<Lsn>,
+        ancestor_timeline: Option<ZTimelineId>,
+        ancestor_lsn: Lsn,
+        latest_gc_cutoff_lsn: Lsn,
+        initdb_lsn: Lsn,
+    }
+
+    impl From<DeTimelineMetadata> for TimelineMetadata {
+        fn from(other: DeTimelineMetadata) -> Self {
+            Self {
+                disk_consistent_lsn: other.disk_consistent_lsn,
+                prev_record_lsn: other.prev_record_lsn,
+                ancestor_timeline: other.ancestor_timeline,
+                ancestor_lsn: other.ancestor_lsn,
+                latest_gc_cutoff_lsn: other.latest_gc_cutoff_lsn,
+                initdb_lsn: other.initdb_lsn,
+            }
+        }
     }
 }
 
@@ -181,14 +204,14 @@ mod tests {
 
     #[test]
     fn metadata_serializes_correctly() {
-        let original_metadata = TimelineMetadata::new(
-            Lsn(0x200),
-            Some(Lsn(0x100)),
-            Some(TIMELINE_ID),
-            Lsn(0),
-            Lsn(0),
-            Lsn(0),
-        );
+        let original_metadata = TimelineMetadata {
+            disk_consistent_lsn: Lsn(0x200),
+            prev_record_lsn: Some(Lsn(0x100)),
+            ancestor_timeline: Some(TIMELINE_ID),
+            ancestor_lsn: Lsn(0),
+            latest_gc_cutoff_lsn: Lsn(0),
+            initdb_lsn: Lsn(0),
+        };
 
         let metadata_bytes = original_metadata
             .to_bytes()
@@ -198,7 +221,7 @@ mod tests {
             .expect("Should deserialize its own bytes");
 
         assert_eq!(
-            deserialized_metadata.body, original_metadata.body,
+            deserialized_metadata, original_metadata,
             "Metadata that was serialized to bytes and deserialized back should not change"
         );
     }

pageserver/src/layered_repository/storage_layer.rs

@@ -2,112 +2,145 @@
 //! Common traits and structs for layers
 //!
 
-use crate::repository::{Key, Value};
-use crate::walrecord::ZenithWalRecord;
+use crate::relish::RelishTag;
+use crate::repository::{BlockNumber, ZenithWalRecord};
+use crate::{ZTenantId, ZTimelineId};
 use anyhow::Result;
 use bytes::Bytes;
-use std::ops::Range;
+use serde::{Deserialize, Serialize};
+use std::fmt;
 use std::path::PathBuf;
 
-use utils::{
-    lsn::Lsn,
-    zid::{ZTenantId, ZTimelineId},
-};
+use zenith_utils::lsn::Lsn;
 
-pub fn range_overlaps<T>(a: &Range<T>, b: &Range<T>) -> bool
-where
-    T: PartialOrd<T>,
-{
-    if a.start < b.start {
-        a.end > b.start
-    } else {
-        b.end > a.start
+// Size of one segment in pages (10 MB)
+pub const RELISH_SEG_SIZE: u32 = 10 * 1024 * 1024 / 8192;
+
+///
+/// Each relish stored in the repository is divided into fixed-sized "segments",
+/// with 10 MB of key-space, or 1280 8k pages each.
+///
+#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy, Serialize, Deserialize)]
+pub struct SegmentTag {
+    pub rel: RelishTag,
+    pub segno: u32,
+}
+
+/// SegmentBlk represents a block number within a segment, or the size of segment.
+///
+/// This is separate from BlockNumber, which is used for block number within the
+/// whole relish. Since this is just a type alias, the compiler will let you mix
+/// them freely, but we use the type alias as documentation to make it clear
+/// which one we're dealing with.
+///
+/// (We could turn this into "struct SegmentBlk(u32)" to forbid accidentally
+/// assigning a BlockNumber to SegmentBlk or vice versa, but that makes
+/// operations more verbose).
+pub type SegmentBlk = u32;
+
+impl fmt::Display for SegmentTag {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}.{}", self.rel, self.segno)
     }
 }
 
-pub fn range_eq<T>(a: &Range<T>, b: &Range<T>) -> bool
-where
-    T: PartialEq<T>,
-{
-    a.start == b.start && a.end == b.end
+impl SegmentTag {
+    /// Given a relish and block number, calculate the corresponding segment and
+    /// block number within the segment.
+    pub const fn from_blknum(rel: RelishTag, blknum: BlockNumber) -> (SegmentTag, SegmentBlk) {
+        (
+            SegmentTag {
+                rel,
+                segno: blknum / RELISH_SEG_SIZE,
+            },
+            blknum % RELISH_SEG_SIZE,
+        )
+    }
 }
 
-/// Struct used to communicate across calls to 'get_value_reconstruct_data'.
 ///
-/// Before first call, you can fill in 'page_img' if you have an older cached
-/// version of the page available. That can save work in
-/// 'get_value_reconstruct_data', as it can stop searching for page versions
-/// when all the WAL records going back to the cached image have been collected.
+/// Represents a version of a page at a specific LSN. The LSN is the key of the
+/// entry in the 'page_versions' hash, it is not duplicated here.
 ///
-/// When get_value_reconstruct_data returns Complete, 'img' is set to an image
-/// of the page, or the oldest WAL record in 'records' is a will_init-type
+/// A page version can be stored as a full page image, or as WAL record that needs
+/// to be applied over the previous page version to reconstruct this version.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum PageVersion {
+    Page(Bytes),
+    Wal(ZenithWalRecord),
+}
+
+///
+/// Struct used to communicate across calls to 'get_page_reconstruct_data'.
+///
+/// Before first call to get_page_reconstruct_data, you can fill in 'page_img'
+/// if you have an older cached version of the page available. That can save
+/// work in 'get_page_reconstruct_data', as it can stop searching for page
+/// versions when all the WAL records going back to the cached image have been
+/// collected.
+///
+/// When get_page_reconstruct_data returns Complete, 'page_img' is set to an
+/// image of the page, or the oldest WAL record in 'records' is a will_init-type
 /// record that initializes the page without requiring a previous image.
 ///
 /// If 'get_page_reconstruct_data' returns Continue, some 'records' may have
 /// been collected, but there are more records outside the current layer. Pass
-/// the same ValueReconstructState struct in the next 'get_value_reconstruct_data'
+/// the same PageReconstructData struct in the next 'get_page_reconstruct_data'
 /// call, to collect more records.
 ///
-#[derive(Debug)]
-pub struct ValueReconstructState {
+pub struct PageReconstructData {
     pub records: Vec<(Lsn, ZenithWalRecord)>,
-    pub img: Option<(Lsn, Bytes)>,
+    pub page_img: Option<(Lsn, Bytes)>,
 }
 
 /// Return value from Layer::get_page_reconstruct_data
-#[derive(Clone, Copy, Debug)]
-pub enum ValueReconstructResult {
+pub enum PageReconstructResult {
     /// Got all the data needed to reconstruct the requested page
     Complete,
     /// This layer didn't contain all the required data, the caller should look up
     /// the predecessor layer at the returned LSN and collect more data from there.
-    Continue,
-
+    Continue(Lsn),
     /// This layer didn't contain data needed to reconstruct the page version at
     /// the returned LSN. This is usually considered an error, but might be OK
     /// in some circumstances.
-    Missing,
+    Missing(Lsn),
 }
 
-/// A Layer contains all data in a "rectangle" consisting of a range of keys and
-/// range of LSNs.
 ///
+/// A Layer corresponds to one RELISH_SEG_SIZE slice of a relish in a range of LSNs.
 /// There are two kinds of layers, in-memory and on-disk layers. In-memory
-/// layers are used to ingest incoming WAL, and provide fast access to the
-/// recent page versions. On-disk layers are stored as files on disk, and are
-/// immutable. This trait presents the common functionality of in-memory and
-/// on-disk layers.
-///
-/// Furthermore, there are two kinds of on-disk layers: delta and image layers.
-/// A delta layer contains all modifications within a range of LSNs and keys.
-/// An image layer is a snapshot of all the data in a key-range, at a single
-/// LSN
+/// layers are used to ingest incoming WAL, and provide fast access
+/// to the recent page versions. On-disk layers are stored as files on disk, and
+/// are immutable. This trait presents the common functionality of
+/// in-memory and on-disk layers.
 ///
 pub trait Layer: Send + Sync {
     fn get_tenant_id(&self) -> ZTenantId;
 
-    /// Identify the timeline this layer belongs to
+    /// Identify the timeline this relish belongs to
     fn get_timeline_id(&self) -> ZTimelineId;
 
-    /// Range of keys that this layer covers
-    fn get_key_range(&self) -> Range<Key>;
+    /// Identify the relish segment
+    fn get_seg_tag(&self) -> SegmentTag;
 
     /// Inclusive start bound of the LSN range that this layer holds
+    fn get_start_lsn(&self) -> Lsn;
+
     /// Exclusive end bound of the LSN range that this layer holds.
     ///
     /// - For an open in-memory layer, this is MAX_LSN.
     /// - For a frozen in-memory layer or a delta layer, this is a valid end bound.
     /// - An image layer represents snapshot at one LSN, so end_lsn is always the snapshot LSN + 1
-    fn get_lsn_range(&self) -> Range<Lsn>;
+    fn get_end_lsn(&self) -> Lsn;
+
+    /// Is the segment represented by this layer dropped by PostgreSQL?
+    fn is_dropped(&self) -> bool;
 
     /// Filename used to store this layer on disk. (Even in-memory layers
     /// implement this, to print a handy unique identifier for the layer for
     /// log messages, even though they're never not on disk.)
     fn filename(&self) -> PathBuf;
 
-    /// If a layer has a corresponding file on a local filesystem, return its absolute path.
-    fn local_path(&self) -> Option<PathBuf>;
-
     ///
     /// Return data needed to reconstruct given page at LSN.
     ///
@@ -120,14 +153,20 @@ pub trait Layer: Send + Sync {
     /// is available. If this returns PageReconstructResult::Continue, look up
     /// the predecessor layer and call again with the same 'reconstruct_data' to
     /// collect more data.
-    fn get_value_reconstruct_data(
+    fn get_page_reconstruct_data(
         &self,
-        key: Key,
-        lsn_range: Range<Lsn>,
-        reconstruct_data: &mut ValueReconstructState,
-    ) -> Result<ValueReconstructResult>;
+        blknum: SegmentBlk,
+        lsn: Lsn,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<PageReconstructResult>;
 
-    /// Does this layer only contain some data for the key-range (incremental),
+    /// Return size of the segment at given LSN. (Only for blocky relations.)
+    fn get_seg_size(&self, lsn: Lsn) -> Result<SegmentBlk>;
+
+    /// Does the segment exist at given LSN? Or was it dropped before it.
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool>;
+
+    /// Does this layer only contain some data for the segment (incremental),
     /// or does it contain a version of every page? This is important to know
     /// for garbage collecting old layers: an incremental layer depends on
     /// the previous non-incremental layer.
@@ -136,12 +175,13 @@ pub trait Layer: Send + Sync {
     /// Returns true for layers that are represented in memory.
     fn is_in_memory(&self) -> bool;
 
-    /// Iterate through all keys and values stored in the layer
-    fn iter(&self) -> Box<dyn Iterator<Item = Result<(Key, Lsn, Value)>> + '_>;
+    /// Release memory used by this layer. There is no corresponding 'load'
+    /// function, that's done implicitly when you call one of the get-functions.
+    fn unload(&self) -> Result<()>;
 
     /// Permanently remove this layer from disk.
     fn delete(&self) -> Result<()>;
 
     /// Dump summary of the contents of the layer to stdout
-    fn dump(&self, verbose: bool) -> Result<()>;
+    fn dump(&self) -> Result<()>;
 }

pageserver/src/lib.rs

@@ -2,16 +2,12 @@ pub mod basebackup;
 pub mod config;
 pub mod http;
 pub mod import_datadir;
-pub mod keyspace;
 pub mod layered_repository;
 pub mod page_cache;
 pub mod page_service;
-pub mod pgdatadir_mapping;
-pub mod profiling;
-pub mod reltag;
+pub mod relish;
+pub mod remote_storage;
 pub mod repository;
-pub mod storage_sync;
-pub mod tenant_config;
 pub mod tenant_mgr;
 pub mod tenant_threads;
 pub mod thread_mgr;
@@ -23,29 +19,12 @@ pub mod walrecord;
 pub mod walredo;
 
 use lazy_static::lazy_static;
-use tracing::info;
-use utils::postgres_backend;
-
-use crate::thread_mgr::ThreadKind;
-use metrics::{register_int_gauge_vec, IntGaugeVec};
-
-use layered_repository::LayeredRepository;
-use pgdatadir_mapping::DatadirTimeline;
-
-/// Current storage format version
-///
-/// This is embedded in the metadata file, and also in the header of all the
-/// layer files. If you make any backwards-incompatible changes to the storage
-/// format, bump this!
-pub const STORAGE_FORMAT_VERSION: u16 = 3;
-
-// Magic constants used to identify different kinds of files
-pub const IMAGE_FILE_MAGIC: u16 = 0x5A60;
-pub const DELTA_FILE_MAGIC: u16 = 0x5A61;
+use zenith_metrics::{register_int_gauge_vec, IntGaugeVec};
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
 lazy_static! {
     static ref LIVE_CONNECTIONS_COUNT: IntGaugeVec = register_int_gauge_vec!(
-        "pageserver_live_connections",
+        "pageserver_live_connections_count",
         "Number of live network connections",
         &["pageserver_connection_kind"]
     )
@@ -57,42 +36,10 @@ pub const LOG_FILE_NAME: &str = "pageserver.log";
 /// Config for the Repository checkpointer
 #[derive(Debug, Clone, Copy)]
 pub enum CheckpointConfig {
+    // Flush in-memory data that is older than this
+    Distance(u64),
     // Flush all in-memory data
     Flush,
     // Flush all in-memory data and reconstruct all page images
     Forced,
 }
-
-pub type RepositoryImpl = LayeredRepository;
-
-pub type DatadirTimelineImpl = DatadirTimeline<RepositoryImpl>;
-
-pub fn shutdown_pageserver(exit_code: i32) {
-    // Shut down the libpq endpoint thread. This prevents new connections from
-    // being accepted.
-    thread_mgr::shutdown_threads(Some(ThreadKind::LibpqEndpointListener), None, None);
-
-    // Shut down any page service threads.
-    postgres_backend::set_pgbackend_shutdown_requested();
-    thread_mgr::shutdown_threads(Some(ThreadKind::PageRequestHandler), None, None);
-
-    // Shut down all the tenants. This flushes everything to disk and kills
-    // the checkpoint and GC threads.
-    tenant_mgr::shutdown_all_tenants();
-
-    // Stop syncing with remote storage.
-    //
-    // FIXME: Does this wait for the sync thread to finish syncing what's queued up?
-    // Should it?
-    thread_mgr::shutdown_threads(Some(ThreadKind::StorageSync), None, None);
-
-    // Shut down the HTTP endpoint last, so that you can still check the server's
-    // status while it's shutting down.
-    thread_mgr::shutdown_threads(Some(ThreadKind::HttpEndpointListener), None, None);
-
-    // There should be nothing left, but let's be sure
-    thread_mgr::shutdown_threads(None, None, None);
-
-    info!("Shut down successfully completed");
-    std::process::exit(exit_code);
-}

pageserver/src/page_cache.rs

@@ -41,28 +41,31 @@ use std::{
     convert::TryInto,
     sync::{
         atomic::{AtomicU8, AtomicUsize, Ordering},
-        RwLock, RwLockReadGuard, RwLockWriteGuard, TryLockError,
+        RwLock, RwLockReadGuard, RwLockWriteGuard,
     },
 };
 
 use once_cell::sync::OnceCell;
 use tracing::error;
-use utils::{
+use zenith_utils::{
     lsn::Lsn,
     zid::{ZTenantId, ZTimelineId},
 };
 
 use crate::layered_repository::writeback_ephemeral_file;
-use crate::repository::Key;
+use crate::{config::PageServerConf, relish::RelTag};
 
 static PAGE_CACHE: OnceCell<PageCache> = OnceCell::new();
-const TEST_PAGE_CACHE_SIZE: usize = 50;
+const TEST_PAGE_CACHE_SIZE: usize = 10;
 
 ///
 /// Initialize the page cache. This must be called once at page server startup.
 ///
-pub fn init(size: usize) {
-    if PAGE_CACHE.set(PageCache::new(size)).is_err() {
+pub fn init(conf: &'static PageServerConf) {
+    if PAGE_CACHE
+        .set(PageCache::new(conf.page_cache_size))
+        .is_err()
+    {
         panic!("page cache already initialized");
     }
 }
@@ -90,7 +93,6 @@ const MAX_USAGE_COUNT: u8 = 5;
 /// CacheKey uniquely identifies a "thing" to cache in the page cache.
 ///
 #[derive(Debug, PartialEq, Eq, Clone)]
-#[allow(clippy::enum_variant_names)]
 enum CacheKey {
     MaterializedPage {
         hash_key: MaterializedPageHashKey,
@@ -100,17 +102,14 @@ enum CacheKey {
         file_id: u64,
         blkno: u32,
     },
-    ImmutableFilePage {
-        file_id: u64,
-        blkno: u32,
-    },
 }
 
 #[derive(Debug, PartialEq, Eq, Hash, Clone)]
 struct MaterializedPageHashKey {
     tenant_id: ZTenantId,
     timeline_id: ZTimelineId,
-    key: Key,
+    rel_tag: RelTag,
+    blknum: u32,
 }
 
 #[derive(Clone)]
@@ -178,8 +177,6 @@ pub struct PageCache {
 
     ephemeral_page_map: RwLock<HashMap<(u64, u32), usize>>,
 
-    immutable_page_map: RwLock<HashMap<(u64, u32), usize>>,
-
     /// The actual buffers with their metadata.
     slots: Box<[Slot]>,
 
@@ -202,12 +199,6 @@ impl std::ops::Deref for PageReadGuard<'_> {
     }
 }
 
-impl AsRef<[u8; PAGE_SZ]> for PageReadGuard<'_> {
-    fn as_ref(&self) -> &[u8; PAGE_SZ] {
-        self.0.buf
-    }
-}
-
 ///
 /// PageWriteGuard is a lease on a buffer for modifying it. The page is kept locked
 /// until the guard is dropped.
@@ -239,12 +230,6 @@ impl std::ops::Deref for PageWriteGuard<'_> {
     }
 }
 
-impl AsMut<[u8; PAGE_SZ]> for PageWriteGuard<'_> {
-    fn as_mut(&mut self) -> &mut [u8; PAGE_SZ] {
-        self.inner.buf
-    }
-}
-
 impl PageWriteGuard<'_> {
     /// Mark that the buffer contents are now valid.
     pub fn mark_valid(&mut self) {
@@ -309,14 +294,16 @@ impl PageCache {
         &self,
         tenant_id: ZTenantId,
         timeline_id: ZTimelineId,
-        key: &Key,
+        rel_tag: RelTag,
+        blknum: u32,
         lsn: Lsn,
     ) -> Option<(Lsn, PageReadGuard)> {
         let mut cache_key = CacheKey::MaterializedPage {
             hash_key: MaterializedPageHashKey {
                 tenant_id,
                 timeline_id,
-                key: *key,
+                rel_tag,
+                blknum,
             },
             lsn,
         };
@@ -339,7 +326,8 @@ impl PageCache {
         &self,
         tenant_id: ZTenantId,
         timeline_id: ZTimelineId,
-        key: Key,
+        rel_tag: RelTag,
+        blknum: u32,
         lsn: Lsn,
         img: &[u8],
     ) {
@@ -347,7 +335,8 @@ impl PageCache {
             hash_key: MaterializedPageHashKey {
                 tenant_id,
                 timeline_id,
-                key,
+                rel_tag,
+                blknum,
             },
             lsn,
         };
@@ -400,36 +389,6 @@ impl PageCache {
         }
     }
 
-    // Section 1.3: Public interface functions for working with immutable file pages.
-
-    pub fn read_immutable_buf(&self, file_id: u64, blkno: u32) -> ReadBufResult {
-        let mut cache_key = CacheKey::ImmutableFilePage { file_id, blkno };
-
-        self.lock_for_read(&mut cache_key)
-    }
-
-    /// Immediately drop all buffers belonging to given file, without writeback
-    pub fn drop_buffers_for_immutable(&self, drop_file_id: u64) {
-        for slot_idx in 0..self.slots.len() {
-            let slot = &self.slots[slot_idx];
-
-            let mut inner = slot.inner.write().unwrap();
-            if let Some(key) = &inner.key {
-                match key {
-                    CacheKey::ImmutableFilePage { file_id, blkno: _ }
-                        if *file_id == drop_file_id =>
-                    {
-                        // remove mapping for old buffer
-                        self.remove_mapping(key);
-                        inner.key = None;
-                        inner.dirty = false;
-                    }
-                    _ => {}
-                }
-            }
-        }
-    }
-
     //
     // Section 2: Internal interface functions for lookup/update.
     //
@@ -627,10 +586,6 @@ impl PageCache {
                 let map = self.ephemeral_page_map.read().unwrap();
                 Some(*map.get(&(*file_id, *blkno))?)
             }
-            CacheKey::ImmutableFilePage { file_id, blkno } => {
-                let map = self.immutable_page_map.read().unwrap();
-                Some(*map.get(&(*file_id, *blkno))?)
-            }
         }
     }
 
@@ -654,10 +609,6 @@ impl PageCache {
                 let map = self.ephemeral_page_map.read().unwrap();
                 Some(*map.get(&(*file_id, *blkno))?)
             }
-            CacheKey::ImmutableFilePage { file_id, blkno } => {
-                let map = self.immutable_page_map.read().unwrap();
-                Some(*map.get(&(*file_id, *blkno))?)
-            }
         }
     }
 
@@ -689,11 +640,6 @@ impl PageCache {
                 map.remove(&(*file_id, *blkno))
                     .expect("could not find old key in mapping");
             }
-            CacheKey::ImmutableFilePage { file_id, blkno } => {
-                let mut map = self.immutable_page_map.write().unwrap();
-                map.remove(&(*file_id, *blkno))
-                    .expect("could not find old key in mapping");
-            }
         }
     }
 
@@ -734,16 +680,6 @@ impl PageCache {
                     }
                 }
             }
-            CacheKey::ImmutableFilePage { file_id, blkno } => {
-                let mut map = self.immutable_page_map.write().unwrap();
-                match map.entry((*file_id, *blkno)) {
-                    Entry::Occupied(entry) => Some(*entry.get()),
-                    Entry::Vacant(entry) => {
-                        entry.insert(slot_idx);
-                        None
-                    }
-                }
-            }
         }
     }
 
@@ -755,33 +691,16 @@ impl PageCache {
     ///
     /// On return, the slot is empty and write-locked.
     fn find_victim(&self) -> (usize, RwLockWriteGuard<SlotInner>) {
-        let iter_limit = self.slots.len() * 10;
+        let iter_limit = self.slots.len() * 2;
         let mut iters = 0;
         loop {
-            iters += 1;
             let slot_idx = self.next_evict_slot.fetch_add(1, Ordering::Relaxed) % self.slots.len();
 
             let slot = &self.slots[slot_idx];
 
-            if slot.dec_usage_count() == 0 {
-                let mut inner = match slot.inner.try_write() {
-                    Ok(inner) => inner,
-                    Err(TryLockError::Poisoned(err)) => {
-                        panic!("buffer lock was poisoned: {:?}", err)
-                    }
-                    Err(TryLockError::WouldBlock) => {
-                        // If we have looped through the whole buffer pool 10 times
-                        // and still haven't found a victim buffer, something's wrong.
-                        // Maybe all the buffers were in locked. That could happen in
-                        // theory, if you have more threads holding buffers locked than
-                        // there are buffers in the pool. In practice, with a reasonably
-                        // large buffer pool it really shouldn't happen.
-                        if iters > iter_limit {
-                            panic!("could not find a victim buffer to evict");
-                        }
-                        continue;
-                    }
-                };
+            if slot.dec_usage_count() == 0 || iters >= iter_limit {
+                let mut inner = slot.inner.write().unwrap();
+
                 if let Some(old_key) = &inner.key {
                     if inner.dirty {
                         if let Err(err) = Self::writeback(old_key, inner.buf) {
@@ -806,6 +725,8 @@ impl PageCache {
                 }
                 return (slot_idx, inner);
             }
+
+            iters += 1;
         }
     }
 
@@ -814,20 +735,12 @@ impl PageCache {
             CacheKey::MaterializedPage {
                 hash_key: _,
                 lsn: _,
-            } => Err(std::io::Error::new(
-                std::io::ErrorKind::Other,
-                "unexpected dirty materialized page",
-            )),
+            } => {
+                panic!("unexpected dirty materialized page");
+            }
             CacheKey::EphemeralPage { file_id, blkno } => {
                 writeback_ephemeral_file(*file_id, *blkno, buf)
             }
-            CacheKey::ImmutableFilePage {
-                file_id: _,
-                blkno: _,
-            } => Err(std::io::Error::new(
-                std::io::ErrorKind::Other,
-                "unexpected dirty immutable page",
-            )),
         }
     }
 
@@ -858,7 +771,6 @@ impl PageCache {
         Self {
             materialized_page_map: Default::default(),
             ephemeral_page_map: Default::default(),
-            immutable_page_map: Default::default(),
             slots,
             next_evict_slot: AtomicUsize::new(0),
         }

pageserver/src/page_service.rs

@@ -20,37 +20,31 @@ use std::str;
 use std::str::FromStr;
 use std::sync::{Arc, RwLockReadGuard};
 use tracing::*;
-use utils::{
-    auth::{self, Claims, JwtAuth, Scope},
-    lsn::Lsn,
-    postgres_backend::{self, is_socket_read_timed_out, AuthType, PostgresBackend},
-    pq_proto::{BeMessage, FeMessage, RowDescriptor, SINGLE_COL_ROWDESC},
-    zid::{ZTenantId, ZTimelineId},
-};
+use zenith_metrics::{register_histogram_vec, HistogramVec};
+use zenith_utils::auth::{self, JwtAuth};
+use zenith_utils::auth::{Claims, Scope};
+use zenith_utils::lsn::Lsn;
+use zenith_utils::postgres_backend::is_socket_read_timed_out;
+use zenith_utils::postgres_backend::PostgresBackend;
+use zenith_utils::postgres_backend::{self, AuthType};
+use zenith_utils::pq_proto::{BeMessage, FeMessage, RowDescriptor, SINGLE_COL_ROWDESC};
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
 use crate::basebackup;
-use crate::config::{PageServerConf, ProfilingConfig};
-use crate::pgdatadir_mapping::{DatadirTimeline, LsnForTimestamp};
-use crate::profiling::profpoint_start;
-use crate::reltag::RelTag;
-use crate::repository::Repository;
+use crate::config::PageServerConf;
+use crate::relish::*;
 use crate::repository::Timeline;
 use crate::tenant_mgr;
 use crate::thread_mgr;
 use crate::thread_mgr::ThreadKind;
 use crate::walreceiver;
 use crate::CheckpointConfig;
-use metrics::{register_histogram_vec, HistogramVec};
-use postgres_ffi::xlog_utils::to_pg_timestamp;
-
-use postgres_ffi::pg_constants;
 
 // Wrapped in libpq CopyData
 enum PagestreamFeMessage {
     Exists(PagestreamExistsRequest),
     Nblocks(PagestreamNblocksRequest),
     GetPage(PagestreamGetPageRequest),
-    DbSize(PagestreamDbSizeRequest),
 }
 
 // Wrapped in libpq CopyData
@@ -59,7 +53,6 @@ enum PagestreamBeMessage {
     Nblocks(PagestreamNblocksResponse),
     GetPage(PagestreamGetPageResponse),
     Error(PagestreamErrorResponse),
-    DbSize(PagestreamDbSizeResponse),
 }
 
 #[derive(Debug)]
@@ -84,13 +77,6 @@ struct PagestreamGetPageRequest {
     blkno: u32,
 }
 
-#[derive(Debug)]
-struct PagestreamDbSizeRequest {
-    latest: bool,
-    lsn: Lsn,
-    dbnode: u32,
-}
-
 #[derive(Debug)]
 struct PagestreamExistsResponse {
     exists: bool,
@@ -111,11 +97,6 @@ struct PagestreamErrorResponse {
     message: String,
 }
 
-#[derive(Debug)]
-struct PagestreamDbSizeResponse {
-    db_size: i64,
-}
-
 impl PagestreamFeMessage {
     fn parse(mut body: Bytes) -> anyhow::Result<PagestreamFeMessage> {
         // TODO these gets can fail
@@ -157,11 +138,6 @@ impl PagestreamFeMessage {
                 },
                 blkno: body.get_u32(),
             })),
-            3 => Ok(PagestreamFeMessage::DbSize(PagestreamDbSizeRequest {
-                latest: body.get_u8() != 0,
-                lsn: Lsn::from(body.get_u64()),
-                dbnode: body.get_u32(),
-            })),
             _ => bail!("unknown smgr message tag: {},'{:?}'", msg_tag, body),
         }
     }
@@ -192,10 +168,6 @@ impl PagestreamBeMessage {
                 bytes.put(resp.message.as_bytes());
                 bytes.put_u8(0); // null terminator
             }
-            Self::DbSize(resp) => {
-                bytes.put_u8(104); /* tag from pagestore_client.h */
-                bytes.put_i64(resp.db_size);
-            }
         }
 
         bytes.into()
@@ -256,7 +228,6 @@ pub fn thread_main(
                     None,
                     None,
                     "serving Page Service thread",
-                    false,
                     move || page_service_conn_main(conf, local_auth, socket, auth_type),
                 ) {
                     // Thread creation failed. Log the error and continue.
@@ -325,7 +296,7 @@ const TIME_BUCKETS: &[f64] = &[
 
 lazy_static! {
     static ref SMGR_QUERY_TIME: HistogramVec = register_histogram_vec!(
-        "pageserver_smgr_query_seconds",
+        "pageserver_smgr_query_time",
         "Time spent on smgr query handling",
         &["smgr_query_type", "tenant_id", "timeline_id"],
         TIME_BUCKETS.into()
@@ -351,17 +322,14 @@ impl PageServerHandler {
         let _enter = info_span!("pagestream", timeline = %timelineid, tenant = %tenantid).entered();
 
         // Check that the timeline exists
-        let timeline = tenant_mgr::get_local_timeline_with_load(tenantid, timelineid)
+        let timeline = tenant_mgr::get_timeline_for_tenant_load(tenantid, timelineid)
             .context("Cannot load local timeline")?;
 
         /* switch client to COPYBOTH */
         pgb.write_message(&BeMessage::CopyBothResponse)?;
 
         while !thread_mgr::is_shutdown_requested() {
-            let msg = pgb.read_message();
-
-            let profiling_guard = profpoint_start(self.conf, ProfilingConfig::PageRequests);
-            match msg {
+            match pgb.read_message() {
                 Ok(message) => {
                     if let Some(message) = message {
                         trace!("query: {:?}", message);
@@ -391,11 +359,6 @@ impl PageServerHandler {
                                 .observe_closure_duration(|| {
                                     self.handle_get_page_at_lsn_request(timeline.as_ref(), &req)
                                 }),
-                            PagestreamFeMessage::DbSize(req) => SMGR_QUERY_TIME
-                                .with_label_values(&["get_db_size", &tenant_id, &timeline_id])
-                                .observe_closure_duration(|| {
-                                    self.handle_db_size_request(timeline.as_ref(), &req)
-                                }),
                         };
 
                         let response = response.unwrap_or_else(|e| {
@@ -418,7 +381,6 @@ impl PageServerHandler {
                     }
                 }
             }
-            drop(profiling_guard);
         }
         Ok(())
     }
@@ -435,8 +397,8 @@ impl PageServerHandler {
     /// In either case, if the page server hasn't received the WAL up to the
     /// requested LSN yet, we will wait for it to arrive. The return value is
     /// the LSN that should be used to look up the page versions.
-    fn wait_or_get_last_lsn<R: Repository>(
-        timeline: &DatadirTimeline<R>,
+    fn wait_or_get_last_lsn(
+        timeline: &dyn Timeline,
         mut lsn: Lsn,
         latest: bool,
         latest_gc_cutoff_lsn: &RwLockReadGuard<Lsn>,
@@ -463,7 +425,7 @@ impl PageServerHandler {
             if lsn <= last_record_lsn {
                 lsn = last_record_lsn;
             } else {
-                timeline.tline.wait_lsn(lsn)?;
+                timeline.wait_lsn(lsn)?;
                 // Since we waited for 'lsn' to arrive, that is now the last
                 // record LSN. (Or close enough for our purposes; the
                 // last-record LSN can advance immediately after we return
@@ -473,7 +435,7 @@ impl PageServerHandler {
             if lsn == Lsn(0) {
                 bail!("invalid LSN(0) in request");
             }
-            timeline.tline.wait_lsn(lsn)?;
+            timeline.wait_lsn(lsn)?;
         }
         ensure!(
             lsn >= **latest_gc_cutoff_lsn,
@@ -483,73 +445,54 @@ impl PageServerHandler {
         Ok(lsn)
     }
 
-    fn handle_get_rel_exists_request<R: Repository>(
+    fn handle_get_rel_exists_request(
         &self,
-        timeline: &DatadirTimeline<R>,
+        timeline: &dyn Timeline,
         req: &PagestreamExistsRequest,
     ) -> Result<PagestreamBeMessage> {
         let _enter = info_span!("get_rel_exists", rel = %req.rel, req_lsn = %req.lsn).entered();
 
-        let latest_gc_cutoff_lsn = timeline.tline.get_latest_gc_cutoff_lsn();
+        let tag = RelishTag::Relation(req.rel);
+        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
         let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn)?;
 
-        let exists = timeline.get_rel_exists(req.rel, lsn)?;
+        let exists = timeline.get_rel_exists(tag, lsn)?;
 
         Ok(PagestreamBeMessage::Exists(PagestreamExistsResponse {
             exists,
         }))
     }
 
-    fn handle_get_nblocks_request<R: Repository>(
+    fn handle_get_nblocks_request(
         &self,
-        timeline: &DatadirTimeline<R>,
+        timeline: &dyn Timeline,
         req: &PagestreamNblocksRequest,
     ) -> Result<PagestreamBeMessage> {
         let _enter = info_span!("get_nblocks", rel = %req.rel, req_lsn = %req.lsn).entered();
-        let latest_gc_cutoff_lsn = timeline.tline.get_latest_gc_cutoff_lsn();
+        let tag = RelishTag::Relation(req.rel);
+        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
         let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn)?;
 
-        let n_blocks = timeline.get_rel_size(req.rel, lsn)?;
+        let n_blocks = timeline.get_relish_size(tag, lsn)?;
+
+        // Return 0 if relation is not found.
+        // This is what postgres smgr expects.
+        let n_blocks = n_blocks.unwrap_or(0);
 
         Ok(PagestreamBeMessage::Nblocks(PagestreamNblocksResponse {
             n_blocks,
         }))
     }
 
-    fn handle_db_size_request<R: Repository>(
+    fn handle_get_page_at_lsn_request(
         &self,
-        timeline: &DatadirTimeline<R>,
-        req: &PagestreamDbSizeRequest,
-    ) -> Result<PagestreamBeMessage> {
-        let _enter = info_span!("get_db_size", dbnode = %req.dbnode, req_lsn = %req.lsn).entered();
-        let latest_gc_cutoff_lsn = timeline.tline.get_latest_gc_cutoff_lsn();
-        let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn)?;
-
-        let all_rels = timeline.list_rels(pg_constants::DEFAULTTABLESPACE_OID, req.dbnode, lsn)?;
-        let mut total_blocks: i64 = 0;
-
-        for rel in all_rels {
-            if rel.forknum == 0 {
-                let n_blocks = timeline.get_rel_size(rel, lsn).unwrap_or(0);
-                total_blocks += n_blocks as i64;
-            }
-        }
-
-        let db_size = total_blocks * pg_constants::BLCKSZ as i64;
-
-        Ok(PagestreamBeMessage::DbSize(PagestreamDbSizeResponse {
-            db_size,
-        }))
-    }
-
-    fn handle_get_page_at_lsn_request<R: Repository>(
-        &self,
-        timeline: &DatadirTimeline<R>,
+        timeline: &dyn Timeline,
         req: &PagestreamGetPageRequest,
     ) -> Result<PagestreamBeMessage> {
         let _enter = info_span!("get_page", rel = %req.rel, blkno = &req.blkno, req_lsn = %req.lsn)
             .entered();
-        let latest_gc_cutoff_lsn = timeline.tline.get_latest_gc_cutoff_lsn();
+        let tag = RelishTag::Relation(req.rel);
+        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
         let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn)?;
         /*
         // Add a 1s delay to some requests. The delayed causes the requests to
@@ -559,7 +502,7 @@ impl PageServerHandler {
             std::thread::sleep(std::time::Duration::from_millis(1000));
         }
         */
-        let page = timeline.get_rel_page_at_lsn(req.rel, req.blkno, lsn)?;
+        let page = timeline.get_page_at_lsn(tag, req.blkno, lsn)?;
 
         Ok(PagestreamBeMessage::GetPage(PagestreamGetPageResponse {
             page,
@@ -571,16 +514,17 @@ impl PageServerHandler {
         pgb: &mut PostgresBackend,
         timelineid: ZTimelineId,
         lsn: Option<Lsn>,
+        prev_lsn: Option<Lsn>,
         tenantid: ZTenantId,
+        full_backup: bool,
     ) -> anyhow::Result<()> {
         let span = info_span!("basebackup", timeline = %timelineid, tenant = %tenantid, lsn = field::Empty);
         let _enter = span.enter();
-        info!("starting");
 
         // check that the timeline exists
-        let timeline = tenant_mgr::get_local_timeline_with_load(tenantid, timelineid)
+        let timeline = tenant_mgr::get_timeline_for_tenant_load(tenantid, timelineid)
             .context("Cannot load local timeline")?;
-        let latest_gc_cutoff_lsn = timeline.tline.get_latest_gc_cutoff_lsn();
+        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
         if let Some(lsn) = lsn {
             timeline
                 .check_lsn_is_in_scope(lsn, &latest_gc_cutoff_lsn)
@@ -593,12 +537,14 @@ impl PageServerHandler {
         /* Send a tarball of the latest layer on the timeline */
         {
             let mut writer = CopyDataSink { pgb };
-            let mut basebackup = basebackup::Basebackup::new(&mut writer, &timeline, lsn)?;
+
+            let mut basebackup =
+                basebackup::Basebackup::new(&mut writer, &timeline, lsn, prev_lsn, full_backup)?;
             span.record("lsn", &basebackup.lsn.to_string().as_str());
             basebackup.send_tarball()?;
         }
         pgb.write_message(&BeMessage::CopyDone)?;
-        info!("done");
+        debug!("CopyDone sent!");
 
         Ok(())
     }
@@ -632,6 +578,7 @@ impl postgres_backend::Handler for PageServerHandler {
         let data = self
             .auth
             .as_ref()
+            .as_ref()
             .unwrap()
             .decode(str::from_utf8(jwt_response)?)?;
 
@@ -692,7 +639,67 @@ impl postgres_backend::Handler for PageServerHandler {
             };
 
             // Check that the timeline exists
-            self.handle_basebackup_request(pgb, timelineid, lsn, tenantid)?;
+            self.handle_basebackup_request(pgb, timelineid, lsn, None, tenantid, false)?;
+            pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
+        }
+        // return pair of prev_lsn and last_lsn
+        else if query_string.starts_with("get_last_record_rlsn ") {
+            let (_, params_raw) = query_string.split_at("get_last_record_rlsn ".len());
+            let params = params_raw.split_whitespace().collect::<Vec<_>>();
+
+            ensure!(
+                params.len() == 2,
+                "invalid param number for get_last_record_rlsn command"
+            );
+
+            let tenantid = ZTenantId::from_str(params[0])?;
+            let timelineid = ZTimelineId::from_str(params[1])?;
+
+            self.check_permission(Some(tenantid))?;
+            let timeline = tenant_mgr::get_timeline_for_tenant_load(tenantid, timelineid)
+                .context("Cannot load local timeline")?;
+
+            let end_of_timeline = timeline.get_last_record_rlsn();
+
+            pgb.write_message_noflush(&BeMessage::RowDescription(&[
+                RowDescriptor::text_col(b"prev_lsn"),
+                RowDescriptor::text_col(b"last_lsn"),
+            ]))?
+            .write_message_noflush(&BeMessage::DataRow(&[
+                Some(end_of_timeline.prev.to_string().as_bytes()),
+                Some(end_of_timeline.last.to_string().as_bytes()),
+            ]))?
+            .write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;
+        }
+        // same as basebackup, but result includes relational data as well
+        else if query_string.starts_with("fullbackup ") {
+            let (_, params_raw) = query_string.split_at("fullbackup ".len());
+            let params = params_raw.split_whitespace().collect::<Vec<_>>();
+
+            ensure!(
+                params.len() >= 2,
+                "invalid param number for fullbackup command"
+            );
+
+            let tenantid = ZTenantId::from_str(params[0])?;
+            let timelineid = ZTimelineId::from_str(params[1])?;
+
+            // The caller is responsible for providing correct lsn and prev_lsn.
+            let lsn = if params.len() > 2 {
+                Some(Lsn::from_str(params[2])?)
+            } else {
+                None
+            };
+            let prev_lsn = if params.len() > 3 {
+                Some(Lsn::from_str(params[3])?)
+            } else {
+                None
+            };
+
+            self.check_permission(Some(tenantid))?;
+
+            // Check that the timeline exists
+            self.handle_basebackup_request(pgb, timelineid, lsn, prev_lsn, tenantid, true)?;
             pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
         } else if query_string.starts_with("callmemaybe ") {
             // callmemaybe <zenith tenantid as hex string> <zenith timelineid as hex string> <connstr>
@@ -712,7 +719,7 @@ impl postgres_backend::Handler for PageServerHandler {
                 info_span!("callmemaybe", timeline = %timelineid, tenant = %tenantid).entered();
 
             // Check that the timeline exists
-            tenant_mgr::get_local_timeline_with_load(tenantid, timelineid)
+            tenant_mgr::get_timeline_for_tenant_load(tenantid, timelineid)
                 .context("Cannot load local timeline")?;
 
             walreceiver::launch_wal_receiver(self.conf, tenantid, timelineid, &connstr)?;
@@ -723,10 +730,7 @@ impl postgres_backend::Handler for PageServerHandler {
             // on connect
             pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
         } else if query_string.starts_with("failpoints ") {
-            ensure!(fail::has_failpoints(), "Cannot manage failpoints because pageserver was compiled without failpoints support");
-
             let (_, failpoints) = query_string.split_at("failpoints ".len());
-
             for failpoint in failpoints.split(';') {
                 if let Some((name, actions)) = failpoint.split_once('=') {
                     info!("cfg failpoint: {} {}", name, actions);
@@ -736,39 +740,6 @@ impl postgres_backend::Handler for PageServerHandler {
                 }
             }
             pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
-        } else if query_string.starts_with("show ") {
-            // show <tenant_id>
-            let (_, params_raw) = query_string.split_at("show ".len());
-            let params = params_raw.split(' ').collect::<Vec<_>>();
-            ensure!(params.len() == 1, "invalid param number for config command");
-            let tenantid = ZTenantId::from_str(params[0])?;
-            let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
-            pgb.write_message_noflush(&BeMessage::RowDescription(&[
-                RowDescriptor::int8_col(b"checkpoint_distance"),
-                RowDescriptor::int8_col(b"compaction_target_size"),
-                RowDescriptor::int8_col(b"compaction_period"),
-                RowDescriptor::int8_col(b"compaction_threshold"),
-                RowDescriptor::int8_col(b"gc_horizon"),
-                RowDescriptor::int8_col(b"gc_period"),
-                RowDescriptor::int8_col(b"image_creation_threshold"),
-                RowDescriptor::int8_col(b"pitr_interval"),
-            ]))?
-            .write_message_noflush(&BeMessage::DataRow(&[
-                Some(repo.get_checkpoint_distance().to_string().as_bytes()),
-                Some(repo.get_compaction_target_size().to_string().as_bytes()),
-                Some(
-                    repo.get_compaction_period()
-                        .as_secs()
-                        .to_string()
-                        .as_bytes(),
-                ),
-                Some(repo.get_compaction_threshold().to_string().as_bytes()),
-                Some(repo.get_gc_horizon().to_string().as_bytes()),
-                Some(repo.get_gc_period().as_secs().to_string().as_bytes()),
-                Some(repo.get_image_creation_threshold().to_string().as_bytes()),
-                Some(repo.get_pitr_interval().as_secs().to_string().as_bytes()),
-            ]))?
-            .write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;
         } else if query_string.starts_with("do_gc ") {
             // Run GC immediately on given timeline.
             // FIXME: This is just for tests. See test_runner/batch_others/test_gc.py.
@@ -786,57 +757,78 @@ impl postgres_backend::Handler for PageServerHandler {
 
             let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;
             let timelineid = ZTimelineId::from_str(caps.get(2).unwrap().as_str())?;
-
-            let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
-
             let gc_horizon: u64 = caps
                 .get(4)
                 .map(|h| h.as_str().parse())
-                .unwrap_or_else(|| Ok(repo.get_gc_horizon()))?;
+                .unwrap_or(Ok(self.conf.gc_horizon))?;
 
             let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
-            // Use tenant's pitr setting
-            let pitr = repo.get_pitr_interval();
-            let result = repo.gc_iteration(Some(timelineid), gc_horizon, pitr, true)?;
+            let result = repo.gc_iteration(Some(timelineid), gc_horizon, true)?;
             pgb.write_message_noflush(&BeMessage::RowDescription(&[
-                RowDescriptor::int8_col(b"layers_total"),
-                RowDescriptor::int8_col(b"layers_needed_by_cutoff"),
-                RowDescriptor::int8_col(b"layers_needed_by_pitr"),
-                RowDescriptor::int8_col(b"layers_needed_by_branches"),
-                RowDescriptor::int8_col(b"layers_not_updated"),
-                RowDescriptor::int8_col(b"layers_removed"),
+                RowDescriptor::int8_col(b"layer_relfiles_total"),
+                RowDescriptor::int8_col(b"layer_relfiles_needed_by_cutoff"),
+                RowDescriptor::int8_col(b"layer_relfiles_needed_by_branches"),
+                RowDescriptor::int8_col(b"layer_relfiles_not_updated"),
+                RowDescriptor::int8_col(b"layer_relfiles_needed_as_tombstone"),
+                RowDescriptor::int8_col(b"layer_relfiles_removed"),
+                RowDescriptor::int8_col(b"layer_relfiles_dropped"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_total"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_needed_by_cutoff"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_needed_by_branches"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_not_updated"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_needed_as_tombstone"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_removed"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_dropped"),
                 RowDescriptor::int8_col(b"elapsed"),
             ]))?
             .write_message_noflush(&BeMessage::DataRow(&[
-                Some(result.layers_total.to_string().as_bytes()),
-                Some(result.layers_needed_by_cutoff.to_string().as_bytes()),
-                Some(result.layers_needed_by_pitr.to_string().as_bytes()),
-                Some(result.layers_needed_by_branches.to_string().as_bytes()),
-                Some(result.layers_not_updated.to_string().as_bytes()),
-                Some(result.layers_removed.to_string().as_bytes()),
+                Some(result.ondisk_relfiles_total.to_string().as_bytes()),
+                Some(
+                    result
+                        .ondisk_relfiles_needed_by_cutoff
+                        .to_string()
+                        .as_bytes(),
+                ),
+                Some(
+                    result
+                        .ondisk_relfiles_needed_by_branches
+                        .to_string()
+                        .as_bytes(),
+                ),
+                Some(result.ondisk_relfiles_not_updated.to_string().as_bytes()),
+                Some(
+                    result
+                        .ondisk_relfiles_needed_as_tombstone
+                        .to_string()
+                        .as_bytes(),
+                ),
+                Some(result.ondisk_relfiles_removed.to_string().as_bytes()),
+                Some(result.ondisk_relfiles_dropped.to_string().as_bytes()),
+                Some(result.ondisk_nonrelfiles_total.to_string().as_bytes()),
+                Some(
+                    result
+                        .ondisk_nonrelfiles_needed_by_cutoff
+                        .to_string()
+                        .as_bytes(),
+                ),
+                Some(
+                    result
+                        .ondisk_nonrelfiles_needed_by_branches
+                        .to_string()
+                        .as_bytes(),
+                ),
+                Some(result.ondisk_nonrelfiles_not_updated.to_string().as_bytes()),
+                Some(
+                    result
+                        .ondisk_nonrelfiles_needed_as_tombstone
+                        .to_string()
+                        .as_bytes(),
+                ),
+                Some(result.ondisk_nonrelfiles_removed.to_string().as_bytes()),
+                Some(result.ondisk_nonrelfiles_dropped.to_string().as_bytes()),
                 Some(result.elapsed.as_millis().to_string().as_bytes()),
             ]))?
             .write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;
-        } else if query_string.starts_with("compact ") {
-            // Run compaction immediately on given timeline.
-            // FIXME This is just for tests. Don't expect this to be exposed to
-            // the users or the api.
-
-            // compact <tenant_id> <timeline_id>
-            let re = Regex::new(r"^compact ([[:xdigit:]]+)\s([[:xdigit:]]+)($|\s)?").unwrap();
-
-            let caps = re
-                .captures(query_string)
-                .with_context(|| format!("Invalid compact: '{}'", query_string))?;
-
-            let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;
-            let timelineid = ZTimelineId::from_str(caps.get(2).unwrap().as_str())?;
-            let timeline = tenant_mgr::get_local_timeline_with_load(tenantid, timelineid)
-                .context("Couldn't load timeline")?;
-            timeline.tline.compact()?;
-
-            pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
-                .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
         } else if query_string.starts_with("checkpoint ") {
             // Run checkpoint immediately on given timeline.
 
@@ -850,46 +842,12 @@ impl postgres_backend::Handler for PageServerHandler {
             let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;
             let timelineid = ZTimelineId::from_str(caps.get(2).unwrap().as_str())?;
 
-            let timeline = tenant_mgr::get_local_timeline_with_load(tenantid, timelineid)
+            let timeline = tenant_mgr::get_timeline_for_tenant_load(tenantid, timelineid)
                 .context("Cannot load local timeline")?;
 
-            timeline.tline.checkpoint(CheckpointConfig::Forced)?;
-
-            // Also compact it.
-            //
-            // FIXME: This probably shouldn't be part of a "checkpoint" command, but a
-            // separate operation. Update the tests if you change this.
-            timeline.tline.compact()?;
-
+            timeline.checkpoint(CheckpointConfig::Forced)?;
             pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
                 .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
-        } else if query_string.starts_with("get_lsn_by_timestamp ") {
-            // Locate LSN of last transaction with timestamp less or equal than sppecified
-            // TODO lazy static
-            let re = Regex::new(r"^get_lsn_by_timestamp ([[:xdigit:]]+) ([[:xdigit:]]+) '(.*)'$")
-                .unwrap();
-            let caps = re
-                .captures(query_string)
-                .with_context(|| format!("invalid get_lsn_by_timestamp: '{}'", query_string))?;
-
-            let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;
-            let timelineid = ZTimelineId::from_str(caps.get(2).unwrap().as_str())?;
-            let timeline = tenant_mgr::get_local_timeline_with_load(tenantid, timelineid)
-                .context("Cannot load local timeline")?;
-
-            let timestamp = humantime::parse_rfc3339(caps.get(3).unwrap().as_str())?;
-            let timestamp_pg = to_pg_timestamp(timestamp);
-
-            pgb.write_message_noflush(&BeMessage::RowDescription(&[RowDescriptor::text_col(
-                b"lsn",
-            )]))?;
-            let result = match timeline.find_lsn_for_timestamp(timestamp_pg)? {
-                LsnForTimestamp::Present(lsn) => format!("{}", lsn),
-                LsnForTimestamp::Future(_lsn) => "future".into(),
-                LsnForTimestamp::Past(_lsn) => "past".into(),
-            };
-            pgb.write_message_noflush(&BeMessage::DataRow(&[Some(result.as_bytes())]))?;
-            pgb.write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;
         } else {
             bail!("unknown command");
         }

pageserver/src/profiling.rs

@@ -1,101 +0,0 @@
-//!
-//! Support for profiling
-//!
-//! This relies on a modified version of the 'pprof-rs' crate. That's not very
-//! nice, so to avoid a hard dependency on that, this is an optional feature.
-//!
-use crate::config::{PageServerConf, ProfilingConfig};
-
-/// The actual implementation is in the `profiling_impl` submodule. If the profiling
-/// feature is not enabled, it's just a dummy implementation that panics if you
-/// try to enabled profiling in the configuration.
-pub use profiling_impl::*;
-
-#[cfg(feature = "profiling")]
-mod profiling_impl {
-    use super::*;
-    use pprof;
-    use std::marker::PhantomData;
-
-    /// Start profiling the current thread. Returns a guard object;
-    /// the profiling continues until the guard is dropped.
-    ///
-    /// Note: profiling is not re-entrant. If you call 'profpoint_start' while
-    /// profiling is already started, nothing happens, and the profiling will be
-    /// stopped when either guard object is dropped.
-    #[inline]
-    pub fn profpoint_start(
-        conf: &crate::config::PageServerConf,
-        point: ProfilingConfig,
-    ) -> Option<ProfilingGuard> {
-        if conf.profiling == point {
-            pprof::start_profiling();
-            Some(ProfilingGuard(PhantomData))
-        } else {
-            None
-        }
-    }
-
-    /// A hack to remove Send and Sync from the ProfilingGuard. Because the
-    /// profiling is attached to current thread.
-    ////
-    /// See comments in https://github.com/rust-lang/rust/issues/68318
-    type PhantomUnsend = std::marker::PhantomData<*mut u8>;
-
-    pub struct ProfilingGuard(PhantomUnsend);
-
-    impl Drop for ProfilingGuard {
-        fn drop(&mut self) {
-            pprof::stop_profiling();
-        }
-    }
-
-    /// Initialize the profiler. This must be called before any 'profpoint_start' calls.
-    pub fn init_profiler(conf: &PageServerConf) -> Option<pprof::ProfilerGuard> {
-        if conf.profiling != ProfilingConfig::Disabled {
-            Some(pprof::ProfilerGuardBuilder::default().build().unwrap())
-        } else {
-            None
-        }
-    }
-
-    /// Exit the profiler. Writes the flamegraph to current workdir.
-    pub fn exit_profiler(_conf: &PageServerConf, profiler_guard: &Option<pprof::ProfilerGuard>) {
-        // Write out the flamegraph
-        if let Some(profiler_guard) = profiler_guard {
-            if let Ok(report) = profiler_guard.report().build() {
-                // this gets written under the workdir
-                let file = std::fs::File::create("flamegraph.svg").unwrap();
-                let mut options = pprof::flamegraph::Options::default();
-                options.image_width = Some(2500);
-                report.flamegraph_with_options(file, &mut options).unwrap();
-            }
-        }
-    }
-}
-
-/// Dummy implementation when compiling without profiling feature or for non-linux OSes.
-#[cfg(not(feature = "profiling"))]
-mod profiling_impl {
-    use super::*;
-
-    pub struct DummyProfilerGuard;
-
-    pub fn profpoint_start(
-        _conf: &PageServerConf,
-        _point: ProfilingConfig,
-    ) -> Option<DummyProfilerGuard> {
-        None
-    }
-
-    pub fn init_profiler(conf: &PageServerConf) -> Option<DummyProfilerGuard> {
-        if conf.profiling != ProfilingConfig::Disabled {
-            // shouldn't happen, we don't allow profiling in the config if the support
-            // for it is disabled.
-            panic!("profiling enabled but the binary was compiled without profiling support");
-        }
-        None
-    }
-
-    pub fn exit_profiler(_conf: &PageServerConf, _guard: &Option<DummyProfilerGuard>) {}
-}

pageserver/src/relish.rs

@@ -0,0 +1,251 @@
+//!
+//! Zenith stores PostgreSQL relations, and some other files, in the
+//! repository.  The relations (i.e. tables and indexes) take up most
+//! of the space in a typical installation, while the other files are
+//! small. We call each relation and other file that is stored in the
+//! repository a "relish". It comes from "rel"-ish, as in "kind of a
+//! rel", because it covers relations as well as other things that are
+//! not relations, but are treated similarly for the purposes of the
+//! storage layer.
+//!
+//! This source file contains the definition of the RelishTag struct,
+//! which uniquely identifies a relish.
+//!
+//! Relishes come in two flavors: blocky and non-blocky. Relations and
+//! SLRUs are blocky, that is, they are divided into 8k blocks, and
+//! the repository tracks their size. Other relishes are non-blocky:
+//! the content of the whole relish is stored as one blob. Block
+//! number must be passed as 0 for all operations on a non-blocky
+//! relish. The one "block" that you store in a non-blocky relish can
+//! have arbitrary size, but they are expected to be small, or you
+//! will have performance issues.
+//!
+//! All relishes are versioned by LSN in the repository.
+//!
+
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+use postgres_ffi::pg_constants;
+use postgres_ffi::relfile_utils::forknumber_to_name;
+use postgres_ffi::{Oid, TransactionId};
+
+///
+/// RelishTag identifies one relish.
+///
+#[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub enum RelishTag {
+    // Relations correspond to PostgreSQL relation forks. Each
+    // PostgreSQL relation fork is considered a separate relish.
+    Relation(RelTag),
+
+    // SLRUs include pg_clog, pg_multixact/members, and
+    // pg_multixact/offsets. There are other SLRUs in PostgreSQL, but
+    // they don't need to be stored permanently (e.g. pg_subtrans),
+    // or we do not support them in zenith yet (pg_commit_ts).
+    //
+    // These are currently never requested directly by the compute
+    // nodes, although in principle that would be possible. However,
+    // when a new compute node is created, these are included in the
+    // tarball that we send to the compute node to initialize the
+    // PostgreSQL data directory.
+    //
+    // Each SLRU segment in PostgreSQL is considered a separate
+    // relish. For example, pg_clog/0000, pg_clog/0001, and so forth.
+    //
+    // SLRU segments are divided into blocks, like relations.
+    Slru { slru: SlruKind, segno: u32 },
+
+    // Miscellaneous other files that need to be included in the
+    // tarball at compute node creation. These are non-blocky, and are
+    // expected to be small.
+
+    //
+    // FileNodeMap represents PostgreSQL's 'pg_filenode.map'
+    // files. They are needed to map catalog table OIDs to filenode
+    // numbers. Usually the mapping is done by looking up a relation's
+    // 'relfilenode' field in the 'pg_class' system table, but that
+    // doesn't work for 'pg_class' itself and a few other such system
+    // relations. See PostgreSQL relmapper.c for details.
+    //
+    // Each database has a map file for its local mapped catalogs,
+    // and there is a separate map file for shared catalogs.
+    //
+    // These files are always 512 bytes long (although we don't check
+    // or care about that in the page server).
+    //
+    FileNodeMap { spcnode: Oid, dbnode: Oid },
+
+    //
+    // State files for prepared transactions (e.g pg_twophase/1234)
+    //
+    TwoPhase { xid: TransactionId },
+
+    // The control file, stored in global/pg_control
+    ControlFile,
+
+    // Special entry that represents PostgreSQL checkpoint. It doesn't
+    // correspond to to any physical file in PostgreSQL, but we use it
+    // to track fields needed to restore the checkpoint data in the
+    // control file, when a compute node is created.
+    Checkpoint,
+}
+
+impl RelishTag {
+    pub const fn is_blocky(&self) -> bool {
+        match self {
+            // These relishes work with blocks
+            RelishTag::Relation(_) | RelishTag::Slru { slru: _, segno: _ } => true,
+
+            // and these don't
+            RelishTag::FileNodeMap {
+                spcnode: _,
+                dbnode: _,
+            }
+            | RelishTag::TwoPhase { xid: _ }
+            | RelishTag::ControlFile
+            | RelishTag::Checkpoint => false,
+        }
+    }
+
+    // Physical relishes represent files and use
+    // RelationSizeEntry to track existing and dropped files.
+    // They can be both blocky and non-blocky.
+    pub const fn is_physical(&self) -> bool {
+        match self {
+            // These relishes represent physical files
+            RelishTag::Relation(_)
+            | RelishTag::Slru { .. }
+            | RelishTag::FileNodeMap { .. }
+            | RelishTag::TwoPhase { .. } => true,
+
+            // and these don't
+            RelishTag::ControlFile | RelishTag::Checkpoint => false,
+        }
+    }
+
+    // convenience function to check if this relish is a normal relation.
+    pub const fn is_relation(&self) -> bool {
+        matches!(self, RelishTag::Relation(_))
+    }
+}
+
+///
+/// Relation data file segment id throughout the Postgres cluster.
+///
+/// Every data file in Postgres is uniquely identified by 4 numbers:
+/// - relation id / node (`relnode`)
+/// - database id (`dbnode`)
+/// - tablespace id (`spcnode`), in short this is a unique id of a separate
+///   directory to store data files.
+/// - forknumber (`forknum`) is used to split different kinds of data of the same relation
+///   between some set of files (`relnode`, `relnode_fsm`, `relnode_vm`).
+///
+/// In native Postgres code `RelFileNode` structure and individual `ForkNumber` value
+/// are used for the same purpose.
+/// [See more related comments here](https:///github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/relfilenode.h#L57).
+///
+#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy, Serialize, Deserialize)]
+pub struct RelTag {
+    pub forknum: u8,
+    pub spcnode: Oid,
+    pub dbnode: Oid,
+    pub relnode: Oid,
+}
+
+/// Display RelTag in the same format that's used in most PostgreSQL debug messages:
+///
+/// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
+///
+impl fmt::Display for RelTag {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if let Some(forkname) = forknumber_to_name(self.forknum) {
+            write!(
+                f,
+                "{}/{}/{}_{}",
+                self.spcnode, self.dbnode, self.relnode, forkname
+            )
+        } else {
+            write!(f, "{}/{}/{}", self.spcnode, self.dbnode, self.relnode)
+        }
+    }
+}
+
+impl RelTag {
+    pub fn to_segfile_name(&self, segno: u32) -> String {
+        let mut name = if self.spcnode == pg_constants::GLOBALTABLESPACE_OID {
+            "global/".to_string()
+        } else {
+            format!("base/{}/", self.dbnode)
+        };
+
+        name += &self.relnode.to_string();
+
+        if let Some(fork_name) = forknumber_to_name(self.forknum) {
+            name += "_";
+            name += fork_name;
+        }
+
+        if segno != 0 {
+            name += ".";
+            name += &segno.to_string();
+        }
+
+        name
+    }
+}
+
+/// Display RelTag in the same format that's used in most PostgreSQL debug messages:
+///
+/// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
+///
+impl fmt::Display for RelishTag {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            RelishTag::Relation(rel) => rel.fmt(f),
+            RelishTag::Slru { slru, segno } => {
+                // e.g. pg_clog/0001
+                write!(f, "{}/{:04X}", slru.to_str(), segno)
+            }
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                write!(f, "relmapper file for spc {} db {}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => {
+                write!(f, "pg_twophase/{:08X}", xid)
+            }
+            RelishTag::ControlFile => {
+                write!(f, "control file")
+            }
+            RelishTag::Checkpoint => {
+                write!(f, "checkpoint")
+            }
+        }
+    }
+}
+
+///
+/// Non-relation transaction status files (clog (a.k.a. pg_xact) and
+/// pg_multixact) in Postgres are handled by SLRU (Simple LRU) buffer,
+/// hence the name.
+///
+/// These files are global for a postgres instance.
+///
+/// These files are divided into segments, which are divided into
+/// pages of the same BLCKSZ as used for relation files.
+///
+#[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub enum SlruKind {
+    Clog,
+    MultiXactMembers,
+    MultiXactOffsets,
+}
+
+impl SlruKind {
+    pub fn to_str(&self) -> &'static str {
+        match self {
+            Self::Clog => "pg_xact",
+            Self::MultiXactMembers => "pg_multixact/members",
+            Self::MultiXactOffsets => "pg_multixact/offsets",
+        }
+    }
+}

pageserver/src/reltag.rs

@@ -1,103 +0,0 @@
-use serde::{Deserialize, Serialize};
-use std::cmp::Ordering;
-use std::fmt;
-
-use postgres_ffi::relfile_utils::forknumber_to_name;
-use postgres_ffi::Oid;
-
-///
-/// Relation data file segment id throughout the Postgres cluster.
-///
-/// Every data file in Postgres is uniquely identified by 4 numbers:
-/// - relation id / node (`relnode`)
-/// - database id (`dbnode`)
-/// - tablespace id (`spcnode`), in short this is a unique id of a separate
-///   directory to store data files.
-/// - forknumber (`forknum`) is used to split different kinds of data of the same relation
-///   between some set of files (`relnode`, `relnode_fsm`, `relnode_vm`).
-///
-/// In native Postgres code `RelFileNode` structure and individual `ForkNumber` value
-/// are used for the same purpose.
-/// [See more related comments here](https:///github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/relfilenode.h#L57).
-///
-// FIXME: should move 'forknum' as last field to keep this consistent with Postgres.
-// Then we could replace the custo Ord and PartialOrd implementations below with
-// deriving them.
-#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize)]
-pub struct RelTag {
-    pub forknum: u8,
-    pub spcnode: Oid,
-    pub dbnode: Oid,
-    pub relnode: Oid,
-}
-
-impl PartialOrd for RelTag {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Ord for RelTag {
-    fn cmp(&self, other: &Self) -> Ordering {
-        let mut cmp = self.spcnode.cmp(&other.spcnode);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.dbnode.cmp(&other.dbnode);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.relnode.cmp(&other.relnode);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.forknum.cmp(&other.forknum);
-
-        cmp
-    }
-}
-
-/// Display RelTag in the same format that's used in most PostgreSQL debug messages:
-///
-/// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
-///
-impl fmt::Display for RelTag {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        if let Some(forkname) = forknumber_to_name(self.forknum) {
-            write!(
-                f,
-                "{}/{}/{}_{}",
-                self.spcnode, self.dbnode, self.relnode, forkname
-            )
-        } else {
-            write!(f, "{}/{}/{}", self.spcnode, self.dbnode, self.relnode)
-        }
-    }
-}
-
-///
-/// Non-relation transaction status files (clog (a.k.a. pg_xact) and
-/// pg_multixact) in Postgres are handled by SLRU (Simple LRU) buffer,
-/// hence the name.
-///
-/// These files are global for a postgres instance.
-///
-/// These files are divided into segments, which are divided into
-/// pages of the same BLCKSZ as used for relation files.
-///
-#[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
-pub enum SlruKind {
-    Clog,
-    MultiXactMembers,
-    MultiXactOffsets,
-}
-
-impl SlruKind {
-    pub fn to_str(&self) -> &'static str {
-        match self {
-            Self::Clog => "pg_xact",
-            Self::MultiXactMembers => "pg_multixact/members",
-            Self::MultiXactOffsets => "pg_multixact/offsets",
-        }
-    }
-}

pageserver/src/remote_storage.rs

@@ -0,0 +1,359 @@
+//! A set of generic storage abstractions for the page server to use when backing up and restoring its state from the external storage.
+//! This particular module serves as a public API border between pageserver and the internal storage machinery.
+//! No other modules from this tree are supposed to be used directly by the external code.
+//!
+//! There are a few components the storage machinery consists of:
+//! * [`RemoteStorage`] trait a CRUD-like generic abstraction to use for adapting external storages with a few implementations:
+//!     * [`local_fs`] allows to use local file system as an external storage
+//!     * [`rust_s3`] uses AWS S3 bucket as an external storage
+//!
+//! * synchronization logic at [`storage_sync`] module that keeps pageserver state (both runtime one and the workdir files) and storage state in sync.
+//! Synchronization internals are split into submodules
+//!     * [`storage_sync::compression`] for a custom remote storage format used to store timeline files in archives
+//!     * [`storage_sync::index`] to keep track of remote tenant files, the metadata and their mappings to local files
+//!     * [`storage_sync::upload`] and [`storage_sync::download`] to manage archive creation and upload; download and extraction, respectively
+//!
+//! * public API via to interact with the external world:
+//!     * [`start_local_timeline_sync`] to launch a background async loop to handle the synchronization
+//!     * [`schedule_timeline_checkpoint_upload`] and [`schedule_timeline_download`] to enqueue a new upload and download tasks,
+//!       to be processed by the async loop
+//!
+//! Here's a schematic overview of all interactions backup and the rest of the pageserver perform:
+//!
+//! +------------------------+                                    +--------->-------+
+//! |                        |  - - - (init async loop) - - - ->  |                 |
+//! |                        |                                    |                 |
+//! |                        |  ------------------------------->  |      async      |
+//! |       pageserver       |    (enqueue timeline sync task)    | upload/download |
+//! |                        |                                    |      loop       |
+//! |                        |  <-------------------------------  |                 |
+//! |                        |  (apply new timeline sync states)  |                 |
+//! +------------------------+                                    +---------<-------+
+//!                                                                         |
+//!                                                                         |
+//!                                          CRUD layer file operations     |
+//!                                     (upload/download/delete/list, etc.) |
+//!                                                                         V
+//!                                                            +------------------------+
+//!                                                            |                        |
+//!                                                            | [`RemoteStorage`] impl |
+//!                                                            |                        |
+//!                                                            | pageserver assumes it  |
+//!                                                            | owns exclusive write   |
+//!                                                            | access to this storage |
+//!                                                            +------------------------+
+//!
+//! First, during startup, the pageserver inits the storage sync thread with the async loop, or leaves the loop uninitialised, if configured so.
+//! The loop inits the storage connection and checks the remote files stored.
+//! This is done once at startup only, relying on the fact that pageserver uses the storage alone (ergo, nobody else uploads the files to the storage but this server).
+//! Based on the remote storage data, the sync logic immediately schedules sync tasks for local timelines and reports about remote only timelines to pageserver, so it can
+//! query their downloads later if they are accessed.
+//!
+//! Some time later, during pageserver checkpoints, in-memory data is flushed onto disk along with its metadata.
+//! If the storage sync loop was successfully started before, pageserver schedules the new checkpoint file uploads after every checkpoint.
+//! The checkpoint uploads are disabled, if no remote storage configuration is provided (no sync loop is started this way either).
+//! See [`crate::layered_repository`] for the upload calls and the adjacent logic.
+//!
+//! Synchronization logic is able to communicate back with updated timeline sync states, [`TimelineSyncState`],
+//! submitted via [`crate::tenant_mgr::set_timeline_states`] function. Tenant manager applies corresponding timeline updates in pageserver's in-memory state.
+//! Such submissions happen in two cases:
+//! * once after the sync loop startup, to signal pageserver which timelines will be synchronized in the near future
+//! * after every loop step, in case a timeline needs to be reloaded or evicted from pageserver's memory
+//!
+//! When the pageserver terminates, the upload loop finishes a current sync task (if any) and exits.
+//!
+//! The storage logic considers `image` as a set of local files, fully representing a certain timeline at given moment (identified with `disk_consistent_lsn`).
+//! Timeline can change its state, by adding more files on disk and advancing its `disk_consistent_lsn`: this happens after pageserver checkpointing and is followed
+//! by the storage upload, if enabled.
+//! Yet timeline cannot alter already existing files, and normally cannot remote those too: only a GC process is capable of removing unused files.
+//! This way, remote storage synchronization relies on the fact that every checkpoint is incremental and local files are "immutable":
+//! * when a certain checkpoint gets uploaded, the sync loop remembers the fact, preventing further reuploads of the same state
+//! * no files are deleted from either local or remote storage, only the missing ones locally/remotely get downloaded/uploaded, local metadata file will be overwritten
+//! when the newer image is downloaded
+//!
+//! To optimize S3 storage (and access), the sync loop compresses the checkpoint files before placing them to S3, and uncompresses them back, keeping track of timeline files and metadata.
+//! Also, the remote file list is queried once only, at startup, to avoid possible extra costs and latency issues.
+//!
+//! NOTES:
+//! * pageserver assumes it has exclusive write access to the remote storage. If supported, the way multiple pageservers can be separated in the same storage
+//! (i.e. using different directories in the local filesystem external storage), but totally up to the storage implementation and not covered with the trait API.
+//!
+//! * the sync tasks may not processed immediately after the submission: if they error and get re-enqueued, their execution might be backed off to ensure error cap is not exceeded too fast.
+//! The sync queue processing also happens in batches, so the sync tasks can wait in the queue for some time.
+
+mod local_fs;
+mod rust_s3;
+mod storage_sync;
+
+use std::{
+    collections::HashMap,
+    ffi, fs,
+    path::{Path, PathBuf},
+    sync::Arc,
+};
+
+use anyhow::{bail, Context};
+use tokio::{io, sync::RwLock};
+use tracing::{error, info};
+use zenith_utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId};
+
+pub use self::storage_sync::index::{RemoteTimelineIndex, TimelineIndexEntry};
+pub use self::storage_sync::{schedule_timeline_checkpoint_upload, schedule_timeline_download};
+use self::{local_fs::LocalFs, rust_s3::S3};
+use crate::{
+    config::{PageServerConf, RemoteStorageKind},
+    layered_repository::metadata::{TimelineMetadata, METADATA_FILE_NAME},
+};
+
+pub use storage_sync::compression;
+
+#[derive(Clone, Copy, Debug)]
+pub enum LocalTimelineInitStatus {
+    LocallyComplete,
+    NeedsSync,
+}
+
+type LocalTimelineInitStatuses = HashMap<ZTenantId, HashMap<ZTimelineId, LocalTimelineInitStatus>>;
+
+/// A structure to combine all synchronization data to share with pageserver after a successful sync loop initialization.
+/// Successful initialization includes a case when sync loop is not started, in which case the startup data is returned still,
+/// to simplify the received code.
+pub struct SyncStartupData {
+    pub remote_index: Arc<RwLock<RemoteTimelineIndex>>,
+    pub local_timeline_init_statuses: LocalTimelineInitStatuses,
+}
+
+/// Based on the config, initiates the remote storage connection and starts a separate thread
+/// that ensures that pageserver and the remote storage are in sync with each other.
+/// If no external configuration connection given, no thread or storage initialization is done.
+/// Along with that, scans tenant files local and remote (if the sync gets enabled) to check the initial timeline states.
+pub fn start_local_timeline_sync(
+    config: &'static PageServerConf,
+) -> anyhow::Result<SyncStartupData> {
+    let local_timeline_files = local_tenant_timeline_files(config)
+        .context("Failed to collect local tenant timeline files")?;
+
+    match &config.remote_storage_config {
+        Some(storage_config) => match &storage_config.storage {
+            RemoteStorageKind::LocalFs(root) => {
+                info!("Using fs root '{}' as a remote storage", root.display());
+                storage_sync::spawn_storage_sync_thread(
+                    config,
+                    local_timeline_files,
+                    LocalFs::new(root.clone(), &config.workdir)?,
+                    storage_config.max_concurrent_sync,
+                    storage_config.max_sync_errors,
+                )
+            },
+            RemoteStorageKind::AwsS3(s3_config) => {
+                info!("Using s3 bucket '{}' in region '{}' as a remote storage, prefix in bucket: '{:?}', bucket endpoint: '{:?}'",
+                    s3_config.bucket_name, s3_config.bucket_region, s3_config.prefix_in_bucket, s3_config.endpoint);
+                storage_sync::spawn_storage_sync_thread(
+                    config,
+                    local_timeline_files,
+                    S3::new(s3_config, &config.workdir)?,
+                    storage_config.max_concurrent_sync,
+                    storage_config.max_sync_errors,
+                )
+            },
+        }
+        .context("Failed to spawn the storage sync thread"),
+        None => {
+            info!("No remote storage configured, skipping storage sync, considering all local timelines with correct metadata files enabled");
+            let mut local_timeline_init_statuses = LocalTimelineInitStatuses::new();
+            for (ZTenantTimelineId { tenant_id, timeline_id }, _) in
+                local_timeline_files
+            {
+                local_timeline_init_statuses
+                    .entry(tenant_id)
+                    .or_default()
+                    .insert(timeline_id, LocalTimelineInitStatus::LocallyComplete);
+            }
+            Ok(SyncStartupData {
+                local_timeline_init_statuses,
+                remote_index: Arc::new(RwLock::new(RemoteTimelineIndex::empty())),
+            })
+        }
+    }
+}
+
+fn local_tenant_timeline_files(
+    config: &'static PageServerConf,
+) -> anyhow::Result<HashMap<ZTenantTimelineId, (TimelineMetadata, Vec<PathBuf>)>> {
+    let mut local_tenant_timeline_files = HashMap::new();
+    let tenants_dir = config.tenants_path();
+    for tenants_dir_entry in fs::read_dir(&tenants_dir)
+        .with_context(|| format!("Failed to list tenants dir {}", tenants_dir.display()))?
+    {
+        match &tenants_dir_entry {
+            Ok(tenants_dir_entry) => {
+                match collect_timelines_for_tenant(config, &tenants_dir_entry.path()) {
+                    Ok(collected_files) => {
+                        local_tenant_timeline_files.extend(collected_files.into_iter())
+                    }
+                    Err(e) => error!(
+                        "Failed to collect tenant files from dir '{}' for entry {:?}, reason: {:#}",
+                        tenants_dir.display(),
+                        tenants_dir_entry,
+                        e
+                    ),
+                }
+            }
+            Err(e) => error!(
+                "Failed to list tenants dir entry {:?} in directory {}, reason: {:?}",
+                tenants_dir_entry,
+                tenants_dir.display(),
+                e
+            ),
+        }
+    }
+
+    Ok(local_tenant_timeline_files)
+}
+
+fn collect_timelines_for_tenant(
+    config: &'static PageServerConf,
+    tenant_path: &Path,
+) -> anyhow::Result<HashMap<ZTenantTimelineId, (TimelineMetadata, Vec<PathBuf>)>> {
+    let mut timelines: HashMap<ZTenantTimelineId, (TimelineMetadata, Vec<PathBuf>)> =
+        HashMap::new();
+    let tenant_id = tenant_path
+        .file_name()
+        .and_then(ffi::OsStr::to_str)
+        .unwrap_or_default()
+        .parse::<ZTenantId>()
+        .context("Could not parse tenant id out of the tenant dir name")?;
+    let timelines_dir = config.timelines_path(&tenant_id);
+
+    for timelines_dir_entry in fs::read_dir(&timelines_dir).with_context(|| {
+        format!(
+            "Failed to list timelines dir entry for tenant {}",
+            tenant_id
+        )
+    })? {
+        match timelines_dir_entry {
+            Ok(timelines_dir_entry) => {
+                let timeline_path = timelines_dir_entry.path();
+                match collect_timeline_files(&timeline_path) {
+                    Ok((timeline_id, metadata, timeline_files)) => {
+                        timelines.insert(
+                            ZTenantTimelineId {
+                                tenant_id,
+                                timeline_id,
+                            },
+                            (metadata, timeline_files),
+                        );
+                    }
+                    Err(e) => error!(
+                        "Failed to process timeline dir contents at '{}', reason: {:?}",
+                        timeline_path.display(),
+                        e
+                    ),
+                }
+            }
+            Err(e) => error!(
+                "Failed to list timelines for entry tenant {}, reason: {:?}",
+                tenant_id, e
+            ),
+        }
+    }
+
+    Ok(timelines)
+}
+
+fn collect_timeline_files(
+    timeline_dir: &Path,
+) -> anyhow::Result<(ZTimelineId, TimelineMetadata, Vec<PathBuf>)> {
+    let mut timeline_files = Vec::new();
+    let mut timeline_metadata_path = None;
+
+    let timeline_id = timeline_dir
+        .file_name()
+        .and_then(ffi::OsStr::to_str)
+        .unwrap_or_default()
+        .parse::<ZTimelineId>()
+        .context("Could not parse timeline id out of the timeline dir name")?;
+    let timeline_dir_entries =
+        fs::read_dir(&timeline_dir).context("Failed to list timeline dir contents")?;
+    for entry in timeline_dir_entries {
+        let entry_path = entry.context("Failed to list timeline dir entry")?.path();
+        if entry_path.is_file() {
+            if entry_path.file_name().and_then(ffi::OsStr::to_str) == Some(METADATA_FILE_NAME) {
+                timeline_metadata_path = Some(entry_path);
+            } else {
+                timeline_files.push(entry_path);
+            }
+        }
+    }
+
+    let timeline_metadata_path = match timeline_metadata_path {
+        Some(path) => path,
+        None => bail!("No metadata file found in the timeline directory"),
+    };
+    let metadata = TimelineMetadata::from_bytes(
+        &fs::read(&timeline_metadata_path).context("Failed to read timeline metadata file")?,
+    )
+    .context("Failed to parse timeline metadata file bytes")?;
+
+    Ok((timeline_id, metadata, timeline_files))
+}
+
+/// Storage (potentially remote) API to manage its state.
+/// This storage tries to be unaware of any layered repository context,
+/// providing basic CRUD operations for storage files.
+#[async_trait::async_trait]
+trait RemoteStorage: Send + Sync {
+    /// A way to uniquely reference a file in the remote storage.
+    type StoragePath;
+
+    /// Attempts to derive the storage path out of the local path, if the latter is correct.
+    fn storage_path(&self, local_path: &Path) -> anyhow::Result<Self::StoragePath>;
+
+    /// Gets the download path of the given storage file.
+    fn local_path(&self, storage_path: &Self::StoragePath) -> anyhow::Result<PathBuf>;
+
+    /// Lists all items the storage has right now.
+    async fn list(&self) -> anyhow::Result<Vec<Self::StoragePath>>;
+
+    /// Streams the local file contents into remote into the remote storage entry.
+    async fn upload(
+        &self,
+        from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
+        to: &Self::StoragePath,
+    ) -> anyhow::Result<()>;
+
+    /// Streams the remote storage entry contents into the buffered writer given, returns the filled writer.
+    async fn download(
+        &self,
+        from: &Self::StoragePath,
+        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
+    ) -> anyhow::Result<()>;
+
+    /// Streams a given byte range of the remote storage entry contents into the buffered writer given, returns the filled writer.
+    async fn download_range(
+        &self,
+        from: &Self::StoragePath,
+        start_inclusive: u64,
+        end_exclusive: Option<u64>,
+        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
+    ) -> anyhow::Result<()>;
+
+    async fn delete(&self, path: &Self::StoragePath) -> anyhow::Result<()>;
+}
+
+fn strip_path_prefix<'a>(prefix: &'a Path, path: &'a Path) -> anyhow::Result<&'a Path> {
+    if prefix == path {
+        anyhow::bail!(
+            "Prefix and the path are equal, cannot strip: '{}'",
+            prefix.display()
+        )
+    } else {
+        path.strip_prefix(prefix).with_context(|| {
+            format!(
+                "Path '{}' is not prefixed with '{}'",
+                path.display(),
+                prefix.display(),
+            )
+        })
+    }
+}

pageserver/src/remote_storage/README.md

@@ -0,0 +1,64 @@
+# Non-implementation details
+
+This document describes the current state of the backup system in pageserver, existing limitations and concerns, why some things are done the way they are the future development plans.
+Detailed description on how the synchronization works and how it fits into the rest of the pageserver can be found in the [storage module](./../remote_storage.rs) and its submodules.
+Ideally, this document should disappear after current implementation concerns are mitigated, with the remaining useful knowledge bits moved into rustdocs.
+
+## Approach
+
+Backup functionality is a new component, appeared way after the core DB functionality was implemented.
+Pageserver layer functionality is also quite volatile at the moment, there's a risk its local file management changes over time.
+
+To avoid adding more chaos into that, backup functionality is currently designed as a relatively standalone component, with the majority of its logic placed in a standalone async loop.
+This way, the backups are managed in background, not affecting directly other pageserver parts: this way the backup and restoration process may lag behind, but eventually keep up with the reality. To track that, a set of prometheus metrics is exposed from pageserver.
+
+## What's done
+
+Current implementation
+* provides remote storage wrappers for AWS S3 and local FS
+* synchronizes the differences with local timelines and remote states as fast as possible
+* uploads new relishes, frozen by pageserver checkpoint thread
+* downloads and registers timelines, found on the remote storage, but missing locally, if those are requested somehow via pageserver (e.g. http api, gc)
+* uses compression when deals with files, for better S3 usage
+* maintains an index of what's stored remotely
+* evicts failing tasks and stops the corresponding timelines
+
+The tasks are delayed with every retry and the retries are capped, to avoid poisonous tasks.
+After any task eviction, or any error at startup checks (e.g. obviously different and wrong local and remote states fot the same timeline),
+the timeline has to be stopped from submitting further checkpoint upload tasks, which is done along the corresponding timeline status change.
+
+No good optimisations or performance testing is done, the feature is disabled by default and gets polished over time.
+It's planned to deal with all questions that are currently on and prepare the feature to be enabled by default in cloud environments.
+
+### Peculiarities
+
+As mentioned, the backup component is rather new and under development currently, so not all things are done properly from the start.
+Here's the list of known compromises with comments:
+
+* Remote storage file model is currently a custom archive format, that's not possible to deserialize without a particular Rust code of ours (including `serde`).
+We also don't optimize the archivation and pack every timeline checkpoint separately, so the resulting blob's size that gets on S3 could be arbitrary.
+But, it's a single blob, which is way better than storing ~780 small files separately.
+
+* Archive index restoration requires reading every blob's head.
+This could be avoided by a background thread/future storing the serialized index in the remote storage.
+
+* no proper file comparison
+
+No file checksum assertion is done currently, but should be (AWS S3 returns file checksums during the `list` operation)
+
+* sad rust-s3 api
+
+rust-s3 is not very pleasant to use:
+1. it returns `anyhow::Result` and it's hard to distinguish "missing file" cases from "no connection" one, for instance
+2. at least one function it its API that we need (`get_object_stream`) has `async` keyword and blocks (!), see details [here](https://github.com/zenithdb/zenith/pull/752#discussion_r728373091)
+3. it's a prerelease library with unclear maintenance status
+4. noisy on debug level
+
+But it's already used in the project, so for now it's reused to avoid bloating the dependency tree.
+Based on previous evaluation, even `rusoto-s3` could be a better choice over this library, but needs further benchmarking.
+
+
+* gc is ignored
+
+So far, we don't adjust the remote storage based on GC thread loop results, only checkpointer loop affects the remote storage.
+Index module could be used as a base to implement a deferred GC mechanism, a "defragmentation" that repacks archives into new ones after GC is done removing the files from the archives.

pageserver/src/remote_storage/local_fs.rs

@@ -1,10 +1,11 @@
 //! Local filesystem acting as a remote storage.
-//! Multiple API users can use the same "storage" of this kind by using different storage roots.
+//! Multiple pageservers can use the same "storage" of this kind by using different storage roots.
 //!
-//! This storage used in tests, but can also be used in cases when a certain persistent
+//! This storage used in pageserver tests, but can also be used in cases when a certain persistent
 //! volume is mounted to the local FS.
 
 use std::{
+    ffi::OsString,
     future::Future,
     path::{Path, PathBuf},
     pin::Pin,
@@ -17,18 +18,16 @@ use tokio::{
 };
 use tracing::*;
 
-use crate::path_with_suffix_extension;
-
-use super::{strip_path_prefix, RemoteStorage, StorageMetadata};
+use super::{strip_path_prefix, RemoteStorage};
 
 pub struct LocalFs {
-    working_directory: PathBuf,
-    storage_root: PathBuf,
+    pageserver_workdir: &'static Path,
+    root: PathBuf,
 }
 
 impl LocalFs {
     /// Attempts to create local FS storage, along with its root directory.
-    pub fn new(root: PathBuf, working_directory: PathBuf) -> anyhow::Result<Self> {
+    pub fn new(root: PathBuf, pageserver_workdir: &'static Path) -> anyhow::Result<Self> {
         if !root.exists() {
             std::fs::create_dir_all(&root).with_context(|| {
                 format!(
@@ -38,15 +37,15 @@ impl LocalFs {
             })?;
         }
         Ok(Self {
-            working_directory,
-            storage_root: root,
+            pageserver_workdir,
+            root,
         })
     }
 
     fn resolve_in_storage(&self, path: &Path) -> anyhow::Result<PathBuf> {
         if path.is_relative() {
-            Ok(self.storage_root.join(path))
-        } else if path.starts_with(&self.storage_root) {
+            Ok(self.root.join(path))
+        } else if path.starts_with(&self.root) {
             Ok(path.to_path_buf())
         } else {
             bail!(
@@ -55,68 +54,46 @@ impl LocalFs {
             )
         }
     }
-
-    async fn read_storage_metadata(
-        &self,
-        file_path: &Path,
-    ) -> anyhow::Result<Option<StorageMetadata>> {
-        let metadata_path = storage_metadata_path(file_path);
-        if metadata_path.exists() && metadata_path.is_file() {
-            let metadata_string = fs::read_to_string(&metadata_path).await.with_context(|| {
-                format!(
-                    "Failed to read metadata from the local storage at '{}'",
-                    metadata_path.display()
-                )
-            })?;
-
-            serde_json::from_str(&metadata_string)
-                .with_context(|| {
-                    format!(
-                        "Failed to deserialize metadata from the local storage at '{}'",
-                        metadata_path.display()
-                    )
-                })
-                .map(|metadata| Some(StorageMetadata(metadata)))
-        } else {
-            Ok(None)
-        }
-    }
 }
 
 #[async_trait::async_trait]
 impl RemoteStorage for LocalFs {
-    type RemoteObjectId = PathBuf;
+    type StoragePath = PathBuf;
 
-    fn remote_object_id(&self, local_path: &Path) -> anyhow::Result<Self::RemoteObjectId> {
-        Ok(self.storage_root.join(
-            strip_path_prefix(&self.working_directory, local_path)
+    fn storage_path(&self, local_path: &Path) -> anyhow::Result<Self::StoragePath> {
+        Ok(self.root.join(
+            strip_path_prefix(self.pageserver_workdir, local_path)
                 .context("local path does not belong to this storage")?,
         ))
     }
 
-    fn local_path(&self, storage_path: &Self::RemoteObjectId) -> anyhow::Result<PathBuf> {
-        let relative_path = strip_path_prefix(&self.storage_root, storage_path)
+    fn local_path(&self, storage_path: &Self::StoragePath) -> anyhow::Result<PathBuf> {
+        let relative_path = strip_path_prefix(&self.root, storage_path)
             .context("local path does not belong to this storage")?;
-        Ok(self.working_directory.join(relative_path))
+        Ok(self.pageserver_workdir.join(relative_path))
     }
 
-    async fn list(&self) -> anyhow::Result<Vec<Self::RemoteObjectId>> {
-        get_all_files(&self.storage_root).await
+    async fn list(&self) -> anyhow::Result<Vec<Self::StoragePath>> {
+        get_all_files(&self.root).await
     }
 
     async fn upload(
         &self,
-        from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
-        from_size_bytes: usize,
-        to: &Self::RemoteObjectId,
-        metadata: Option<StorageMetadata>,
+        mut from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
+        to: &Self::StoragePath,
     ) -> anyhow::Result<()> {
         let target_file_path = self.resolve_in_storage(to)?;
         create_target_directory(&target_file_path).await?;
         // We need this dance with sort of durable rename (without fsyncs)
         // to prevent partial uploads. This was really hit when pageserver shutdown
         // cancelled the upload and partial file was left on the fs
-        let temp_file_path = path_with_suffix_extension(&target_file_path, "temp");
+        let mut temp_extension = target_file_path
+            .extension()
+            .unwrap_or_default()
+            .to_os_string();
+
+        temp_extension.push(OsString::from(".temp"));
+        let temp_file_path = target_file_path.with_extension(temp_extension);
         let mut destination = io::BufWriter::new(
             fs::OpenOptions::new()
                 .write(true)
@@ -131,11 +108,7 @@ impl RemoteStorage for LocalFs {
                 })?,
         );
 
-        let from_size_bytes = from_size_bytes as u64;
-        // Require to read 1 byte more than the expected to check later, that the stream and its size match.
-        let mut buffer_to_read = from.take(from_size_bytes + 1);
-
-        let bytes_read = io::copy(&mut buffer_to_read, &mut destination)
+        io::copy(&mut from, &mut destination)
             .await
             .with_context(|| {
                 format!(
@@ -144,19 +117,6 @@ impl RemoteStorage for LocalFs {
                 )
             })?;
 
-        ensure!(
-            bytes_read == from_size_bytes,
-            "Provided stream has actual size {} fthat is smaller than the given stream size {}",
-            bytes_read,
-            from_size_bytes
-        );
-
-        ensure!(
-            buffer_to_read.read(&mut [0]).await? == 0,
-            "Provided stream has bigger size than the given stream size {}",
-            from_size_bytes
-        );
-
         destination.flush().await.with_context(|| {
             format!(
                 "Failed to upload (flush temp) file to the local storage at '{}'",
@@ -172,31 +132,14 @@ impl RemoteStorage for LocalFs {
                     target_file_path.display()
                 )
             })?;
-
-        if let Some(storage_metadata) = metadata {
-            let storage_metadata_path = storage_metadata_path(&target_file_path);
-            fs::write(
-                &storage_metadata_path,
-                serde_json::to_string(&storage_metadata.0)
-                    .context("Failed to serialize storage metadata as json")?,
-            )
-            .await
-            .with_context(|| {
-                format!(
-                    "Failed to write metadata to the local storage at '{}'",
-                    storage_metadata_path.display()
-                )
-            })?;
-        }
-
         Ok(())
     }
 
     async fn download(
         &self,
-        from: &Self::RemoteObjectId,
+        from: &Self::StoragePath,
         to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>> {
+    ) -> anyhow::Result<()> {
         let file_path = self.resolve_in_storage(from)?;
 
         if file_path.exists() && file_path.is_file() {
@@ -219,8 +162,7 @@ impl RemoteStorage for LocalFs {
                 )
             })?;
             source.flush().await?;
-
-            self.read_storage_metadata(&file_path).await
+            Ok(())
         } else {
             bail!(
                 "File '{}' either does not exist or is not a file",
@@ -229,13 +171,13 @@ impl RemoteStorage for LocalFs {
         }
     }
 
-    async fn download_byte_range(
+    async fn download_range(
         &self,
-        from: &Self::RemoteObjectId,
+        from: &Self::StoragePath,
         start_inclusive: u64,
         end_exclusive: Option<u64>,
         to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>> {
+    ) -> anyhow::Result<()> {
         if let Some(end_exclusive) = end_exclusive {
             ensure!(
                 end_exclusive > start_inclusive,
@@ -244,7 +186,7 @@ impl RemoteStorage for LocalFs {
                 end_exclusive
             );
             if start_inclusive == end_exclusive.saturating_sub(1) {
-                return Ok(None);
+                return Ok(());
             }
         }
         let file_path = self.resolve_in_storage(from)?;
@@ -278,8 +220,7 @@ impl RemoteStorage for LocalFs {
                     file_path.display()
                 )
             })?;
-
-            self.read_storage_metadata(&file_path).await
+            Ok(())
         } else {
             bail!(
                 "File '{}' either does not exist or is not a file",
@@ -288,7 +229,7 @@ impl RemoteStorage for LocalFs {
         }
     }
 
-    async fn delete(&self, path: &Self::RemoteObjectId) -> anyhow::Result<()> {
+    async fn delete(&self, path: &Self::StoragePath) -> anyhow::Result<()> {
         let file_path = self.resolve_in_storage(path)?;
         if file_path.exists() && file_path.is_file() {
             Ok(fs::remove_file(file_path).await?)
@@ -301,10 +242,6 @@ impl RemoteStorage for LocalFs {
     }
 }
 
-fn storage_metadata_path(original_path: &Path) -> PathBuf {
-    path_with_suffix_extension(original_path, "metadata")
-}
-
 fn get_all_files<'a, P>(
     directory_path: P,
 ) -> Pin<Box<dyn Future<Output = anyhow::Result<Vec<PathBuf>>> + Send + Sync + 'a>>
@@ -354,30 +291,29 @@ async fn create_target_directory(target_file_path: &Path) -> anyhow::Result<()>
 
 #[cfg(test)]
 mod pure_tests {
-    use tempfile::tempdir;
+    use crate::{
+        layered_repository::metadata::METADATA_FILE_NAME,
+        repository::repo_harness::{RepoHarness, TIMELINE_ID},
+    };
 
     use super::*;
 
     #[test]
     fn storage_path_positive() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
+        let repo_harness = RepoHarness::create("storage_path_positive")?;
         let storage_root = PathBuf::from("somewhere").join("else");
         let storage = LocalFs {
-            working_directory: workdir.clone(),
-            storage_root: storage_root.clone(),
+            pageserver_workdir: &repo_harness.conf.workdir,
+            root: storage_root.clone(),
         };
 
-        let local_path = workdir
-            .join("timelines")
-            .join("some_timeline")
-            .join("file_name");
-        let expected_path = storage_root.join(local_path.strip_prefix(&workdir)?);
+        let local_path = repo_harness.timeline_path(&TIMELINE_ID).join("file_name");
+        let expected_path = storage_root.join(local_path.strip_prefix(&repo_harness.conf.workdir)?);
 
         assert_eq!(
             expected_path,
-            storage.remote_object_id(&local_path).expect("Matching path should map to storage path normally"),
-            "File paths from workdir should be stored in local fs storage with the same path they have relative to the workdir"
+            storage.storage_path(&local_path).expect("Matching path should map to storage path normally"),
+            "File paths from pageserver workdir should be stored in local fs storage with the same path they have relative to the workdir"
         );
 
         Ok(())
@@ -387,7 +323,7 @@ mod pure_tests {
     fn storage_path_negatives() -> anyhow::Result<()> {
         #[track_caller]
         fn storage_path_error(storage: &LocalFs, mismatching_path: &Path) -> String {
-            match storage.remote_object_id(mismatching_path) {
+            match storage.storage_path(mismatching_path) {
                 Ok(wrong_path) => panic!(
                     "Expected path '{}' to error, but got storage path: {:?}",
                     mismatching_path.display(),
@@ -397,16 +333,16 @@ mod pure_tests {
             }
         }
 
-        let workdir = tempdir()?.path().to_owned();
+        let repo_harness = RepoHarness::create("storage_path_negatives")?;
         let storage_root = PathBuf::from("somewhere").join("else");
         let storage = LocalFs {
-            working_directory: workdir.clone(),
-            storage_root,
+            pageserver_workdir: &repo_harness.conf.workdir,
+            root: storage_root,
         };
 
-        let error_string = storage_path_error(&storage, &workdir);
+        let error_string = storage_path_error(&storage, &repo_harness.conf.workdir);
         assert!(error_string.contains("does not belong to this storage"));
-        assert!(error_string.contains(workdir.to_str().unwrap()));
+        assert!(error_string.contains(repo_harness.conf.workdir.to_str().unwrap()));
 
         let mismatching_path_str = "/something/else";
         let error_message = storage_path_error(&storage, Path::new(mismatching_path_str));
@@ -415,7 +351,7 @@ mod pure_tests {
             "Error should mention wrong path"
         );
         assert!(
-            error_message.contains(workdir.to_str().unwrap()),
+            error_message.contains(repo_harness.conf.workdir.to_str().unwrap()),
             "Error should mention server workdir"
         );
         assert!(error_message.contains("does not belong to this storage"));
@@ -425,28 +361,29 @@ mod pure_tests {
 
     #[test]
     fn local_path_positive() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
+        let repo_harness = RepoHarness::create("local_path_positive")?;
         let storage_root = PathBuf::from("somewhere").join("else");
         let storage = LocalFs {
-            working_directory: workdir.clone(),
-            storage_root: storage_root.clone(),
+            pageserver_workdir: &repo_harness.conf.workdir,
+            root: storage_root.clone(),
         };
 
         let name = "not a metadata";
-        let local_path = workdir.join("timelines").join("some_timeline").join(name);
+        let local_path = repo_harness.timeline_path(&TIMELINE_ID).join(name);
         assert_eq!(
             local_path,
             storage
-                .local_path(&storage_root.join(local_path.strip_prefix(&workdir)?))
+                .local_path(
+                    &storage_root.join(local_path.strip_prefix(&repo_harness.conf.workdir)?)
+                )
                 .expect("For a valid input, valid local path should be parsed"),
             "Should be able to parse metadata out of the correctly named remote delta file"
         );
 
-        let local_metadata_path = workdir
-            .join("timelines")
-            .join("some_timeline")
-            .join("metadata");
-        let remote_metadata_path = storage.remote_object_id(&local_metadata_path)?;
+        let local_metadata_path = repo_harness
+            .timeline_path(&TIMELINE_ID)
+            .join(METADATA_FILE_NAME);
+        let remote_metadata_path = storage.storage_path(&local_metadata_path)?;
         assert_eq!(
             local_metadata_path,
             storage
@@ -472,10 +409,11 @@ mod pure_tests {
             }
         }
 
+        let repo_harness = RepoHarness::create("local_path_negatives")?;
         let storage_root = PathBuf::from("somewhere").join("else");
         let storage = LocalFs {
-            working_directory: tempdir()?.path().to_owned(),
-            storage_root,
+            pageserver_workdir: &repo_harness.conf.workdir,
+            root: storage_root,
         };
 
         let totally_wrong_path = "wrong_wrong_wrong";
@@ -487,19 +425,16 @@ mod pure_tests {
 
     #[test]
     fn download_destination_matches_original_path() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-        let original_path = workdir
-            .join("timelines")
-            .join("some_timeline")
-            .join("some name");
+        let repo_harness = RepoHarness::create("download_destination_matches_original_path")?;
+        let original_path = repo_harness.timeline_path(&TIMELINE_ID).join("some name");
 
         let storage_root = PathBuf::from("somewhere").join("else");
         let dummy_storage = LocalFs {
-            working_directory: workdir,
-            storage_root,
+            pageserver_workdir: &repo_harness.conf.workdir,
+            root: storage_root,
         };
 
-        let storage_path = dummy_storage.remote_object_id(&original_path)?;
+        let storage_path = dummy_storage.storage_path(&original_path)?;
         let download_destination = dummy_storage.local_path(&storage_path)?;
 
         assert_eq!(
@@ -514,22 +449,23 @@ mod pure_tests {
 #[cfg(test)]
 mod fs_tests {
     use super::*;
+    use crate::repository::repo_harness::{RepoHarness, TIMELINE_ID};
 
-    use std::{collections::HashMap, io::Write};
+    use std::io::Write;
     use tempfile::tempdir;
 
     #[tokio::test]
     async fn upload_file() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
+        let repo_harness = RepoHarness::create("upload_file")?;
         let storage = create_storage()?;
 
-        let (file, size) = create_file_for_upload(
-            &storage.working_directory.join("whatever"),
+        let source = create_file_for_upload(
+            &storage.pageserver_workdir.join("whatever"),
             "whatever_contents",
         )
         .await?;
         let target_path = PathBuf::from("/").join("somewhere").join("else");
-        match storage.upload(file, size, &target_path, None).await {
+        match storage.upload(source, &target_path).await {
             Ok(()) => panic!("Should not allow storing files with wrong target path"),
             Err(e) => {
                 let message = format!("{:?}", e);
@@ -539,14 +475,14 @@ mod fs_tests {
         }
         assert!(storage.list().await?.is_empty());
 
-        let target_path_1 = upload_dummy_file(&workdir, &storage, "upload_1", None).await?;
+        let target_path_1 = upload_dummy_file(&repo_harness, &storage, "upload_1").await?;
         assert_eq!(
             storage.list().await?,
             vec![target_path_1.clone()],
             "Should list a single file after first upload"
         );
 
-        let target_path_2 = upload_dummy_file(&workdir, &storage, "upload_2", None).await?;
+        let target_path_2 = upload_dummy_file(&repo_harness, &storage, "upload_2").await?;
         assert_eq!(
             list_files_sorted(&storage).await?,
             vec![target_path_1.clone(), target_path_2.clone()],
@@ -557,25 +493,22 @@ mod fs_tests {
     }
 
     fn create_storage() -> anyhow::Result<LocalFs> {
-        LocalFs::new(tempdir()?.path().to_owned(), tempdir()?.path().to_owned())
+        let pageserver_workdir = Box::leak(Box::new(tempdir()?.path().to_owned()));
+        let storage = LocalFs::new(tempdir()?.path().to_owned(), pageserver_workdir)?;
+        Ok(storage)
     }
 
     #[tokio::test]
     async fn download_file() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
+        let repo_harness = RepoHarness::create("download_file")?;
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&workdir, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;
 
         let mut content_bytes = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let metadata = storage.download(&upload_target, &mut content_bytes).await?;
-        assert!(
-            metadata.is_none(),
-            "No metadata should be returned for no metadata upload"
-        );
-
+        storage.download(&upload_target, &mut content_bytes).await?;
         content_bytes.flush().await?;
+
         let contents = String::from_utf8(content_bytes.into_inner().into_inner())?;
         assert_eq!(
             dummy_contents(upload_name),
@@ -597,20 +530,15 @@ mod fs_tests {
 
     #[tokio::test]
     async fn download_file_range_positive() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
+        let repo_harness = RepoHarness::create("download_file_range_positive")?;
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&workdir, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;
 
         let mut full_range_bytes = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let metadata = storage
-            .download_byte_range(&upload_target, 0, None, &mut full_range_bytes)
+        storage
+            .download_range(&upload_target, 0, None, &mut full_range_bytes)
             .await?;
-        assert!(
-            metadata.is_none(),
-            "No metadata should be returned for no metadata upload"
-        );
         full_range_bytes.flush().await?;
         assert_eq!(
             dummy_contents(upload_name),
@@ -620,18 +548,14 @@ mod fs_tests {
 
         let mut zero_range_bytes = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
         let same_byte = 1_000_000_000;
-        let metadata = storage
-            .download_byte_range(
+        storage
+            .download_range(
                 &upload_target,
                 same_byte,
                 Some(same_byte + 1), // exclusive end
                 &mut zero_range_bytes,
             )
             .await?;
-        assert!(
-            metadata.is_none(),
-            "No metadata should be returned for no metadata upload"
-        );
         zero_range_bytes.flush().await?;
         assert!(
             zero_range_bytes.into_inner().into_inner().is_empty(),
@@ -642,19 +566,14 @@ mod fs_tests {
         let (first_part_local, second_part_local) = uploaded_bytes.split_at(3);
 
         let mut first_part_remote = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let metadata = storage
-            .download_byte_range(
+        storage
+            .download_range(
                 &upload_target,
                 0,
                 Some(first_part_local.len() as u64),
                 &mut first_part_remote,
             )
             .await?;
-        assert!(
-            metadata.is_none(),
-            "No metadata should be returned for no metadata upload"
-        );
-
         first_part_remote.flush().await?;
         let first_part_remote = first_part_remote.into_inner().into_inner();
         assert_eq!(
@@ -664,19 +583,14 @@ mod fs_tests {
         );
 
         let mut second_part_remote = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let metadata = storage
-            .download_byte_range(
+        storage
+            .download_range(
                 &upload_target,
                 first_part_local.len() as u64,
                 Some((first_part_local.len() + second_part_local.len()) as u64),
                 &mut second_part_remote,
             )
             .await?;
-        assert!(
-            metadata.is_none(),
-            "No metadata should be returned for no metadata upload"
-        );
-
         second_part_remote.flush().await?;
         let second_part_remote = second_part_remote.into_inner().into_inner();
         assert_eq!(
@@ -690,17 +604,16 @@ mod fs_tests {
 
     #[tokio::test]
     async fn download_file_range_negative() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
+        let repo_harness = RepoHarness::create("download_file_range_negative")?;
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&workdir, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;
 
         let start = 10000;
         let end = 234;
         assert!(start > end, "Should test an incorrect range");
         match storage
-            .download_byte_range(&upload_target, start, Some(end), &mut io::sink())
+            .download_range(&upload_target, start, Some(end), &mut io::sink())
             .await
         {
             Ok(_) => panic!("Should not allow downloading wrong ranges"),
@@ -714,7 +627,7 @@ mod fs_tests {
 
         let non_existing_path = PathBuf::from("somewhere").join("else");
         match storage
-            .download_byte_range(&non_existing_path, 1, Some(3), &mut io::sink())
+            .download_range(&non_existing_path, 1, Some(3), &mut io::sink())
             .await
         {
             Ok(_) => panic!("Should not allow downloading non-existing storage file ranges"),
@@ -729,11 +642,10 @@ mod fs_tests {
 
     #[tokio::test]
     async fn delete_file() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
+        let repo_harness = RepoHarness::create("delete_file")?;
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&workdir, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;
 
         storage.delete(&upload_target).await?;
         assert!(storage.list().await?.is_empty());
@@ -749,85 +661,31 @@ mod fs_tests {
         Ok(())
     }
 
-    #[tokio::test]
-    async fn file_with_metadata() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
-        let storage = create_storage()?;
-        let upload_name = "upload_1";
-        let metadata = StorageMetadata(HashMap::from([
-            ("one".to_string(), "1".to_string()),
-            ("two".to_string(), "2".to_string()),
-        ]));
-        let upload_target =
-            upload_dummy_file(&workdir, &storage, upload_name, Some(metadata.clone())).await?;
-
-        let mut content_bytes = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let full_download_metadata = storage.download(&upload_target, &mut content_bytes).await?;
-
-        content_bytes.flush().await?;
-        let contents = String::from_utf8(content_bytes.into_inner().into_inner())?;
-        assert_eq!(
-            dummy_contents(upload_name),
-            contents,
-            "We should upload and download the same contents"
-        );
-
-        assert_eq!(
-            full_download_metadata.as_ref(),
-            Some(&metadata),
-            "We should get the same metadata back for full download"
-        );
-
-        let uploaded_bytes = dummy_contents(upload_name).into_bytes();
-        let (first_part_local, _) = uploaded_bytes.split_at(3);
-
-        let mut first_part_remote = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let partial_download_metadata = storage
-            .download_byte_range(
-                &upload_target,
-                0,
-                Some(first_part_local.len() as u64),
-                &mut first_part_remote,
-            )
-            .await?;
-        first_part_remote.flush().await?;
-        let first_part_remote = first_part_remote.into_inner().into_inner();
-        assert_eq!(
-            first_part_local,
-            first_part_remote.as_slice(),
-            "First part bytes should be returned when requested"
-        );
-
-        assert_eq!(
-            partial_download_metadata.as_ref(),
-            Some(&metadata),
-            "We should get the same metadata back for partial download"
-        );
-
-        Ok(())
-    }
-
     async fn upload_dummy_file(
-        workdir: &Path,
+        harness: &RepoHarness,
         storage: &LocalFs,
         name: &str,
-        metadata: Option<StorageMetadata>,
     ) -> anyhow::Result<PathBuf> {
-        let timeline_path = workdir.join("timelines").join("some_timeline");
-        let relative_timeline_path = timeline_path.strip_prefix(&workdir)?;
-        let storage_path = storage.storage_root.join(relative_timeline_path).join(name);
-
-        let from_path = storage.working_directory.join(name);
-        let (file, size) = create_file_for_upload(&from_path, &dummy_contents(name)).await?;
-        storage.upload(file, size, &storage_path, metadata).await?;
+        let timeline_path = harness.timeline_path(&TIMELINE_ID);
+        let relative_timeline_path = timeline_path.strip_prefix(&harness.conf.workdir)?;
+        let storage_path = storage.root.join(relative_timeline_path).join(name);
+        storage
+            .upload(
+                create_file_for_upload(
+                    &storage.pageserver_workdir.join(name),
+                    &dummy_contents(name),
+                )
+                .await?,
+                &storage_path,
+            )
+            .await?;
         Ok(storage_path)
     }
 
     async fn create_file_for_upload(
         path: &Path,
         contents: &str,
-    ) -> anyhow::Result<(io::BufReader<fs::File>, usize)> {
+    ) -> anyhow::Result<io::BufReader<fs::File>> {
         std::fs::create_dir_all(path.parent().unwrap())?;
         let mut file_for_writing = std::fs::OpenOptions::new()
             .write(true)
@@ -835,10 +693,8 @@ mod fs_tests {
             .open(path)?;
         write!(file_for_writing, "{}", contents)?;
         drop(file_for_writing);
-        let file_size = path.metadata()?.len() as usize;
-        Ok((
-            io::BufReader::new(fs::OpenOptions::new().read(true).open(&path).await?),
-            file_size,
+        Ok(io::BufReader::new(
+            fs::OpenOptions::new().read(true).open(&path).await?,
         ))
     }
 

pageserver/src/remote_storage/rust_s3.rs

@@ -0,0 +1,438 @@
+//! AWS S3 storage wrapper around `rust_s3` library.
+//!
+//! Respects `prefix_in_bucket` property from [`S3Config`],
+//! allowing multiple pageservers to independently work with the same S3 bucket, if
+//! their bucket prefixes are both specified and different.
+
+use std::path::{Path, PathBuf};
+
+use anyhow::Context;
+use s3::{bucket::Bucket, creds::Credentials, region::Region};
+use tokio::io::{self, AsyncWriteExt};
+use tracing::debug;
+
+use crate::{
+    config::S3Config,
+    remote_storage::{strip_path_prefix, RemoteStorage},
+};
+
+const S3_FILE_SEPARATOR: char = '/';
+
+#[derive(Debug, Eq, PartialEq)]
+pub struct S3ObjectKey(String);
+
+impl S3ObjectKey {
+    fn key(&self) -> &str {
+        &self.0
+    }
+
+    fn download_destination(
+        &self,
+        pageserver_workdir: &Path,
+        prefix_to_strip: Option<&str>,
+    ) -> PathBuf {
+        let path_without_prefix = match prefix_to_strip {
+            Some(prefix) => self.0.strip_prefix(prefix).unwrap_or_else(|| {
+                panic!(
+                    "Could not strip prefix '{}' from S3 object key '{}'",
+                    prefix, self.0
+                )
+            }),
+            None => &self.0,
+        };
+
+        pageserver_workdir.join(
+            path_without_prefix
+                .split(S3_FILE_SEPARATOR)
+                .collect::<PathBuf>(),
+        )
+    }
+}
+
+/// AWS S3 storage.
+pub struct S3 {
+    pageserver_workdir: &'static Path,
+    bucket: Bucket,
+    prefix_in_bucket: Option<String>,
+}
+
+impl S3 {
+    /// Creates the storage, errors if incorrect AWS S3 configuration provided.
+    pub fn new(aws_config: &S3Config, pageserver_workdir: &'static Path) -> anyhow::Result<Self> {
+        debug!(
+            "Creating s3 remote storage around bucket {}",
+            aws_config.bucket_name
+        );
+        let region = match aws_config.endpoint.clone() {
+            Some(endpoint) => Region::Custom {
+                endpoint,
+                region: aws_config.bucket_region.clone(),
+            },
+            None => aws_config
+                .bucket_region
+                .parse::<Region>()
+                .context("Failed to parse the s3 region from config")?,
+        };
+
+        let credentials = Credentials::new(
+            aws_config.access_key_id.as_deref(),
+            aws_config.secret_access_key.as_deref(),
+            None,
+            None,
+            None,
+        )
+        .context("Failed to create the s3 credentials")?;
+
+        let prefix_in_bucket = aws_config.prefix_in_bucket.as_deref().map(|prefix| {
+            let mut prefix = prefix;
+            while prefix.starts_with(S3_FILE_SEPARATOR) {
+                prefix = &prefix[1..]
+            }
+
+            let mut prefix = prefix.to_string();
+            while prefix.ends_with(S3_FILE_SEPARATOR) {
+                prefix.pop();
+            }
+            prefix
+        });
+
+        Ok(Self {
+            bucket: Bucket::new_with_path_style(
+                aws_config.bucket_name.as_str(),
+                region,
+                credentials,
+            )
+            .context("Failed to create the s3 bucket")?,
+            pageserver_workdir,
+            prefix_in_bucket,
+        })
+    }
+}
+
+#[async_trait::async_trait]
+impl RemoteStorage for S3 {
+    type StoragePath = S3ObjectKey;
+
+    fn storage_path(&self, local_path: &Path) -> anyhow::Result<Self::StoragePath> {
+        let relative_path = strip_path_prefix(self.pageserver_workdir, local_path)?;
+        let mut key = self.prefix_in_bucket.clone().unwrap_or_default();
+        for segment in relative_path {
+            key.push(S3_FILE_SEPARATOR);
+            key.push_str(&segment.to_string_lossy());
+        }
+        Ok(S3ObjectKey(key))
+    }
+
+    fn local_path(&self, storage_path: &Self::StoragePath) -> anyhow::Result<PathBuf> {
+        Ok(storage_path
+            .download_destination(self.pageserver_workdir, self.prefix_in_bucket.as_deref()))
+    }
+
+    async fn list(&self) -> anyhow::Result<Vec<Self::StoragePath>> {
+        let list_response = self
+            .bucket
+            .list(self.prefix_in_bucket.clone().unwrap_or_default(), None)
+            .await
+            .context("Failed to list s3 objects")?;
+
+        Ok(list_response
+            .into_iter()
+            .flat_map(|response| response.contents)
+            .map(|s3_object| S3ObjectKey(s3_object.key))
+            .collect())
+    }
+
+    async fn upload(
+        &self,
+        mut from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
+        to: &Self::StoragePath,
+    ) -> anyhow::Result<()> {
+        let mut upload_contents = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
+        io::copy(&mut from, &mut upload_contents)
+            .await
+            .context("Failed to read the upload contents")?;
+        upload_contents
+            .flush()
+            .await
+            .context("Failed to read the upload contents")?;
+        let upload_contents = upload_contents.into_inner().into_inner();
+
+        let (_, code) = self
+            .bucket
+            .put_object(to.key(), &upload_contents)
+            .await
+            .with_context(|| format!("Failed to create s3 object with key {}", to.key()))?;
+        if code != 200 {
+            Err(anyhow::format_err!(
+                "Received non-200 exit code during creating object with key '{}', code: {}",
+                to.key(),
+                code
+            ))
+        } else {
+            Ok(())
+        }
+    }
+
+    async fn download(
+        &self,
+        from: &Self::StoragePath,
+        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
+    ) -> anyhow::Result<()> {
+        let (data, code) = self
+            .bucket
+            .get_object(from.key())
+            .await
+            .with_context(|| format!("Failed to download s3 object with key {}", from.key()))?;
+        if code != 200 {
+            Err(anyhow::format_err!(
+                "Received non-200 exit code during downloading object, code: {}",
+                code
+            ))
+        } else {
+            // we don't have to write vector into the destination this way, `to_write_all` would be enough.
+            // but we want to prepare for migration on `rusoto`, that has a streaming HTTP body instead here, with
+            // which it makes more sense to use `io::copy`.
+            io::copy(&mut data.as_slice(), to)
+                .await
+                .context("Failed to write downloaded data into the destination buffer")?;
+            Ok(())
+        }
+    }
+
+    async fn download_range(
+        &self,
+        from: &Self::StoragePath,
+        start_inclusive: u64,
+        end_exclusive: Option<u64>,
+        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
+    ) -> anyhow::Result<()> {
+        // S3 accepts ranges as https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35
+        // and needs both ends to be exclusive
+        let end_inclusive = end_exclusive.map(|end| end.saturating_sub(1));
+        let (data, code) = self
+            .bucket
+            .get_object_range(from.key(), start_inclusive, end_inclusive)
+            .await
+            .with_context(|| format!("Failed to download s3 object with key {}", from.key()))?;
+        if code != 206 {
+            Err(anyhow::format_err!(
+                "Received non-206 exit code during downloading object range, code: {}",
+                code
+            ))
+        } else {
+            // see `download` function above for the comment on why `Vec<u8>` buffer is copied this way
+            io::copy(&mut data.as_slice(), to)
+                .await
+                .context("Failed to write downloaded range into the destination buffer")?;
+            Ok(())
+        }
+    }
+
+    async fn delete(&self, path: &Self::StoragePath) -> anyhow::Result<()> {
+        let (_, code) = self
+            .bucket
+            .delete_object(path.key())
+            .await
+            .with_context(|| format!("Failed to delete s3 object with key {}", path.key()))?;
+        if code != 204 {
+            Err(anyhow::format_err!(
+                "Received non-204 exit code during deleting object with key '{}', code: {}",
+                path.key(),
+                code
+            ))
+        } else {
+            Ok(())
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{
+        layered_repository::metadata::METADATA_FILE_NAME,
+        repository::repo_harness::{RepoHarness, TIMELINE_ID},
+    };
+
+    use super::*;
+
+    #[test]
+    fn download_destination() -> anyhow::Result<()> {
+        let repo_harness = RepoHarness::create("download_destination")?;
+
+        let local_path = repo_harness.timeline_path(&TIMELINE_ID).join("test_name");
+        let relative_path = local_path.strip_prefix(&repo_harness.conf.workdir)?;
+
+        let key = S3ObjectKey(format!(
+            "{}{}",
+            S3_FILE_SEPARATOR,
+            relative_path
+                .iter()
+                .map(|segment| segment.to_str().unwrap())
+                .collect::<Vec<_>>()
+                .join(&S3_FILE_SEPARATOR.to_string()),
+        ));
+
+        assert_eq!(
+            local_path,
+            key.download_destination(&repo_harness.conf.workdir, None),
+            "Download destination should consist of s3 path joined with the pageserver workdir prefix"
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn storage_path_positive() -> anyhow::Result<()> {
+        let repo_harness = RepoHarness::create("storage_path_positive")?;
+
+        let segment_1 = "matching";
+        let segment_2 = "file";
+        let local_path = &repo_harness.conf.workdir.join(segment_1).join(segment_2);
+
+        let storage = dummy_storage(&repo_harness.conf.workdir);
+
+        let expected_key = S3ObjectKey(format!(
+            "{}{SEPARATOR}{}{SEPARATOR}{}",
+            storage.prefix_in_bucket.as_deref().unwrap_or_default(),
+            segment_1,
+            segment_2,
+            SEPARATOR = S3_FILE_SEPARATOR,
+        ));
+
+        let actual_key = storage
+            .storage_path(local_path)
+            .expect("Matching path should map to S3 path normally");
+        assert_eq!(
+            expected_key,
+            actual_key,
+            "S3 key from the matching path should contain all segments after the workspace prefix, separated with S3 separator"
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn storage_path_negatives() -> anyhow::Result<()> {
+        #[track_caller]
+        fn storage_path_error(storage: &S3, mismatching_path: &Path) -> String {
+            match storage.storage_path(mismatching_path) {
+                Ok(wrong_key) => panic!(
+                    "Expected path '{}' to error, but got S3 key: {:?}",
+                    mismatching_path.display(),
+                    wrong_key,
+                ),
+                Err(e) => e.to_string(),
+            }
+        }
+
+        let repo_harness = RepoHarness::create("storage_path_negatives")?;
+        let storage = dummy_storage(&repo_harness.conf.workdir);
+
+        let error_message = storage_path_error(&storage, &repo_harness.conf.workdir);
+        assert!(
+            error_message.contains("Prefix and the path are equal"),
+            "Message '{}' does not contain the required string",
+            error_message
+        );
+
+        let mismatching_path = PathBuf::from("somewhere").join("else");
+        let error_message = storage_path_error(&storage, &mismatching_path);
+        assert!(
+            error_message.contains(mismatching_path.to_str().unwrap()),
+            "Error should mention wrong path"
+        );
+        assert!(
+            error_message.contains(repo_harness.conf.workdir.to_str().unwrap()),
+            "Error should mention server workdir"
+        );
+        assert!(
+            error_message.contains("is not prefixed with"),
+            "Message '{}' does not contain a required string",
+            error_message
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn local_path_positive() -> anyhow::Result<()> {
+        let repo_harness = RepoHarness::create("local_path_positive")?;
+        let storage = dummy_storage(&repo_harness.conf.workdir);
+        let timeline_dir = repo_harness.timeline_path(&TIMELINE_ID);
+        let relative_timeline_path = timeline_dir.strip_prefix(&repo_harness.conf.workdir)?;
+
+        let s3_key = create_s3_key(
+            &relative_timeline_path.join("not a metadata"),
+            storage.prefix_in_bucket.as_deref(),
+        );
+        assert_eq!(
+            s3_key.download_destination(
+                &repo_harness.conf.workdir,
+                storage.prefix_in_bucket.as_deref()
+            ),
+            storage
+                .local_path(&s3_key)
+                .expect("For a valid input, valid S3 info should be parsed"),
+            "Should be able to parse metadata out of the correctly named remote delta file"
+        );
+
+        let s3_key = create_s3_key(
+            &relative_timeline_path.join(METADATA_FILE_NAME),
+            storage.prefix_in_bucket.as_deref(),
+        );
+        assert_eq!(
+            s3_key.download_destination(
+                &repo_harness.conf.workdir,
+                storage.prefix_in_bucket.as_deref()
+            ),
+            storage
+                .local_path(&s3_key)
+                .expect("For a valid input, valid S3 info should be parsed"),
+            "Should be able to parse metadata out of the correctly named remote metadata file"
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn download_destination_matches_original_path() -> anyhow::Result<()> {
+        let repo_harness = RepoHarness::create("download_destination_matches_original_path")?;
+        let original_path = repo_harness.timeline_path(&TIMELINE_ID).join("some name");
+
+        let dummy_storage = dummy_storage(&repo_harness.conf.workdir);
+
+        let key = dummy_storage.storage_path(&original_path)?;
+        let download_destination = dummy_storage.local_path(&key)?;
+
+        assert_eq!(
+            original_path, download_destination,
+            "'original path -> storage key -> matching fs path' transformation should produce the same path as the input one for the correct path"
+        );
+
+        Ok(())
+    }
+
+    fn dummy_storage(pageserver_workdir: &'static Path) -> S3 {
+        S3 {
+            pageserver_workdir,
+            bucket: Bucket::new(
+                "dummy-bucket",
+                "us-east-1".parse().unwrap(),
+                Credentials::anonymous().unwrap(),
+            )
+            .unwrap(),
+            prefix_in_bucket: Some("dummy_prefix/".to_string()),
+        }
+    }
+
+    fn create_s3_key(relative_file_path: &Path, prefix: Option<&str>) -> S3ObjectKey {
+        S3ObjectKey(relative_file_path.iter().fold(
+            prefix.unwrap_or_default().to_string(),
+            |mut path_string, segment| {
+                path_string.push(S3_FILE_SEPARATOR);
+                path_string.push_str(segment.to_str().unwrap());
+                path_string
+            },
+        ))
+    }
+}

pageserver/src/remote_storage/storage_sync/compression.rs

@@ -0,0 +1,613 @@
+//! A set of structs to represent a compressed part of the timeline, and methods to asynchronously compress and uncompress a stream of data,
+//! without holding the entire data in memory.
+//! For the latter, both compress and uncompress functions operate buffered streams (currently hardcoded size of [`ARCHIVE_STREAM_BUFFER_SIZE_BYTES`]),
+//! not attempting to hold the entire archive in memory.
+//!
+//! The compression is done with <a href="https://datatracker.ietf.org/doc/html/rfc8878">zstd</a> streaming algorithm via the `async-compression` crate.
+//! The crate does not contain any knobs to tweak the compression, but otherwise is one of the only ones that's both async and has an API to manage the part of an archive.
+//! Zstd was picked as the best algorithm among the ones available in the crate, after testing the initial timeline file compression.
+//!
+//! Archiving is almost agnostic to timeline file types, with an exception of the metadata file, that's currently distinguished in the [un]compression code.
+//! The metadata file is treated separately when [de]compression is involved, to reduce the risk of corrupting the metadata file.
+//! When compressed, the metadata file is always required and stored as the last file in the archive stream.
+//! When uncompressed, the metadata file gets naturally uncompressed last, to ensure that all other relishes are decompressed successfully first.
+//!
+//! Archive structure:
+//! +----------------------------------------+
+//! | header | file_1, ..., file_k, metadata |
+//! +----------------------------------------+
+//!
+//! The archive consists of two separate zstd archives:
+//! * header archive, that contains all files names and their sizes and relative paths in the timeline directory
+//! Header is a Rust structure, serialized into bytes and compressed with zstd.
+//! * files archive, that has metadata file as the last one, all compressed with zstd into a single binary blob
+//!
+//! Header offset is stored in the file name, along with the `disk_consistent_lsn` from the metadata file.
+//! See [`parse_archive_name`] and [`ARCHIVE_EXTENSION`] for the name details, example: `00000000016B9150-.zst_9732`.
+//! This way, the header could be retrieved without reading an entire archive file.
+
+use std::{
+    collections::BTreeSet,
+    future::Future,
+    io::Cursor,
+    path::{Path, PathBuf},
+    sync::Arc,
+};
+
+use anyhow::{bail, ensure, Context};
+use async_compression::tokio::bufread::{ZstdDecoder, ZstdEncoder};
+use serde::{Deserialize, Serialize};
+use tokio::{
+    fs,
+    io::{self, AsyncReadExt, AsyncWriteExt},
+};
+use tracing::*;
+use zenith_utils::{bin_ser::BeSer, lsn::Lsn};
+
+use crate::layered_repository::metadata::{TimelineMetadata, METADATA_FILE_NAME};
+
+use super::index::RelativePath;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArchiveHeader {
+    /// All regular timeline files, excluding the metadata file.
+    pub files: Vec<FileEntry>,
+    // Metadata file name is known to the system, as its location relative to the timeline dir,
+    // so no need to store anything but its size in bytes.
+    pub metadata_file_size: u64,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
+pub struct FileEntry {
+    /// Uncompressed file size, bytes.
+    pub size: u64,
+    /// A path, relative to the directory root, used when compressing the directory contents.
+    pub subpath: RelativePath,
+}
+
+const ARCHIVE_EXTENSION: &str = "-.zst_";
+const ARCHIVE_STREAM_BUFFER_SIZE_BYTES: usize = 4 * 1024 * 1024;
+
+/// Streams an archive of files given into a stream target, defined by the closure.
+///
+/// The closure approach is picked for cases like S3, where we would need a name of the file before we can get a stream to write the bytes into.
+/// Current idea is to place the header size in the name of the file, to enable the fast partial remote file index restoration without actually reading remote storage file contents.
+///
+/// Performs the compression in multiple steps:
+/// * prepares an archive header, stripping the `source_dir` prefix from the `files`
+/// * generates the name of the archive
+/// * prepares archive producer future, knowing the header and the file list
+/// An `impl AsyncRead` and `impl AsyncWrite` pair of connected streams is created to implement the partial contents streaming.
+/// The writer end gets into the archive producer future, to put the header and a stream of compressed files.
+/// * prepares archive consumer future, by executing the provided closure
+/// The closure gets the reader end stream and the name of the file to create a future that would stream the file contents elsewhere.
+/// * runs and waits for both futures to complete
+/// * on a successful completion of both futures, header, its size and the user-defined consumer future return data is returned
+/// Due to the design above, the archive name and related data is visible inside the consumer future only, so it's possible to return the data,
+/// needed for future processing.
+pub async fn archive_files_as_stream<Cons, ConsRet, Fut>(
+    source_dir: &Path,
+    files: impl Iterator<Item = &PathBuf>,
+    metadata: &TimelineMetadata,
+    create_archive_consumer: Cons,
+) -> anyhow::Result<(ArchiveHeader, u64, ConsRet)>
+where
+    Cons: FnOnce(Box<dyn io::AsyncRead + Unpin + Send + Sync + 'static>, String) -> Fut
+        + Send
+        + 'static,
+    Fut: Future<Output = anyhow::Result<ConsRet>> + Send + 'static,
+    ConsRet: Send + Sync + 'static,
+{
+    let metadata_bytes = metadata
+        .to_bytes()
+        .context("Failed to create metadata bytes")?;
+    let (archive_header, compressed_header_bytes) =
+        prepare_header(source_dir, files, &metadata_bytes)
+            .await
+            .context("Failed to prepare file for archivation")?;
+
+    let header_size = compressed_header_bytes.len() as u64;
+    let (write, read) = io::duplex(ARCHIVE_STREAM_BUFFER_SIZE_BYTES);
+    let archive_filler = write_archive_contents(
+        source_dir.to_path_buf(),
+        archive_header.clone(),
+        metadata_bytes,
+        write,
+    );
+    let archive_name = archive_name(metadata.disk_consistent_lsn(), header_size);
+    let archive_stream =
+        Cursor::new(compressed_header_bytes).chain(ZstdEncoder::new(io::BufReader::new(read)));
+
+    let (archive_creation_result, archive_upload_result) = tokio::join!(
+        tokio::spawn(archive_filler),
+        tokio::spawn(async move {
+            create_archive_consumer(Box::new(archive_stream), archive_name).await
+        })
+    );
+    archive_creation_result
+        .context("Failed to spawn archive creation future")?
+        .context("Failed to create an archive")?;
+    let upload_return_value = archive_upload_result
+        .context("Failed to spawn archive upload future")?
+        .context("Failed to upload the archive")?;
+
+    Ok((archive_header, header_size, upload_return_value))
+}
+
+/// Similar to [`archive_files_as_stream`], creates a pair of streams to uncompress the 2nd part of the archive,
+/// that contains files and is located after the header.
+/// S3 allows downloading partial file contents for a given file key (i.e. name), to accommodate this retrieval,
+/// a closure is used.
+/// Same concepts with two concurrent futures, user-defined closure, future and return value apply here, but the
+/// consumer and the receiver ends are swapped, since the uncompression happens.
+pub async fn uncompress_file_stream_with_index<Prod, ProdRet, Fut>(
+    destination_dir: PathBuf,
+    files_to_skip: Arc<BTreeSet<PathBuf>>,
+    disk_consistent_lsn: Lsn,
+    header: ArchiveHeader,
+    header_size: u64,
+    create_archive_file_part: Prod,
+) -> anyhow::Result<ProdRet>
+where
+    Prod: FnOnce(Box<dyn io::AsyncWrite + Unpin + Send + Sync + 'static>, String) -> Fut
+        + Send
+        + 'static,
+    Fut: Future<Output = anyhow::Result<ProdRet>> + Send + 'static,
+    ProdRet: Send + Sync + 'static,
+{
+    let (write, mut read) = io::duplex(ARCHIVE_STREAM_BUFFER_SIZE_BYTES);
+    let archive_name = archive_name(disk_consistent_lsn, header_size);
+
+    let (archive_download_result, archive_uncompress_result) = tokio::join!(
+        tokio::spawn(async move { create_archive_file_part(Box::new(write), archive_name).await }),
+        tokio::spawn(async move {
+            uncompress_with_header(&files_to_skip, &destination_dir, header, &mut read).await
+        })
+    );
+
+    let download_value = archive_download_result
+        .context("Failed to spawn archive download future")?
+        .context("Failed to download an archive")?;
+    archive_uncompress_result
+        .context("Failed to spawn archive uncompress future")?
+        .context("Failed to uncompress the archive")?;
+
+    Ok(download_value)
+}
+
+/// Reads archive header from the stream given:
+/// * parses the file name to get the header size
+/// * reads the exact amount of bytes
+/// * uncompresses and deserializes those
+pub async fn read_archive_header<A: io::AsyncRead + Send + Sync + Unpin>(
+    archive_name: &str,
+    from: &mut A,
+) -> anyhow::Result<ArchiveHeader> {
+    let (_, header_size) = parse_archive_name(Path::new(archive_name))?;
+
+    let mut compressed_header_bytes = vec![0; header_size as usize];
+    from.read_exact(&mut compressed_header_bytes)
+        .await
+        .with_context(|| {
+            format!(
+                "Failed to read header header from the archive {}",
+                archive_name
+            )
+        })?;
+
+    let mut header_bytes = Vec::new();
+    ZstdDecoder::new(io::BufReader::new(compressed_header_bytes.as_slice()))
+        .read_to_end(&mut header_bytes)
+        .await
+        .context("Failed to decompress a header from the archive")?;
+
+    Ok(ArchiveHeader::des(&header_bytes)
+        .context("Failed to deserialize a header from the archive")?)
+}
+
+/// Reads the archive metadata out of the archive name:
+/// * `disk_consistent_lsn` of the checkpoint that was archived
+/// * size of the archive header
+pub fn parse_archive_name(archive_path: &Path) -> anyhow::Result<(Lsn, u64)> {
+    let archive_name = archive_path
+        .file_name()
+        .with_context(|| format!("Archive '{}' has no file name", archive_path.display()))?
+        .to_string_lossy();
+    let (lsn_str, header_size_str) =
+        archive_name
+            .rsplit_once(ARCHIVE_EXTENSION)
+            .with_context(|| {
+                format!(
+                    "Archive '{}' has incorrect extension, expected to contain '{}'",
+                    archive_path.display(),
+                    ARCHIVE_EXTENSION
+                )
+            })?;
+    let disk_consistent_lsn = Lsn::from_hex(lsn_str).with_context(|| {
+        format!(
+            "Archive '{}' has an invalid disk consistent lsn in its extension",
+            archive_path.display(),
+        )
+    })?;
+    let header_size = header_size_str.parse::<u64>().with_context(|| {
+        format!(
+            "Archive '{}' has an invalid a header offset number in its extension",
+            archive_path.display(),
+        )
+    })?;
+    Ok((disk_consistent_lsn, header_size))
+}
+
+fn archive_name(disk_consistent_lsn: Lsn, header_size: u64) -> String {
+    let archive_name = format!(
+        "{:016X}{ARCHIVE_EXTENSION}{}",
+        u64::from(disk_consistent_lsn),
+        header_size,
+        ARCHIVE_EXTENSION = ARCHIVE_EXTENSION,
+    );
+    archive_name
+}
+
+pub async fn uncompress_with_header(
+    files_to_skip: &BTreeSet<PathBuf>,
+    destination_dir: &Path,
+    header: ArchiveHeader,
+    archive_after_header: impl io::AsyncRead + Send + Sync + Unpin,
+) -> anyhow::Result<()> {
+    debug!("Uncompressing archive into {}", destination_dir.display());
+    let mut archive = ZstdDecoder::new(io::BufReader::new(archive_after_header));
+
+    if !destination_dir.exists() {
+        fs::create_dir_all(&destination_dir)
+            .await
+            .with_context(|| {
+                format!(
+                    "Failed to create target directory at {}",
+                    destination_dir.display()
+                )
+            })?;
+    } else if !destination_dir.is_dir() {
+        bail!(
+            "Destination path '{}' is not a valid directory",
+            destination_dir.display()
+        );
+    }
+    debug!("Will extract {} files from the archive", header.files.len());
+    for entry in header.files {
+        uncompress_entry(
+            &mut archive,
+            &entry.subpath.as_path(destination_dir),
+            entry.size,
+            files_to_skip,
+        )
+        .await
+        .with_context(|| format!("Failed to uncompress archive entry {:?}", entry))?;
+    }
+    uncompress_entry(
+        &mut archive,
+        &destination_dir.join(METADATA_FILE_NAME),
+        header.metadata_file_size,
+        files_to_skip,
+    )
+    .await
+    .context("Failed to uncompress the metadata entry")?;
+    Ok(())
+}
+
+async fn uncompress_entry(
+    archive: &mut ZstdDecoder<io::BufReader<impl io::AsyncRead + Send + Sync + Unpin>>,
+    destination_path: &Path,
+    entry_size: u64,
+    files_to_skip: &BTreeSet<PathBuf>,
+) -> anyhow::Result<()> {
+    if let Some(parent) = destination_path.parent() {
+        fs::create_dir_all(parent).await.with_context(|| {
+            format!(
+                "Failed to create parent directory for {}",
+                destination_path.display()
+            )
+        })?;
+    };
+
+    if files_to_skip.contains(destination_path) {
+        debug!("Skipping {}", destination_path.display());
+        copy_n_bytes(entry_size, archive, &mut io::sink())
+            .await
+            .context("Failed to skip bytes in the archive")?;
+        return Ok(());
+    }
+
+    let mut destination =
+        io::BufWriter::new(fs::File::create(&destination_path).await.with_context(|| {
+            format!(
+                "Failed to open file {} for extraction",
+                destination_path.display()
+            )
+        })?);
+    copy_n_bytes(entry_size, archive, &mut destination)
+        .await
+        .with_context(|| {
+            format!(
+                "Failed to write extracted archive contents into file {}",
+                destination_path.display()
+            )
+        })?;
+    destination
+        .flush()
+        .await
+        .context("Failed to flush the streaming archive bytes")?;
+    Ok(())
+}
+
+async fn write_archive_contents(
+    source_dir: PathBuf,
+    header: ArchiveHeader,
+    metadata_bytes: Vec<u8>,
+    mut archive_input: io::DuplexStream,
+) -> anyhow::Result<()> {
+    debug!("Starting writing files into archive");
+    for file_entry in header.files {
+        let path = file_entry.subpath.as_path(&source_dir);
+        let mut source_file =
+            io::BufReader::new(fs::File::open(&path).await.with_context(|| {
+                format!(
+                    "Failed to open file for archiving to path {}",
+                    path.display()
+                )
+            })?);
+        let bytes_written = io::copy(&mut source_file, &mut archive_input)
+            .await
+            .with_context(|| {
+                format!(
+                    "Failed to open add a file into archive, file path {}",
+                    path.display()
+                )
+            })?;
+        ensure!(
+            file_entry.size == bytes_written,
+            "File {} was written to the archive incompletely",
+            path.display()
+        );
+        trace!(
+            "Added file '{}' ({} bytes) into the archive",
+            path.display(),
+            bytes_written
+        );
+    }
+    let metadata_bytes_written = io::copy(&mut metadata_bytes.as_slice(), &mut archive_input)
+        .await
+        .context("Failed to add metadata into the archive")?;
+    ensure!(
+        header.metadata_file_size == metadata_bytes_written,
+        "Metadata file was written to the archive incompletely",
+    );
+
+    archive_input
+        .shutdown()
+        .await
+        .context("Failed to finalize the archive")?;
+    debug!("Successfully streamed all files into the archive");
+    Ok(())
+}
+
+async fn prepare_header(
+    source_dir: &Path,
+    files: impl Iterator<Item = &PathBuf>,
+    metadata_bytes: &[u8],
+) -> anyhow::Result<(ArchiveHeader, Vec<u8>)> {
+    let mut archive_files = Vec::new();
+    for file_path in files {
+        let file_metadata = fs::metadata(file_path).await.with_context(|| {
+            format!(
+                "Failed to read metadata during archive indexing for {}",
+                file_path.display()
+            )
+        })?;
+        ensure!(
+            file_metadata.is_file(),
+            "Archive indexed path {} is not a file",
+            file_path.display()
+        );
+
+        if file_path.file_name().and_then(|name| name.to_str()) != Some(METADATA_FILE_NAME) {
+            let entry = FileEntry {
+                subpath: RelativePath::new(source_dir, file_path).with_context(|| {
+                    format!(
+                        "File '{}' does not belong to pageserver workspace",
+                        file_path.display()
+                    )
+                })?,
+                size: file_metadata.len(),
+            };
+            archive_files.push(entry);
+        }
+    }
+
+    let header = ArchiveHeader {
+        files: archive_files,
+        metadata_file_size: metadata_bytes.len() as u64,
+    };
+
+    debug!("Appending a header for {} files", header.files.len());
+    let header_bytes = header.ser().context("Failed to serialize a header")?;
+    debug!("Header bytes len {}", header_bytes.len());
+    let mut compressed_header_bytes = Vec::new();
+    ZstdEncoder::new(io::BufReader::new(header_bytes.as_slice()))
+        .read_to_end(&mut compressed_header_bytes)
+        .await
+        .context("Failed to compress header bytes")?;
+    debug!(
+        "Compressed header bytes len {}",
+        compressed_header_bytes.len()
+    );
+    Ok((header, compressed_header_bytes))
+}
+
+async fn copy_n_bytes(
+    n: u64,
+    from: &mut (impl io::AsyncRead + Send + Sync + Unpin),
+    into: &mut (impl io::AsyncWrite + Send + Sync + Unpin),
+) -> anyhow::Result<()> {
+    let bytes_written = io::copy(&mut from.take(n), into).await?;
+    ensure!(
+        bytes_written == n,
+        "Failed to read exactly {} bytes from the input, bytes written: {}",
+        n,
+        bytes_written,
+    );
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use tokio::{fs, io::AsyncSeekExt};
+
+    use crate::repository::repo_harness::{RepoHarness, TIMELINE_ID};
+
+    use super::*;
+
+    #[tokio::test]
+    async fn compress_and_uncompress() -> anyhow::Result<()> {
+        let repo_harness = RepoHarness::create("compress_and_uncompress")?;
+        let timeline_dir = repo_harness.timeline_path(&TIMELINE_ID);
+        init_directory(
+            &timeline_dir,
+            vec![
+                ("first", "first_contents"),
+                ("second", "second_contents"),
+                (METADATA_FILE_NAME, "wrong_metadata"),
+            ],
+        )
+        .await?;
+        let timeline_files = list_file_paths_with_contents(&timeline_dir).await?;
+        assert_eq!(
+            timeline_files,
+            vec![
+                (
+                    timeline_dir.join("first"),
+                    FileContents::Text("first_contents".to_string())
+                ),
+                (
+                    timeline_dir.join(METADATA_FILE_NAME),
+                    FileContents::Text("wrong_metadata".to_string())
+                ),
+                (
+                    timeline_dir.join("second"),
+                    FileContents::Text("second_contents".to_string())
+                ),
+            ],
+            "Initial timeline contents should contain two normal files and a wrong metadata file"
+        );
+
+        let metadata = TimelineMetadata::new(Lsn(0x30), None, None, Lsn(0), Lsn(0), Lsn(0));
+        let paths_to_archive = timeline_files
+            .into_iter()
+            .map(|(path, _)| path)
+            .collect::<Vec<_>>();
+
+        let tempdir = tempfile::tempdir()?;
+        let base_path = tempdir.path().to_path_buf();
+        let (header, header_size, archive_target) = archive_files_as_stream(
+            &timeline_dir,
+            paths_to_archive.iter(),
+            &metadata,
+            move |mut archive_streamer, archive_name| async move {
+                let archive_target = base_path.join(&archive_name);
+                let mut archive_file = fs::File::create(&archive_target).await?;
+                io::copy(&mut archive_streamer, &mut archive_file).await?;
+                Ok(archive_target)
+            },
+        )
+        .await?;
+
+        let mut file = fs::File::open(&archive_target).await?;
+        file.seek(io::SeekFrom::Start(header_size)).await?;
+        let target_dir = tempdir.path().join("extracted");
+        uncompress_with_header(&BTreeSet::new(), &target_dir, header, file).await?;
+
+        let extracted_files = list_file_paths_with_contents(&target_dir).await?;
+
+        assert_eq!(
+            extracted_files,
+            vec![
+                (
+                    target_dir.join("first"),
+                    FileContents::Text("first_contents".to_string())
+                ),
+                (
+                    target_dir.join(METADATA_FILE_NAME),
+                    FileContents::Binary(metadata.to_bytes()?)
+                ),
+                (
+                    target_dir.join("second"),
+                    FileContents::Text("second_contents".to_string())
+                ),
+            ],
+            "Extracted files should contain all local timeline files besides its metadata, which should be taken from the arguments"
+        );
+
+        Ok(())
+    }
+
+    async fn init_directory(
+        root: &Path,
+        files_with_contents: Vec<(&str, &str)>,
+    ) -> anyhow::Result<()> {
+        fs::create_dir_all(root).await?;
+        for (file_name, contents) in files_with_contents {
+            fs::File::create(root.join(file_name))
+                .await?
+                .write_all(contents.as_bytes())
+                .await?;
+        }
+        Ok(())
+    }
+
+    #[derive(PartialEq, Eq, PartialOrd, Ord)]
+    enum FileContents {
+        Text(String),
+        Binary(Vec<u8>),
+    }
+
+    impl std::fmt::Debug for FileContents {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            match self {
+                Self::Text(text) => f.debug_tuple("Text").field(text).finish(),
+                Self::Binary(bytes) => f
+                    .debug_tuple("Binary")
+                    .field(&format!("{} bytes", bytes.len()))
+                    .finish(),
+            }
+        }
+    }
+
+    async fn list_file_paths_with_contents(
+        root: &Path,
+    ) -> anyhow::Result<Vec<(PathBuf, FileContents)>> {
+        let mut file_paths = Vec::new();
+
+        let mut dir_listings = vec![fs::read_dir(root).await?];
+        while let Some(mut dir_listing) = dir_listings.pop() {
+            while let Some(entry) = dir_listing.next_entry().await? {
+                let entry_path = entry.path();
+                if entry_path.is_file() {
+                    let contents = match String::from_utf8(fs::read(&entry_path).await?) {
+                        Ok(text) => FileContents::Text(text),
+                        Err(e) => FileContents::Binary(e.into_bytes()),
+                    };
+                    file_paths.push((entry_path, contents));
+                } else if entry_path.is_dir() {
+                    dir_listings.push(fs::read_dir(entry_path).await?);
+                } else {
+                    info!(
+                        "Skipping path '{}' as it's not a file or a directory",
+                        entry_path.display()
+                    );
+                }
+            }
+        }
+
+        file_paths.sort();
+        Ok(file_paths)
+    }
+}

pageserver/src/remote_storage/storage_sync/delete.rs

@@ -1,223 +0,0 @@
-//! Timeline synchrnonization logic to delete a bulk of timeline's remote files from the remote storage.
-
-use anyhow::Context;
-use futures::stream::{FuturesUnordered, StreamExt};
-use tracing::{debug, error, info};
-use utils::zid::ZTenantTimelineId;
-
-use crate::remote_storage::{
-    storage_sync::{SyncQueue, SyncTask},
-    RemoteStorage,
-};
-
-use super::{LayersDeletion, SyncData};
-
-/// Attempts to remove the timleline layers from the remote storage.
-/// If the task had not adjusted the metadata before, the deletion will fail.
-pub(super) async fn delete_timeline_layers<'a, P, S>(
-    storage: &'a S,
-    sync_queue: &SyncQueue,
-    sync_id: ZTenantTimelineId,
-    mut delete_data: SyncData<LayersDeletion>,
-) -> bool
-where
-    P: std::fmt::Debug + Send + Sync + 'static,
-    S: RemoteStorage<RemoteObjectId = P> + Send + Sync + 'static,
-{
-    if !delete_data.data.deletion_registered {
-        error!("Cannot delete timeline layers before the deletion metadata is not registered, reenqueueing");
-        delete_data.retries += 1;
-        sync_queue.push(sync_id, SyncTask::Delete(delete_data));
-        return false;
-    }
-
-    if delete_data.data.layers_to_delete.is_empty() {
-        info!("No layers to delete, skipping");
-        return true;
-    }
-
-    let layers_to_delete = delete_data
-        .data
-        .layers_to_delete
-        .drain()
-        .collect::<Vec<_>>();
-    debug!("Layers to delete: {layers_to_delete:?}");
-    info!("Deleting {} timeline layers", layers_to_delete.len());
-
-    let mut delete_tasks = layers_to_delete
-        .into_iter()
-        .map(|local_layer_path| async {
-            let storage_path = match storage.storage_path(&local_layer_path).with_context(|| {
-                format!(
-                    "Failed to get the layer storage path for local path '{}'",
-                    local_layer_path.display()
-                )
-            }) {
-                Ok(path) => path,
-                Err(e) => return Err((e, local_layer_path)),
-            };
-
-            match storage.delete(&storage_path).await.with_context(|| {
-                format!(
-                    "Failed to delete remote layer from storage at '{:?}'",
-                    storage_path
-                )
-            }) {
-                Ok(()) => Ok(local_layer_path),
-                Err(e) => Err((e, local_layer_path)),
-            }
-        })
-        .collect::<FuturesUnordered<_>>();
-
-    let mut errored = false;
-    while let Some(deletion_result) = delete_tasks.next().await {
-        match deletion_result {
-            Ok(local_layer_path) => {
-                debug!(
-                    "Successfully deleted layer {} for timeline {sync_id}",
-                    local_layer_path.display()
-                );
-                delete_data.data.deleted_layers.insert(local_layer_path);
-            }
-            Err((e, local_layer_path)) => {
-                errored = true;
-                error!(
-                    "Failed to delete layer {} for timeline {sync_id}: {e:?}",
-                    local_layer_path.display()
-                );
-                delete_data.data.layers_to_delete.insert(local_layer_path);
-            }
-        }
-    }
-
-    if errored {
-        debug!("Reenqueuing failed delete task for timeline {sync_id}");
-        delete_data.retries += 1;
-        sync_queue.push(sync_id, SyncTask::Delete(delete_data));
-    }
-    errored
-}
-
-#[cfg(test)]
-mod tests {
-    use std::{collections::HashSet, num::NonZeroUsize};
-
-    use itertools::Itertools;
-    use tempfile::tempdir;
-    use tokio::fs;
-    use utils::lsn::Lsn;
-
-    use crate::{
-        remote_storage::{
-            storage_sync::test_utils::{create_local_timeline, dummy_metadata},
-            LocalFs,
-        },
-        repository::repo_harness::{RepoHarness, TIMELINE_ID},
-    };
-
-    use super::*;
-
-    #[tokio::test]
-    async fn delete_timeline_negative() -> anyhow::Result<()> {
-        let harness = RepoHarness::create("delete_timeline_negative")?;
-        let (sync_queue, _) = SyncQueue::new(NonZeroUsize::new(100).unwrap());
-        let sync_id = ZTenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-        let storage = LocalFs::new(tempdir()?.path().to_path_buf(), &harness.conf.workdir)?;
-
-        let deleted = delete_timeline_layers(
-            &storage,
-            &sync_queue,
-            sync_id,
-            SyncData {
-                retries: 1,
-                data: LayersDeletion {
-                    deleted_layers: HashSet::new(),
-                    layers_to_delete: HashSet::new(),
-                    deletion_registered: false,
-                },
-            },
-        )
-        .await;
-
-        assert!(
-            !deleted,
-            "Should not start the deletion for task with delete metadata unregistered"
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn delete_timeline() -> anyhow::Result<()> {
-        let harness = RepoHarness::create("delete_timeline")?;
-        let (sync_queue, _) = SyncQueue::new(NonZeroUsize::new(100).unwrap());
-
-        let sync_id = ZTenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-        let layer_files = ["a", "b", "c", "d"];
-        let storage = LocalFs::new(tempdir()?.path().to_path_buf(), &harness.conf.workdir)?;
-        let current_retries = 3;
-        let metadata = dummy_metadata(Lsn(0x30));
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let timeline_upload =
-            create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?;
-        for local_path in timeline_upload.layers_to_upload {
-            let remote_path = storage.storage_path(&local_path)?;
-            let remote_parent_dir = remote_path.parent().unwrap();
-            if !remote_parent_dir.exists() {
-                fs::create_dir_all(&remote_parent_dir).await?;
-            }
-            fs::copy(&local_path, &remote_path).await?;
-        }
-        assert_eq!(
-            storage
-                .list()
-                .await?
-                .into_iter()
-                .map(|remote_path| storage.local_path(&remote_path).unwrap())
-                .filter_map(|local_path| { Some(local_path.file_name()?.to_str()?.to_owned()) })
-                .sorted()
-                .collect::<Vec<_>>(),
-            layer_files
-                .iter()
-                .map(|layer_str| layer_str.to_string())
-                .sorted()
-                .collect::<Vec<_>>(),
-            "Expect to have all layer files remotely before deletion"
-        );
-
-        let deleted = delete_timeline_layers(
-            &storage,
-            &sync_queue,
-            sync_id,
-            SyncData {
-                retries: current_retries,
-                data: LayersDeletion {
-                    deleted_layers: HashSet::new(),
-                    layers_to_delete: HashSet::from([
-                        local_timeline_path.join("a"),
-                        local_timeline_path.join("c"),
-                        local_timeline_path.join("something_different"),
-                    ]),
-                    deletion_registered: true,
-                },
-            },
-        )
-        .await;
-        assert!(deleted, "Should be able to delete timeline files");
-
-        assert_eq!(
-            storage
-                .list()
-                .await?
-                .into_iter()
-                .map(|remote_path| storage.local_path(&remote_path).unwrap())
-                .filter_map(|local_path| { Some(local_path.file_name()?.to_str()?.to_owned()) })
-                .sorted()
-                .collect::<Vec<_>>(),
-            vec!["b".to_string(), "d".to_string()],
-            "Expect to have only non-deleted files remotely"
-        );
-
-        Ok(())
-    }
-}

pageserver/src/remote_storage/storage_sync/download.rs

@@ -0,0 +1,333 @@
+//! Timeline synchrnonization logic to put files from archives on remote storage into pageserver's local directory.
+
+use std::{borrow::Cow, collections::BTreeSet, path::PathBuf, sync::Arc};
+
+use anyhow::{ensure, Context};
+use tokio::{fs, sync::RwLock};
+use tracing::{debug, error, trace, warn};
+use zenith_utils::zid::ZTenantId;
+
+use crate::{
+    config::PageServerConf,
+    layered_repository::metadata::{metadata_path, TimelineMetadata},
+    remote_storage::{
+        storage_sync::{
+            compression, fetch_full_index, index::TimelineIndexEntryInner, sync_queue, SyncKind,
+            SyncTask,
+        },
+        RemoteStorage, ZTenantTimelineId,
+    },
+};
+
+use super::{
+    index::{ArchiveId, RemoteTimeline, RemoteTimelineIndex},
+    TimelineDownload,
+};
+
+/// Timeline download result, with extra data, needed for downloading.
+pub(super) enum DownloadedTimeline {
+    /// Remote timeline data is either absent or corrupt, no download possible.
+    Abort,
+    /// Remote timeline data is found, its latest checkpoint's metadata contents (disk_consistent_lsn) is known.
+    /// Initial download failed due to some error, the download task is rescheduled for another retry.
+    FailedAndRescheduled,
+    /// Remote timeline data is found, its latest checkpoint's metadata contents (disk_consistent_lsn) is known.
+    /// Initial download successful.
+    Successful,
+}
+
+/// Attempts to download and uncompress files from all remote archives for the timeline given.
+/// Timeline files that already exist locally are skipped during the download, but the local metadata file is
+/// updated in the end of every checkpoint archive extraction.
+///
+/// On an error, bumps the retries count and reschedules the download, with updated archive skip list
+/// (for any new successful archive downloads and extractions).
+pub(super) async fn download_timeline<
+    P: std::fmt::Debug + Send + Sync + 'static,
+    S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
+>(
+    conf: &'static PageServerConf,
+    remote_assets: Arc<(S, Arc<RwLock<RemoteTimelineIndex>>)>,
+    sync_id: ZTenantTimelineId,
+    mut download: TimelineDownload,
+    retries: u32,
+) -> DownloadedTimeline {
+    debug!("Downloading layers for sync id {}", sync_id);
+
+    let ZTenantTimelineId {
+        tenant_id,
+        timeline_id,
+    } = sync_id;
+    let index = &remote_assets.1;
+
+    let index_read = index.read().await;
+    let remote_timeline = match index_read.timeline_entry(&sync_id) {
+        None => {
+            error!("Cannot download: no timeline is present in the index for given id");
+            return DownloadedTimeline::Abort;
+        }
+
+        Some(index_entry) => match index_entry.inner() {
+            TimelineIndexEntryInner::Full(remote_timeline) => Cow::Borrowed(remote_timeline),
+            TimelineIndexEntryInner::Description(_) => {
+                // we do not check here for awaits_download because it is ok
+                // to call this function while the download is in progress
+                // so it is not a concurrent download, it is the same one
+
+                let remote_disk_consistent_lsn = index_entry.disk_consistent_lsn();
+                drop(index_read);
+                debug!("Found timeline description for the given ids, downloading the full index");
+                match fetch_full_index(
+                    remote_assets.as_ref(),
+                    &conf.timeline_path(&timeline_id, &tenant_id),
+                    sync_id,
+                )
+                .await
+                {
+                    Ok(remote_timeline) => Cow::Owned(remote_timeline),
+                    Err(e) => {
+                        error!("Failed to download full timeline index: {:?}", e);
+
+                        return match remote_disk_consistent_lsn {
+                            Some(_) => {
+                                sync_queue::push(SyncTask::new(
+                                    sync_id,
+                                    retries,
+                                    SyncKind::Download(download),
+                                ));
+                                DownloadedTimeline::FailedAndRescheduled
+                            }
+                            None => {
+                                error!("Cannot download: no disk consistent Lsn is present for the index entry");
+                                DownloadedTimeline::Abort
+                            }
+                        };
+                    }
+                }
+            }
+        },
+    };
+    if remote_timeline.checkpoints().max().is_none() {
+        debug!("Cannot download: no disk consistent Lsn is present for the remote timeline");
+        return DownloadedTimeline::Abort;
+    };
+
+    debug!("Downloading timeline archives");
+    let archives_to_download = remote_timeline
+        .checkpoints()
+        .map(ArchiveId)
+        .filter(|remote_archive| !download.archives_to_skip.contains(remote_archive))
+        .collect::<Vec<_>>();
+
+    let archives_total = archives_to_download.len();
+    debug!("Downloading {} archives of a timeline", archives_total);
+    trace!("Archives to download: {:?}", archives_to_download);
+
+    for (archives_downloaded, archive_id) in archives_to_download.into_iter().enumerate() {
+        match try_download_archive(
+            conf,
+            sync_id,
+            Arc::clone(&remote_assets),
+            &remote_timeline,
+            archive_id,
+            Arc::clone(&download.files_to_skip),
+        )
+        .await
+        {
+            Err(e) => {
+                let archives_left = archives_total - archives_downloaded;
+                error!(
+                    "Failed to download archive {:?} (archives downloaded: {}; archives left: {}) for tenant {} timeline {}, requeueing the download: {:?}",
+                    archive_id, archives_downloaded, archives_left, tenant_id, timeline_id, e
+                );
+                sync_queue::push(SyncTask::new(
+                    sync_id,
+                    retries,
+                    SyncKind::Download(download),
+                ));
+                return DownloadedTimeline::FailedAndRescheduled;
+            }
+            Ok(()) => {
+                debug!("Successfully downloaded archive {:?}", archive_id);
+                download.archives_to_skip.insert(archive_id);
+            }
+        }
+    }
+
+    debug!("Finished downloading all timeline's archives");
+    DownloadedTimeline::Successful
+}
+
+async fn try_download_archive<
+    P: Send + Sync + 'static,
+    S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
+>(
+    conf: &'static PageServerConf,
+    ZTenantTimelineId {
+        tenant_id,
+        timeline_id,
+    }: ZTenantTimelineId,
+    remote_assets: Arc<(S, Arc<RwLock<RemoteTimelineIndex>>)>,
+    remote_timeline: &RemoteTimeline,
+    archive_id: ArchiveId,
+    files_to_skip: Arc<BTreeSet<PathBuf>>,
+) -> anyhow::Result<()> {
+    debug!("Downloading archive {:?}", archive_id);
+    let archive_to_download = remote_timeline
+        .archive_data(archive_id)
+        .with_context(|| format!("Archive {:?} not found in remote storage", archive_id))?;
+    let (archive_header, header_size) = remote_timeline
+        .restore_header(archive_id)
+        .context("Failed to restore header when downloading an archive")?;
+
+    match read_local_metadata(conf, timeline_id, tenant_id).await {
+        Ok(local_metadata) => ensure!(
+            // need to allow `<=` instead of `<` due to cases when a failed archive can be redownloaded
+            local_metadata.disk_consistent_lsn() <= archive_to_download.disk_consistent_lsn(),
+            "Cannot download archive with Lsn {} since it's earlier than local Lsn {}",
+            archive_to_download.disk_consistent_lsn(),
+            local_metadata.disk_consistent_lsn()
+        ),
+        Err(e) => warn!("Failed to read local metadata file, assuming it's safe to override its with the download. Read: {:#}", e),
+    }
+    compression::uncompress_file_stream_with_index(
+        conf.timeline_path(&timeline_id, &tenant_id),
+        files_to_skip,
+        archive_to_download.disk_consistent_lsn(),
+        archive_header,
+        header_size,
+        move |mut archive_target, archive_name| async move {
+            let archive_local_path = conf
+                .timeline_path(&timeline_id, &tenant_id)
+                .join(&archive_name);
+            let remote_storage = &remote_assets.0;
+            remote_storage
+                .download_range(
+                    &remote_storage.storage_path(&archive_local_path)?,
+                    header_size,
+                    None,
+                    &mut archive_target,
+                )
+                .await
+        },
+    )
+    .await?;
+
+    Ok(())
+}
+
+async fn read_local_metadata(
+    conf: &'static PageServerConf,
+    timeline_id: zenith_utils::zid::ZTimelineId,
+    tenant_id: ZTenantId,
+) -> anyhow::Result<TimelineMetadata> {
+    let local_metadata_path = metadata_path(conf, timeline_id, tenant_id);
+    let local_metadata_bytes = fs::read(&local_metadata_path)
+        .await
+        .context("Failed to read local metadata file bytes")?;
+    Ok(TimelineMetadata::from_bytes(&local_metadata_bytes)
+        .context("Failed to read local metadata files bytes")?)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::BTreeSet;
+
+    use tempfile::tempdir;
+    use tokio::fs;
+    use zenith_utils::lsn::Lsn;
+
+    use crate::{
+        remote_storage::{
+            local_fs::LocalFs,
+            storage_sync::test_utils::{
+                assert_index_descriptions, assert_timeline_files_match, create_local_timeline,
+                dummy_metadata, ensure_correct_timeline_upload, expect_timeline,
+            },
+        },
+        repository::repo_harness::{RepoHarness, TIMELINE_ID},
+    };
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_download_timeline() -> anyhow::Result<()> {
+        let repo_harness = RepoHarness::create("test_download_timeline")?;
+        let sync_id = ZTenantTimelineId::new(repo_harness.tenant_id, TIMELINE_ID);
+        let storage = LocalFs::new(tempdir()?.path().to_owned(), &repo_harness.conf.workdir)?;
+        let index = Arc::new(RwLock::new(
+            RemoteTimelineIndex::try_parse_descriptions_from_paths(
+                repo_harness.conf,
+                storage
+                    .list()
+                    .await?
+                    .into_iter()
+                    .map(|storage_path| storage.local_path(&storage_path).unwrap()),
+            ),
+        ));
+        let remote_assets = Arc::new((storage, index));
+        let storage = &remote_assets.0;
+        let index = &remote_assets.1;
+
+        let regular_timeline_path = repo_harness.timeline_path(&TIMELINE_ID);
+        let regular_timeline = create_local_timeline(
+            &repo_harness,
+            TIMELINE_ID,
+            &["a", "b"],
+            dummy_metadata(Lsn(0x30)),
+        )?;
+        ensure_correct_timeline_upload(
+            &repo_harness,
+            Arc::clone(&remote_assets),
+            TIMELINE_ID,
+            regular_timeline,
+        )
+        .await;
+        // upload multiple checkpoints for the same timeline
+        let regular_timeline = create_local_timeline(
+            &repo_harness,
+            TIMELINE_ID,
+            &["c", "d"],
+            dummy_metadata(Lsn(0x40)),
+        )?;
+        ensure_correct_timeline_upload(
+            &repo_harness,
+            Arc::clone(&remote_assets),
+            TIMELINE_ID,
+            regular_timeline,
+        )
+        .await;
+
+        fs::remove_dir_all(&regular_timeline_path).await?;
+        let remote_regular_timeline = expect_timeline(index, sync_id).await;
+
+        download_timeline(
+            repo_harness.conf,
+            Arc::clone(&remote_assets),
+            sync_id,
+            TimelineDownload {
+                files_to_skip: Arc::new(BTreeSet::new()),
+                archives_to_skip: BTreeSet::new(),
+            },
+            0,
+        )
+        .await;
+        assert_index_descriptions(
+            index,
+            RemoteTimelineIndex::try_parse_descriptions_from_paths(
+                repo_harness.conf,
+                remote_assets
+                    .0
+                    .list()
+                    .await
+                    .unwrap()
+                    .into_iter()
+                    .map(|storage_path| storage.local_path(&storage_path).unwrap()),
+            ),
+        )
+        .await;
+        assert_timeline_files_match(&repo_harness, TIMELINE_ID, remote_regular_timeline);
+
+        Ok(())
+    }
+}

pageserver/src/remote_storage/storage_sync/index.rs

@@ -0,0 +1,476 @@
+//! In-memory index to track the tenant files on the remote strorage, mitigating the storage format differences between the local and remote files.
+//! Able to restore itself from the storage archive data and reconstruct archive indices on demand.
+//!
+//! The index is intended to be portable, so deliberately does not store any local paths inside.
+//! This way in the future, the index could be restored fast from its serialized stored form.
+
+use std::{
+    collections::{BTreeMap, BTreeSet, HashMap},
+    path::{Path, PathBuf},
+};
+
+use anyhow::{bail, ensure, Context};
+use serde::{Deserialize, Serialize};
+use tracing::*;
+use zenith_utils::{
+    lsn::Lsn,
+    zid::{ZTenantId, ZTimelineId},
+};
+
+use crate::{
+    config::PageServerConf,
+    layered_repository::TIMELINES_SEGMENT_NAME,
+    remote_storage::{
+        storage_sync::compression::{parse_archive_name, FileEntry},
+        ZTenantTimelineId,
+    },
+};
+
+use super::compression::ArchiveHeader;
+
+/// A part of the filesystem path, that needs a root to become a path again.
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub struct RelativePath(String);
+
+impl RelativePath {
+    /// Attempts to strip off the base from path, producing a relative path or an error.
+    pub fn new<P: AsRef<Path>>(base: &Path, path: P) -> anyhow::Result<Self> {
+        let relative = path
+            .as_ref()
+            .strip_prefix(base)
+            .context("path is not relative to base")?;
+        Ok(RelativePath(relative.to_string_lossy().to_string()))
+    }
+
+    /// Joins the relative path with the base path.
+    pub fn as_path(&self, base: &Path) -> PathBuf {
+        base.join(&self.0)
+    }
+}
+
+/// An index to track tenant files that exist on the remote storage.
+/// Currently, timeline archive files are tracked only.
+#[derive(Debug, Clone)]
+pub struct RemoteTimelineIndex {
+    timeline_entries: HashMap<ZTenantTimelineId, TimelineIndexEntry>,
+}
+
+impl RemoteTimelineIndex {
+    pub fn empty() -> Self {
+        Self {
+            timeline_entries: HashMap::new(),
+        }
+    }
+
+    /// Attempts to parse file paths (not checking the file contents) and find files
+    /// that can be tracked wiht the index.
+    /// On parse falures, logs the error and continues, so empty index can be created from not suitable paths.
+    pub fn try_parse_descriptions_from_paths<P: AsRef<Path>>(
+        conf: &'static PageServerConf,
+        paths: impl Iterator<Item = P>,
+    ) -> Self {
+        let mut index = Self::empty();
+        for path in paths {
+            if let Err(e) = try_parse_index_entry(&mut index, conf, path.as_ref()) {
+                debug!(
+                    "Failed to parse path '{}' as index entry: {:#}",
+                    path.as_ref().display(),
+                    e
+                );
+            }
+        }
+        index
+    }
+
+    pub fn timeline_entry(&self, id: &ZTenantTimelineId) -> Option<&TimelineIndexEntry> {
+        self.timeline_entries.get(id)
+    }
+
+    pub fn timeline_entry_mut(
+        &mut self,
+        id: &ZTenantTimelineId,
+    ) -> Option<&mut TimelineIndexEntry> {
+        self.timeline_entries.get_mut(id)
+    }
+
+    pub fn add_timeline_entry(&mut self, id: ZTenantTimelineId, entry: TimelineIndexEntry) {
+        self.timeline_entries.insert(id, entry);
+    }
+
+    pub fn upgrade_timeline_entry(
+        &mut self,
+        id: &ZTenantTimelineId,
+        remote_timeline: RemoteTimeline,
+    ) -> anyhow::Result<()> {
+        let mut entry = self.timeline_entries.get_mut(id).ok_or(anyhow::anyhow!(
+            "timeline is unexpectedly missing from remote index"
+        ))?;
+
+        if !matches!(entry.inner, TimelineIndexEntryInner::Description(_)) {
+            anyhow::bail!("timeline entry is not a description entry")
+        };
+
+        entry.inner = TimelineIndexEntryInner::Full(remote_timeline);
+
+        Ok(())
+    }
+
+    pub fn all_sync_ids(&self) -> impl Iterator<Item = ZTenantTimelineId> + '_ {
+        self.timeline_entries.keys().copied()
+    }
+
+    pub fn set_awaits_download(
+        &mut self,
+        id: &ZTenantTimelineId,
+        awaits_download: bool,
+    ) -> anyhow::Result<()> {
+        self.timeline_entry_mut(id)
+            .ok_or_else(|| anyhow::anyhow!("unknown timeline sync {}", id))?
+            .set_awaits_download(awaits_download);
+        Ok(())
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub struct DescriptionTimelineIndexEntry {
+    pub description: BTreeMap<ArchiveId, ArchiveDescription>,
+    pub awaits_download: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FullTimelineIndexEntry {
+    pub remote_timeline: RemoteTimeline,
+    pub awaits_download: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum TimelineIndexEntryInner {
+    Description(BTreeMap<ArchiveId, ArchiveDescription>),
+    Full(RemoteTimeline),
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TimelineIndexEntry {
+    inner: TimelineIndexEntryInner,
+    awaits_download: bool,
+}
+
+impl TimelineIndexEntry {
+    pub fn new(inner: TimelineIndexEntryInner, awaits_download: bool) -> Self {
+        Self {
+            inner,
+            awaits_download,
+        }
+    }
+
+    pub fn inner(&self) -> &TimelineIndexEntryInner {
+        &self.inner
+    }
+
+    pub fn inner_mut(&mut self) -> &mut TimelineIndexEntryInner {
+        &mut self.inner
+    }
+
+    pub fn uploaded_checkpoints(&self) -> BTreeSet<Lsn> {
+        match &self.inner {
+            TimelineIndexEntryInner::Description(description) => {
+                description.keys().map(|archive_id| archive_id.0).collect()
+            }
+            TimelineIndexEntryInner::Full(remote_timeline) => remote_timeline
+                .checkpoint_archives
+                .keys()
+                .map(|archive_id| archive_id.0)
+                .collect(),
+        }
+    }
+
+    /// Gets latest uploaded checkpoint's disk consisten Lsn for the corresponding timeline.
+    pub fn disk_consistent_lsn(&self) -> Option<Lsn> {
+        match &self.inner {
+            TimelineIndexEntryInner::Description(description) => {
+                description.keys().map(|archive_id| archive_id.0).max()
+            }
+            TimelineIndexEntryInner::Full(remote_timeline) => remote_timeline
+                .checkpoint_archives
+                .keys()
+                .map(|archive_id| archive_id.0)
+                .max(),
+        }
+    }
+
+    pub fn get_awaits_download(&self) -> bool {
+        self.awaits_download
+    }
+
+    pub fn set_awaits_download(&mut self, awaits_download: bool) {
+        self.awaits_download = awaits_download;
+    }
+}
+
+/// Checkpoint archive's id, corresponding to the `disk_consistent_lsn` from the timeline's metadata file during checkpointing.
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
+pub struct ArchiveId(pub(super) Lsn);
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
+struct FileId(ArchiveId, ArchiveEntryNumber);
+
+type ArchiveEntryNumber = usize;
+
+/// All archives and files in them, representing a certain timeline.
+/// Uses file and archive IDs to reference those without ownership issues.
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct RemoteTimeline {
+    timeline_files: BTreeMap<FileId, FileEntry>,
+    checkpoint_archives: BTreeMap<ArchiveId, CheckpointArchive>,
+}
+
+/// Archive metadata, enough to restore a header with the timeline data.
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct CheckpointArchive {
+    disk_consistent_lsn: Lsn,
+    metadata_file_size: u64,
+    files: BTreeSet<FileId>,
+    archive_header_size: u64,
+}
+
+impl CheckpointArchive {
+    pub fn disk_consistent_lsn(&self) -> Lsn {
+        self.disk_consistent_lsn
+    }
+}
+
+impl RemoteTimeline {
+    pub fn empty() -> Self {
+        Self {
+            timeline_files: BTreeMap::new(),
+            checkpoint_archives: BTreeMap::new(),
+        }
+    }
+
+    pub fn checkpoints(&self) -> impl Iterator<Item = Lsn> + '_ {
+        self.checkpoint_archives
+            .values()
+            .map(CheckpointArchive::disk_consistent_lsn)
+    }
+
+    /// Lists all relish files in the given remote timeline. Omits the metadata file.
+    pub fn stored_files(&self, timeline_dir: &Path) -> BTreeSet<PathBuf> {
+        self.timeline_files
+            .values()
+            .map(|file_entry| file_entry.subpath.as_path(timeline_dir))
+            .collect()
+    }
+
+    pub fn contains_checkpoint_at(&self, disk_consistent_lsn: Lsn) -> bool {
+        self.checkpoint_archives
+            .contains_key(&ArchiveId(disk_consistent_lsn))
+    }
+
+    pub fn archive_data(&self, archive_id: ArchiveId) -> Option<&CheckpointArchive> {
+        self.checkpoint_archives.get(&archive_id)
+    }
+
+    /// Restores a header of a certain remote archive from the memory data.
+    /// Returns the header and its compressed size in the archive, both can be used to uncompress that archive.
+    pub fn restore_header(&self, archive_id: ArchiveId) -> anyhow::Result<(ArchiveHeader, u64)> {
+        let archive = self
+            .checkpoint_archives
+            .get(&archive_id)
+            .with_context(|| format!("Archive {:?} not found", archive_id))?;
+
+        let mut header_files = Vec::with_capacity(archive.files.len());
+        for (expected_archive_position, archive_file) in archive.files.iter().enumerate() {
+            let &FileId(archive_id, archive_position) = archive_file;
+            ensure!(
+                expected_archive_position == archive_position,
+                "Archive header is corrupt, file # {} from archive {:?} header is missing",
+                expected_archive_position,
+                archive_id,
+            );
+
+            let timeline_file = self.timeline_files.get(archive_file).with_context(|| {
+                format!(
+                    "File with id {:?} not found for archive {:?}",
+                    archive_file, archive_id
+                )
+            })?;
+            header_files.push(timeline_file.clone());
+        }
+
+        Ok((
+            ArchiveHeader {
+                files: header_files,
+                metadata_file_size: archive.metadata_file_size,
+            },
+            archive.archive_header_size,
+        ))
+    }
+
+    /// Updates (creates, if necessary) the data about certain archive contents.
+    pub fn update_archive_contents(
+        &mut self,
+        disk_consistent_lsn: Lsn,
+        header: ArchiveHeader,
+        header_size: u64,
+    ) {
+        let archive_id = ArchiveId(disk_consistent_lsn);
+        let mut common_archive_files = BTreeSet::new();
+        for (file_index, file_entry) in header.files.into_iter().enumerate() {
+            let file_id = FileId(archive_id, file_index);
+            self.timeline_files.insert(file_id, file_entry);
+            common_archive_files.insert(file_id);
+        }
+
+        let metadata_file_size = header.metadata_file_size;
+        self.checkpoint_archives
+            .entry(archive_id)
+            .or_insert_with(|| CheckpointArchive {
+                metadata_file_size,
+                files: BTreeSet::new(),
+                archive_header_size: header_size,
+                disk_consistent_lsn,
+            })
+            .files
+            .extend(common_archive_files.into_iter());
+    }
+}
+
+/// Metadata abput timeline checkpoint archive, parsed from its remote storage path.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ArchiveDescription {
+    pub header_size: u64,
+    pub disk_consistent_lsn: Lsn,
+    pub archive_name: String,
+}
+
+fn try_parse_index_entry(
+    index: &mut RemoteTimelineIndex,
+    conf: &'static PageServerConf,
+    path: &Path,
+) -> anyhow::Result<()> {
+    let tenants_dir = conf.tenants_path();
+    let tenant_id = path
+        .strip_prefix(&tenants_dir)
+        .with_context(|| {
+            format!(
+                "Path '{}' does not belong to tenants directory '{}'",
+                path.display(),
+                tenants_dir.display(),
+            )
+        })?
+        .iter()
+        .next()
+        .with_context(|| format!("Found no tenant id in path '{}'", path.display()))?
+        .to_string_lossy()
+        .parse::<ZTenantId>()
+        .with_context(|| format!("Failed to parse tenant id from path '{}'", path.display()))?;
+
+    let timelines_path = conf.timelines_path(&tenant_id);
+    match path.strip_prefix(&timelines_path) {
+        Ok(timelines_subpath) => {
+            let mut segments = timelines_subpath.iter();
+            let timeline_id = segments
+                .next()
+                .with_context(|| {
+                    format!(
+                        "{} directory of tenant {} (path '{}') is not an index entry",
+                        TIMELINES_SEGMENT_NAME,
+                        tenant_id,
+                        path.display()
+                    )
+                })?
+                .to_string_lossy()
+                .parse::<ZTimelineId>()
+                .with_context(|| {
+                    format!("Failed to parse timeline id from path '{}'", path.display())
+                })?;
+
+            let (disk_consistent_lsn, header_size) =
+                parse_archive_name(path).with_context(|| {
+                    format!(
+                        "Failed to parse archive name out in path '{}'",
+                        path.display()
+                    )
+                })?;
+
+            let archive_name = path
+                .file_name()
+                .with_context(|| format!("Archive '{}' has no file name", path.display()))?
+                .to_string_lossy()
+                .to_string();
+
+            let sync_id = ZTenantTimelineId {
+                tenant_id,
+                timeline_id,
+            };
+            let timeline_index_entry = index.timeline_entries.entry(sync_id).or_insert_with(|| {
+                TimelineIndexEntry::new(
+                    TimelineIndexEntryInner::Description(BTreeMap::default()),
+                    false,
+                )
+            });
+            match timeline_index_entry.inner_mut() {
+                TimelineIndexEntryInner::Description(description) => {
+                    description.insert(
+                        ArchiveId(disk_consistent_lsn),
+                        ArchiveDescription {
+                            header_size,
+                            disk_consistent_lsn,
+                            archive_name,
+                        },
+                    );
+                }
+                TimelineIndexEntryInner::Full(_) => {
+                    bail!("Cannot add parsed archive description to its full context in index with sync id {}", sync_id)
+                }
+            }
+        }
+        Err(timelines_strip_error) => {
+            bail!(
+                "Path '{}' is not an archive entry '{}'",
+                path.display(),
+                timelines_strip_error,
+            )
+        }
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn header_restoration_preserves_file_order() {
+        let header = ArchiveHeader {
+            files: vec![
+                FileEntry {
+                    size: 5,
+                    subpath: RelativePath("one".to_string()),
+                },
+                FileEntry {
+                    size: 1,
+                    subpath: RelativePath("two".to_string()),
+                },
+                FileEntry {
+                    size: 222,
+                    subpath: RelativePath("zero".to_string()),
+                },
+            ],
+            metadata_file_size: 5,
+        };
+
+        let lsn = Lsn(1);
+        let mut remote_timeline = RemoteTimeline::empty();
+        remote_timeline.update_archive_contents(lsn, header.clone(), 15);
+
+        let (restored_header, _) = remote_timeline
+            .restore_header(ArchiveId(lsn))
+            .expect("Should be able to restore header from a valid remote timeline");
+
+        assert_eq!(
+            header, restored_header,
+            "Header restoration should preserve file order"
+        );
+    }
+}