Remove fields that are present in TenantConf from global PageserverConf to avoid confusion

Add checksum to tenantconf file.
Save tenant conf file to tenant's directory, when tenant is created.

.circleci/ansible/deploy.yaml

@@ -1,14 +1,14 @@
-- name: Upload Neon binaries
+- name: Upload Zenith binaries
   hosts: storage
   gather_facts: False
   remote_user: admin
 
   tasks:
 
-    - name: get latest version of Neon binaries
+    - name: get latest version of Zenith binaries
       register: current_version_file
       set_fact:
-        current_version: "{{ lookup('file', '.neon_current_version') | trim }}"
+        current_version: "{{ lookup('file', '.zenith_current_version') | trim }}"
       tags:
       - pageserver
       - safekeeper
@@ -19,11 +19,11 @@
       - pageserver
       - safekeeper
 
-    - name: upload and extract Neon binaries to /usr/local
+    - name: upload and extract Zenith binaries to /usr/local
       ansible.builtin.unarchive:
         owner: root
         group: root
-        src: neon_install.tar.gz
+        src: zenith_install.tar.gz
         dest: /usr/local
       become: true
       tags:
@@ -63,18 +63,21 @@
       tags:
       - pageserver
 
-    - name: update remote storage (s3) config
-      lineinfile:
-        path: /storage/pageserver/data/pageserver.toml
-        line: "{{ item }}"
-      loop:
-        - "[remote_storage]"
-        - "bucket_name = '{{ bucket_name }}'"
-        - "bucket_region = '{{ bucket_region }}'"
-        - "prefix_in_bucket = '{{ inventory_hostname }}'"
-      become: true
-      tags:
-      - pageserver
+    # It seems that currently S3 integration does not play well
+    # even with fresh pageserver without a burden of old data.
+    # TODO: turn this back on once the issue is solved.
+    # - name: update remote storage (s3) config
+    #   lineinfile:
+    #     path: /storage/pageserver/data/pageserver.toml
+    #     line: "{{ item }}"
+    #   loop:
+    #     - "[remote_storage]"
+    #     - "bucket_name = '{{ bucket_name }}'"
+    #     - "bucket_region = '{{ bucket_region }}'"
+    #     - "prefix_in_bucket = '{{ inventory_hostname }}'"
+    #   become: true
+    #   tags:
+    #   - pageserver
 
     - name: upload systemd service definition
       ansible.builtin.template:

.circleci/ansible/get_binaries.sh

@@ -4,10 +4,10 @@ set -e
 
 RELEASE=${RELEASE:-false}
 
-# look at docker hub for latest tag for neon docker image
+# look at docker hub for latest tag fo zenith docker image
 if [ "${RELEASE}" = "true" ]; then
     echo "search latest relase tag"
-    VERSION=$(curl -s https://registry.hub.docker.com/v1/repositories/neondatabase/neon/tags |jq -r -S '.[].name' | grep release | sed 's/release-//g' | tail -1)
+    VERSION=$(curl -s https://registry.hub.docker.com/v1/repositories/zenithdb/zenith/tags |jq -r -S '.[].name' | grep release | sed 's/release-//g' | tail -1)
     if [ -z "${VERSION}" ]; then
         echo "no any docker tags found, exiting..."
         exit 1
@@ -16,7 +16,7 @@ if [ "${RELEASE}" = "true" ]; then
     fi
 else
     echo "search latest dev tag"
-    VERSION=$(curl -s https://registry.hub.docker.com/v1/repositories/neondatabase/neon/tags |jq -r -S '.[].name' | grep -v release | tail -1)
+    VERSION=$(curl -s https://registry.hub.docker.com/v1/repositories/zenithdb/zenith/tags |jq -r -S '.[].name' | grep -v release | tail -1)
     if [ -z "${VERSION}" ]; then
         echo "no any docker tags found, exiting..."
         exit 1
@@ -28,25 +28,25 @@ fi
 echo "found ${VERSION}"
 
 # do initial cleanup
-rm -rf neon_install postgres_install.tar.gz neon_install.tar.gz .neon_current_version
-mkdir neon_install
+rm -rf zenith_install postgres_install.tar.gz zenith_install.tar.gz .zenith_current_version
+mkdir zenith_install
 
 # retrive binaries from docker image
 echo "getting binaries from docker image"
-docker pull --quiet neondatabase/neon:${TAG}
-ID=$(docker create neondatabase/neon:${TAG})
+docker pull --quiet zenithdb/zenith:${TAG}
+ID=$(docker create zenithdb/zenith:${TAG})
 docker cp ${ID}:/data/postgres_install.tar.gz .
-tar -xzf postgres_install.tar.gz -C neon_install
-docker cp ${ID}:/usr/local/bin/pageserver neon_install/bin/
-docker cp ${ID}:/usr/local/bin/safekeeper neon_install/bin/
-docker cp ${ID}:/usr/local/bin/proxy neon_install/bin/
-docker cp ${ID}:/usr/local/bin/postgres neon_install/bin/
+tar -xzf postgres_install.tar.gz -C zenith_install
+docker cp ${ID}:/usr/local/bin/pageserver zenith_install/bin/
+docker cp ${ID}:/usr/local/bin/safekeeper zenith_install/bin/
+docker cp ${ID}:/usr/local/bin/proxy zenith_install/bin/
+docker cp ${ID}:/usr/local/bin/postgres zenith_install/bin/
 docker rm -vf ${ID}
 
 # store version to file (for ansible playbooks) and create binaries tarball
-echo ${VERSION} > neon_install/.neon_current_version
-echo ${VERSION} > .neon_current_version
-tar -czf neon_install.tar.gz -C neon_install .
+echo ${VERSION} > zenith_install/.zenith_current_version
+echo ${VERSION} > .zenith_current_version
+tar -czf zenith_install.tar.gz -C zenith_install .
 
 # do final cleaup
-rm -rf neon_install postgres_install.tar.gz
+rm -rf zenith_install postgres_install.tar.gz

.circleci/ansible/production.hosts

@@ -14,4 +14,3 @@ safekeepers
 console_mgmt_base_url = http://console-release.local
 bucket_name           = zenith-storage-oregon
 bucket_region         = us-west-2
-etcd_endpoints        = etcd-release.local:2379

.circleci/ansible/staging.hosts

@@ -5,6 +5,7 @@ zenith-us-stage-ps-2 console_region_id=27
 [safekeepers]
 zenith-us-stage-sk-1 console_region_id=27
 zenith-us-stage-sk-2 console_region_id=27
+zenith-us-stage-sk-3 console_region_id=27
 zenith-us-stage-sk-4 console_region_id=27
 
 [storage:children]
@@ -15,4 +16,3 @@ safekeepers
 console_mgmt_base_url = http://console-staging.local
 bucket_name           = zenith-staging-storage-us-east-1
 bucket_region         = us-east-1
-etcd_endpoints        = etcd-staging.local:2379

.circleci/ansible/systemd/safekeeper.service

@@ -6,7 +6,7 @@ After=network.target auditd.service
 Type=simple
 User=safekeeper
 Environment=RUST_BACKTRACE=1 ZENITH_REPO_DIR=/storage/safekeeper/data LD_LIBRARY_PATH=/usr/local/lib
-ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}.local:6500 --listen-http {{ inventory_hostname }}.local:7676 -p {{ first_pageserver }}:6400 -D /storage/safekeeper/data --broker-endpoints={{ etcd_endpoints }}
+ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}.local:6500 --listen-http {{ inventory_hostname }}.local:7676 -p {{ first_pageserver }}:6400 -D /storage/safekeeper/data
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=mixed
 KillSignal=SIGINT

.circleci/config.yml

@@ -1,18 +1,18 @@
 version: 2.1
 
 executors:
-  neon-xlarge-executor:
+  zenith-xlarge-executor:
     resource_class: xlarge
     docker:
       # NB: when changed, do not forget to update rust image tag in all Dockerfiles
-      - image: zimg/rust:1.58
-  neon-executor:
+      - image: zimg/rust:1.56
+  zenith-executor:
     docker:
-      - image: zimg/rust:1.58
+      - image: zimg/rust:1.56
 
 jobs:
   check-codestyle-rust:
-    executor: neon-xlarge-executor
+    executor: zenith-xlarge-executor
     steps:
       - checkout
       - run:
@@ -22,7 +22,7 @@ jobs:
 
   # A job to build postgres
   build-postgres:
-    executor: neon-xlarge-executor
+    executor: zenith-xlarge-executor
     parameters:
       build_type:
         type: enum
@@ -67,9 +67,9 @@ jobs:
           paths:
             - tmp_install
 
-  # A job to build Neon rust code
-  build-neon:
-    executor: neon-xlarge-executor
+  # A job to build zenith rust code
+  build-zenith:
+    executor: zenith-xlarge-executor
     parameters:
       build_type:
         type: enum
@@ -113,7 +113,7 @@ jobs:
               CARGO_FLAGS=
             elif [[ $BUILD_TYPE == "release" ]]; then
               cov_prefix=()
-              CARGO_FLAGS="--release --features profiling"
+              CARGO_FLAGS=--release
             fi
 
             export CARGO_INCREMENTAL=0
@@ -132,6 +132,20 @@ jobs:
             - ~/.cargo/git
             - target
 
+        # Run style checks
+        # has to run separately from cargo fmt section
+        # since needs to run with dependencies
+      - run:
+          name: cargo clippy
+          command: |
+            if [[ $BUILD_TYPE == "debug" ]]; then
+              cov_prefix=(scripts/coverage "--profraw-prefix=$CIRCLE_JOB" --dir=/tmp/zenith/coverage run)
+            elif [[ $BUILD_TYPE == "release" ]]; then
+              cov_prefix=()
+            fi
+
+            "${cov_prefix[@]}" ./run_clippy.sh
+
         # Run rust unit tests
       - run:
           name: cargo test
@@ -209,7 +223,7 @@ jobs:
             - "*"
 
   check-codestyle-python:
-    executor: neon-executor
+    executor: zenith-executor
     steps:
       - checkout
       - restore_cache:
@@ -232,7 +246,7 @@ jobs:
           command: poetry run mypy .
 
   run-pytest:
-    executor: neon-executor
+    executor: zenith-executor
     parameters:
       # pytest args to specify the tests to run.
       #
@@ -355,7 +369,7 @@ jobs:
           when: always
           command: |
             du -sh /tmp/test_output/*
-            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "safekeeper.log" ! -name "regression.diffs" ! -name "junit.xml" ! -name "*.filediff" ! -name "*.stdout" ! -name "*.stderr" ! -name "flamegraph.svg" -delete
+            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "safekeeper.log" ! -name "regression.diffs" ! -name "junit.xml" ! -name "*.filediff" ! -name "*.stdout" ! -name "*.stderr" -delete
             du -sh /tmp/test_output/*
       - store_artifacts:
           path: /tmp/test_output
@@ -376,7 +390,7 @@ jobs:
             - "*"
 
   coverage-report:
-    executor: neon-xlarge-executor
+    executor: zenith-xlarge-executor
     steps:
       - attach_workspace:
           at: /tmp/zenith
@@ -391,7 +405,7 @@ jobs:
       - run:
           name: Build coverage report
           command: |
-            COMMIT_URL=https://github.com/neondatabase/neon/commit/$CIRCLE_SHA1
+            COMMIT_URL=https://github.com/zenithdb/zenith/commit/$CIRCLE_SHA1
 
             scripts/coverage \
               --dir=/tmp/zenith/coverage report \
@@ -402,11 +416,11 @@ jobs:
           name: Upload coverage report
           command: |
             LOCAL_REPO=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
-            REPORT_URL=https://neondatabase.github.io/zenith-coverage-data/$CIRCLE_SHA1
-            COMMIT_URL=https://github.com/neondatabase/neon/commit/$CIRCLE_SHA1
+            REPORT_URL=https://zenithdb.github.io/zenith-coverage-data/$CIRCLE_SHA1
+            COMMIT_URL=https://github.com/zenithdb/zenith/commit/$CIRCLE_SHA1
 
             scripts/git-upload \
-              --repo=https://$VIP_VAP_ACCESS_TOKEN@github.com/neondatabase/zenith-coverage-data.git \
+              --repo=https://$VIP_VAP_ACCESS_TOKEN@github.com/zenithdb/zenith-coverage-data.git \
               --message="Add code coverage for $COMMIT_URL" \
               copy /tmp/zenith/coverage/report $CIRCLE_SHA1 # COPY FROM TO_RELATIVE
 
@@ -423,7 +437,7 @@ jobs:
                 \"target_url\": \"$REPORT_URL\"
               }"
 
-  # Build neondatabase/neon:latest image and push it to Docker hub
+  # Build zenithdb/zenith:latest image and push it to Docker hub
   docker-image:
     docker:
       - image: cimg/base:2021.04
@@ -437,18 +451,18 @@ jobs:
       - run:
           name: Build and push Docker image
           command: |
-            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
             DOCKER_TAG=$(git log --oneline|wc -l)
             docker build \
               --pull \
               --build-arg GIT_VERSION=${CIRCLE_SHA1} \
               --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
               --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
-              --tag neondatabase/neon:${DOCKER_TAG} --tag neondatabase/neon:latest .
-            docker push neondatabase/neon:${DOCKER_TAG}
-            docker push neondatabase/neon:latest
+              --tag zenithdb/zenith:${DOCKER_TAG} --tag zenithdb/zenith:latest .
+            docker push zenithdb/zenith:${DOCKER_TAG}
+            docker push zenithdb/zenith:latest
 
-  # Build neondatabase/compute-node:latest image and push it to Docker hub
+  # Build zenithdb/compute-node:latest image and push it to Docker hub
   docker-image-compute:
     docker:
       - image: cimg/base:2021.04
@@ -456,31 +470,31 @@ jobs:
       - checkout
       - setup_remote_docker:
           docker_layer_caching: true
-      # Build neondatabase/compute-tools:latest image and push it to Docker hub
+      # Build zenithdb/compute-tools:latest image and push it to Docker hub
       # TODO: this should probably also use versioned tag, not just :latest.
       # XXX: but should it? We build and use it only locally now.
       - run:
           name: Build and push compute-tools Docker image
           command: |
-            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
             docker build \
               --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
               --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
-              --tag neondatabase/compute-tools:latest -f Dockerfile.compute-tools .
-            docker push neondatabase/compute-tools:latest
+              --tag zenithdb/compute-tools:latest -f Dockerfile.compute-tools .
+            docker push zenithdb/compute-tools:latest
       - run:
           name: Init postgres submodule
           command: git submodule update --init --depth 1
       - run:
           name: Build and push compute-node Docker image
           command: |
-            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
             DOCKER_TAG=$(git log --oneline|wc -l)
-            docker build --tag neondatabase/compute-node:${DOCKER_TAG} --tag neondatabase/compute-node:latest vendor/postgres
-            docker push neondatabase/compute-node:${DOCKER_TAG}
-            docker push neondatabase/compute-node:latest
+            docker build --tag zenithdb/compute-node:${DOCKER_TAG} --tag zenithdb/compute-node:latest vendor/postgres
+            docker push zenithdb/compute-node:${DOCKER_TAG}
+            docker push zenithdb/compute-node:latest
 
-  # Build production neondatabase/neon:release image and push it to Docker hub
+  # Build production zenithdb/zenith:release image and push it to Docker hub
   docker-image-release:
     docker:
       - image: cimg/base:2021.04
@@ -494,18 +508,18 @@ jobs:
       - run:
           name: Build and push Docker image
           command: |
-            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
             DOCKER_TAG="release-$(git log --oneline|wc -l)"
             docker build \
               --pull \
               --build-arg GIT_VERSION=${CIRCLE_SHA1} \
               --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
               --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
-              --tag neondatabase/neon:${DOCKER_TAG} --tag neondatabase/neon:release .
-            docker push neondatabase/neon:${DOCKER_TAG}
-            docker push neondatabase/neon:release
+              --tag zenithdb/zenith:${DOCKER_TAG} --tag zenithdb/zenith:release .
+            docker push zenithdb/zenith:${DOCKER_TAG}
+            docker push zenithdb/zenith:release
 
-  # Build production neondatabase/compute-node:release image and push it to Docker hub
+  # Build production zenithdb/compute-node:release image and push it to Docker hub
   docker-image-compute-release:
     docker:
       - image: cimg/base:2021.04
@@ -513,29 +527,29 @@ jobs:
       - checkout
       - setup_remote_docker:
           docker_layer_caching: true
-      # Build neondatabase/compute-tools:release image and push it to Docker hub
+      # Build zenithdb/compute-tools:release image and push it to Docker hub
       # TODO: this should probably also use versioned tag, not just :latest.
       # XXX: but should it? We build and use it only locally now.
       - run:
           name: Build and push compute-tools Docker image
           command: |
-            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
             docker build \
               --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
               --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
-              --tag neondatabase/compute-tools:release -f Dockerfile.compute-tools .
-            docker push neondatabase/compute-tools:release
+              --tag zenithdb/compute-tools:release -f Dockerfile.compute-tools .
+            docker push zenithdb/compute-tools:release
       - run:
           name: Init postgres submodule
           command: git submodule update --init --depth 1
       - run:
           name: Build and push compute-node Docker image
           command: |
-            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
+            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
             DOCKER_TAG="release-$(git log --oneline|wc -l)"
-            docker build --tag neondatabase/compute-node:${DOCKER_TAG} --tag neondatabase/compute-node:release vendor/postgres
-            docker push neondatabase/compute-node:${DOCKER_TAG}
-            docker push neondatabase/compute-node:release
+            docker build --tag zenithdb/compute-node:${DOCKER_TAG} --tag zenithdb/compute-node:release vendor/postgres
+            docker push zenithdb/compute-node:${DOCKER_TAG}
+            docker push zenithdb/compute-node:release
 
   deploy-staging:
     docker:
@@ -561,7 +575,7 @@ jobs:
             rm -f ssh-key ssh-key-cert.pub
 
             ansible-playbook deploy.yaml -i staging.hosts
-            rm -f neon_install.tar.gz .neon_current_version
+            rm -f zenith_install.tar.gz .zenith_current_version
 
   deploy-staging-proxy:
     docker:
@@ -579,7 +593,7 @@ jobs:
           name: Setup helm v3
           command: |
             curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
-            helm repo add zenithdb https://neondatabase.github.io/helm-charts
+            helm repo add zenithdb https://zenithdb.github.io/helm-charts
       - run:
           name: Re-deploy proxy
           command: |
@@ -611,7 +625,7 @@ jobs:
             rm -f ssh-key ssh-key-cert.pub
 
             ansible-playbook deploy.yaml -i production.hosts
-            rm -f neon_install.tar.gz .neon_current_version
+            rm -f zenith_install.tar.gz .zenith_current_version
 
   deploy-release-proxy:
     docker:
@@ -629,7 +643,7 @@ jobs:
           name: Setup helm v3
           command: |
             curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
-            helm repo add zenithdb https://neondatabase.github.io/helm-charts
+            helm repo add zenithdb https://zenithdb.github.io/helm-charts
       - run:
           name: Re-deploy proxy
           command: |
@@ -658,7 +672,7 @@ jobs:
             --data \
               "{
                 \"state\": \"pending\",
-                \"context\": \"neon-cloud-e2e\",
+                \"context\": \"zenith-remote-ci\",
                 \"description\": \"[$REMOTE_REPO] Remote CI job is about to start\"
               }"
       - run:
@@ -674,7 +688,7 @@ jobs:
               "{
                 \"ref\": \"main\",
                 \"inputs\": {
-                  \"ci_job_name\": \"neon-cloud-e2e\",
+                  \"ci_job_name\": \"zenith-remote-ci\",
                   \"commit_hash\": \"$CIRCLE_SHA1\",
                   \"remote_repo\": \"$LOCAL_REPO\"
                 }
@@ -690,8 +704,8 @@ workflows:
           matrix:
             parameters:
               build_type: ["debug", "release"]
-      - build-neon:
-          name: build-neon-<< matrix.build_type >>
+      - build-zenith:
+          name: build-zenith-<< matrix.build_type >>
           matrix:
             parameters:
               build_type: ["debug", "release"]
@@ -706,7 +720,7 @@ workflows:
           test_selection: batch_pg_regress
           needs_postgres_source: true
           requires:
-            - build-neon-<< matrix.build_type >>
+            - build-zenith-<< matrix.build_type >>
       - run-pytest:
           name: other-tests-<< matrix.build_type >>
           matrix:
@@ -714,7 +728,7 @@ workflows:
               build_type: ["debug", "release"]
           test_selection: batch_others
           requires:
-            - build-neon-<< matrix.build_type >>
+            - build-zenith-<< matrix.build_type >>
       - run-pytest:
           name: benchmarks
           context: PERF_TEST_RESULT_CONNSTR
@@ -723,7 +737,7 @@ workflows:
           run_in_parallel: false
           save_perf_report: true
           requires:
-            - build-neon-release
+            - build-zenith-release
       - coverage-report:
           # Context passes credentials for gh api
           context: CI_ACCESS_TOKEN
@@ -814,11 +828,11 @@ workflows:
       - remote-ci-trigger:
           # Context passes credentials for gh api
           context: CI_ACCESS_TOKEN
-          remote_repo: "neondatabase/cloud"
+          remote_repo: "zenithdb/console"
           requires:
             # XXX: Successful build doesn't mean everything is OK, but
             # the job to be triggered takes so much time to complete (~22 min)
             # that it's better not to wait for the commented-out steps
-            - build-neon-release
+            - build-zenith-debug
             # - pg_regress-tests-release
             # - other-tests-release

.circleci/helm-values/production.proxy.yaml

@@ -1,9 +1,6 @@
 # Helm chart values for zenith-proxy.
 # This is a YAML-formatted file.
 
-image:
-  repository: neondatabase/neon
-
 settings:
   authEndpoint: "https://console.zenith.tech/authenticate_proxy_request/"
   uri: "https://console.zenith.tech/psql_session/"

.circleci/helm-values/staging.proxy.yaml

@@ -1,9 +1,6 @@
 # Helm chart values for zenith-proxy.
 # This is a YAML-formatted file.
 
-image:
-  repository: neondatabase/neon
-
 settings:
   authEndpoint: "https://console.stage.zenith.tech/authenticate_proxy_request/"
   uri: "https://console.stage.zenith.tech/psql_session/"

.config/hakari.toml

@@ -10,8 +10,6 @@ dep-format-version = "2"
 # Hakari works much better with the new feature resolver.
 # For more about the new feature resolver, see:
 # https://blog.rust-lang.org/2021/03/25/Rust-1.51.0.html#cargos-new-feature-resolver
-# Have to keep the resolver still here since hakari requires this field,
-# despite it's now the default for 2021 edition & cargo.
 resolver = "2"
 
 # Add triples corresponding to platforms commonly used by developers here.

.github/workflows/benchmarking.yml

@@ -26,7 +26,7 @@ jobs:
     runs-on: [self-hosted, zenith-benchmarker]
 
     env:
-      POSTGRES_DISTRIB_DIR: "/usr/pgsql-13"
+      PG_BIN: "/usr/pgsql-13/bin"
 
     steps:
     - name: Checkout zenith repo
@@ -51,7 +51,7 @@ jobs:
         echo Poetry
         poetry --version
         echo Pgbench
-        $POSTGRES_DISTRIB_DIR/bin/pgbench --version
+        $PG_BIN/pgbench --version
 
     # FIXME cluster setup is skipped due to various changes in console API
     # for now pre created cluster is used. When API gain some stability
@@ -66,7 +66,7 @@ jobs:
 
         echo "Starting cluster"
         # wake up the cluster
-        $POSTGRES_DISTRIB_DIR/bin/psql $BENCHMARK_CONNSTR -c "SELECT 1"
+        $PG_BIN/psql $BENCHMARK_CONNSTR -c "SELECT 1"
 
     - name: Run benchmark
       # pgbench is installed system wide from official repo
@@ -83,11 +83,8 @@ jobs:
       # sudo yum install postgresql13-contrib
       # actual binaries are located in /usr/pgsql-13/bin/
       env:
-        # The pgbench test runs two tests of given duration against each scale.
-        # So the total runtime with these parameters is 2 * 2 * 300 = 1200, or 20 minutes.
-        # Plus time needed to initialize the test databases.
-        TEST_PG_BENCH_DURATIONS_MATRIX: "300"
-        TEST_PG_BENCH_SCALES_MATRIX: "10,100"
+        TEST_PG_BENCH_TRANSACTIONS_MATRIX: "5000,10000,20000"
+        TEST_PG_BENCH_SCALES_MATRIX: "10,15"
         PLATFORM: "zenith-staging"
         BENCHMARK_CONNSTR: "${{ secrets.BENCHMARK_STAGING_CONNSTR }}"
         REMOTE_ENV: "1" # indicate to test harness that we do not have zenith binaries locally

.github/workflows/testing.yml

@@ -36,7 +36,8 @@ jobs:
 
       - name: Install macOs postgres dependencies
         if: matrix.os == 'macos-latest'
-        run: brew install flex bison
+        run: |
+          brew install flex bison
 
       - name: Set pg revision for caching
         id: pg_ver
@@ -52,7 +53,8 @@ jobs:
 
       - name: Build postgres
         if: steps.cache_pg.outputs.cache-hit != 'true'
-        run: make postgres
+        run: |
+          make postgres
 
       - name: Cache cargo deps
         id: cache_cargo
@@ -62,10 +64,13 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('./Cargo.lock') }}
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
 
-      - name: Run cargo clippy
-        run: ./run_clippy.sh
+      # Use `env CARGO_INCREMENTAL=0` to mitigate https://github.com/rust-lang/rust/issues/91696 for rustc 1.57.0
+      - name: Run cargo build
+        run: |
+          env CARGO_INCREMENTAL=0 cargo build --workspace --bins --examples --tests
 
       - name: Run cargo test
-        run: cargo test --all --all-targets
+        run: |
+          env CARGO_INCREMENTAL=0 cargo test -- --nocapture --test-threads=1

Cargo.toml

@@ -3,19 +3,22 @@ members = [
     "compute_tools",
     "control_plane",
     "pageserver",
+    "postgres_ffi",
     "proxy",
-    "safekeeper",
+    "walkeeper",
     "workspace_hack",
     "zenith",
-    "libs/*",
+    "zenith_metrics",
+    "zenith_utils",
 ]
+resolver = "2"
 
 [profile.release]
 # This is useful for profiling and, to some extent, debug.
 # Besides, debug info should not affect the performance.
 debug = true
 
-# This is only needed for proxy's tests.
-# TODO: we should probably fork `tokio-postgres-rustls` instead.
+# This is only needed for proxy's tests
+# TODO: we should probably fork tokio-postgres-rustls instead
 [patch.crates-io]
-tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }

Dockerfile

@@ -1,5 +1,7 @@
 # Build Postgres
-FROM zimg/rust:1.58 AS pg-build
+#
+#FROM zimg/rust:1.56 AS pg-build
+FROM zenithdb/build:buster-20220309 AS pg-build
 WORKDIR /pg
 
 USER root
@@ -9,26 +11,27 @@ COPY Makefile Makefile
 
 ENV BUILD_TYPE release
 RUN set -e \
-    && mold -run make -j $(nproc) -s postgres \
+    && make -j $(nproc) -s postgres \
     && rm -rf tmp_install/build \
     && tar -C tmp_install -czf /postgres_install.tar.gz .
 
 # Build zenith binaries
-FROM zimg/rust:1.58 AS build
+#
+#FROM zimg/rust:1.56 AS build
+FROM zenithdb/build:buster-20220309 AS build
 ARG GIT_VERSION=local
 
 ARG CACHEPOT_BUCKET=zenith-rust-cachepot
 ARG AWS_ACCESS_KEY_ID
 ARG AWS_SECRET_ACCESS_KEY
+ENV RUSTC_WRAPPER /usr/local/cargo/bin/cachepot
 
 COPY --from=pg-build /pg/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
 COPY . .
 
 # Show build caching stats to check if it was used in the end.
 # Has to be the part of the same RUN since cachepot daemon is killed in the end of this RUN, loosing the compilation stats.
-RUN set -e \
-    && sudo -E "PATH=$PATH" mold -run cargo build --release \
-    && cachepot -s
+RUN cargo build --release && /usr/local/cargo/bin/cachepot -s
 
 # Build final image
 #

Dockerfile.build

@@ -0,0 +1,23 @@
+FROM rust:1.56.1-slim-buster
+WORKDIR /home/circleci/project
+
+RUN set -e \
+    && apt-get update \
+    && apt-get -yq install \
+        automake \
+        libtool \
+        build-essential \
+        bison \
+        flex \
+        libreadline-dev \
+        zlib1g-dev \
+        libxml2-dev \
+        libseccomp-dev \
+        pkg-config \
+        libssl-dev \
+        clang
+
+RUN set -e \
+    && rustup component add clippy \
+    && cargo install cargo-audit \
+    && cargo install --git https://github.com/paritytech/cachepot

Dockerfile.compute-tools

@@ -1,18 +1,19 @@
 # First transient image to build compute_tools binaries
 # NB: keep in sync with rust image version in .circle/config.yml
-FROM zimg/rust:1.58 AS rust-build
+FROM zenithdb/build:buster-20220309 AS rust-build
+
+WORKDIR /zenith
 
 ARG CACHEPOT_BUCKET=zenith-rust-cachepot
 ARG AWS_ACCESS_KEY_ID
 ARG AWS_SECRET_ACCESS_KEY
+ENV RUSTC_WRAPPER /usr/local/cargo/bin/cachepot
 
 COPY . .
 
-RUN set -e \
-    && sudo -E "PATH=$PATH" mold -run cargo build -p compute_tools --release \
-    && cachepot -s
+RUN cargo build -p compute_tools --release && /usr/local/cargo/bin/cachepot -s
 
 # Final image that only has one binary
 FROM debian:buster-slim
 
-COPY --from=rust-build /home/circleci/project/target/release/zenith_ctl /usr/local/bin/zenith_ctl
+COPY --from=rust-build /zenith/target/release/zenith_ctl /usr/local/bin/zenith_ctl

README.md

@@ -1,22 +1,19 @@
-# Neon
+# Zenith
 
-Neon is a serverless open source alternative to AWS Aurora Postgres. It separates storage and compute and substitutes PostgreSQL storage layer by redistributing data across a cluster of nodes.
-
-The project used to be called "Zenith". Many of the commands and code comments
-still refer to "zenith", but we are in the process of renaming things.
+Zenith is a serverless open source alternative to AWS Aurora Postgres. It separates storage and compute and substitutes PostgreSQL storage layer by redistributing data across a cluster of nodes.
 
 ## Architecture overview
 
-A Neon installation consists of compute nodes and Neon storage engine.
+A Zenith installation consists of compute nodes and Zenith storage engine.
 
-Compute nodes are stateless PostgreSQL nodes, backed by Neon storage engine.
+Compute nodes are stateless PostgreSQL nodes, backed by Zenith storage engine.
 
-Neon storage engine consists of two major components:
+Zenith storage engine consists of two major components:
 - Pageserver. Scalable storage backend for compute nodes.
 - WAL service. The service that receives WAL from compute node and ensures that it is stored durably.
 
 Pageserver consists of:
-- Repository - Neon storage implementation.
+- Repository - Zenith storage implementation.
 - WAL receiver - service that receives WAL from WAL service and stores it in the repository.
 - Page service - service that communicates with compute nodes and responds with pages from the repository.
 - WAL redo - service that builds pages from base images and WAL records on Page service request.
@@ -31,17 +28,17 @@ apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libsec
 libssl-dev clang pkg-config libpq-dev
 ```
 
-[Rust] 1.58 or later is also required.
+[Rust] 1.56.1 or later is also required.
 
 To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively.
 
 To run the integration tests or Python scripts (not required to use the code), install
 Python (3.7 or higher), and install python3 packages using `./scripts/pysync` (requires poetry) in the project directory.
 
-2. Build neon and patched postgres
+2. Build zenith and patched postgres
 ```sh
-git clone --recursive https://github.com/neondatabase/neon.git
-cd neon
+git clone --recursive https://github.com/zenithdb/zenith.git
+cd zenith
 make -j5
 ```
 
@@ -129,7 +126,7 @@ INSERT 0 1
 ## Running tests
 
 ```sh
-git clone --recursive https://github.com/neondatabase/neon.git
+git clone --recursive https://github.com/zenithdb/zenith.git
 make # builds also postgres and installs it to ./tmp_install
 ./scripts/pytest
 ```
@@ -144,14 +141,14 @@ To view your `rustdoc` documentation in a browser, try running `cargo doc --no-d
 
 ### Postgres-specific terms
 
-Due to Neon's very close relation with PostgreSQL internals, there are numerous specific terms used.
+Due to Zenith's very close relation with PostgreSQL internals, there are numerous specific terms used.
 Same applies to certain spelling: i.e. we use MB to denote 1024 * 1024 bytes, while MiB would be technically more correct, it's inconsistent with what PostgreSQL code and its documentation use.
 
 To get more familiar with this aspect, refer to:
 
-- [Neon glossary](/docs/glossary.md)
+- [Zenith glossary](/docs/glossary.md)
 - [PostgreSQL glossary](https://www.postgresql.org/docs/13/glossary.html)
-- Other PostgreSQL documentation and sources (Neon fork sources can be found [here](https://github.com/neondatabase/postgres))
+- Other PostgreSQL documentation and sources (Zenith fork sources can be found [here](https://github.com/zenithdb/postgres))
 
 ## Join the development
 

compute_tools/Cargo.toml

@@ -11,11 +11,10 @@ clap = "3.0"
 env_logger = "0.9"
 hyper = { version = "0.14", features = ["full"] }
 log = { version = "0.4", features = ["std", "serde"] }
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 regex = "1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 tar = "0.4"
 tokio = { version = "1.17", features = ["macros", "rt", "rt-multi-thread"] }
-tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
 workspace_hack = { version = "0.1", path = "../workspace_hack" }

compute_tools/src/bin/zenith_ctl.rs

@@ -38,7 +38,6 @@ use clap::Arg;
 use log::info;
 use postgres::{Client, NoTls};
 
-use compute_tools::checker::create_writablity_check_data;
 use compute_tools::config;
 use compute_tools::http_api::launch_http_server;
 use compute_tools::logger::*;
@@ -129,8 +128,6 @@ fn run_compute(state: &Arc<RwLock<ComputeState>>) -> Result<ExitStatus> {
 
     handle_roles(&read_state.spec, &mut client)?;
     handle_databases(&read_state.spec, &mut client)?;
-    handle_grants(&read_state.spec, &mut client)?;
-    create_writablity_check_data(&mut client)?;
 
     // 'Close' connection
     drop(client);
@@ -158,12 +155,12 @@ fn run_compute(state: &Arc<RwLock<ComputeState>>) -> Result<ExitStatus> {
 }
 
 fn main() -> Result<()> {
-    // TODO: re-use `utils::logging` later
+    // TODO: re-use `zenith_utils::logging` later
     init_logger(DEFAULT_LOG_LEVEL)?;
 
     // Env variable is set by `cargo`
     let version: Option<&str> = option_env!("CARGO_PKG_VERSION");
-    let matches = clap::Command::new("zenith_ctl")
+    let matches = clap::App::new("zenith_ctl")
         .version(version.unwrap_or("unknown"))
         .arg(
             Arg::new("connstr")

compute_tools/src/checker.rs

@@ -1,46 +0,0 @@
-use std::sync::{Arc, RwLock};
-
-use anyhow::{anyhow, Result};
-use log::error;
-use postgres::Client;
-use tokio_postgres::NoTls;
-
-use crate::zenith::ComputeState;
-
-pub fn create_writablity_check_data(client: &mut Client) -> Result<()> {
-    let query = "
-    CREATE TABLE IF NOT EXISTS health_check (
-        id serial primary key,
-        updated_at timestamptz default now()
-    );
-    INSERT INTO health_check VALUES (1, now())
-        ON CONFLICT (id) DO UPDATE
-         SET updated_at = now();";
-    let result = client.simple_query(query)?;
-    if result.len() < 2 {
-        return Err(anyhow::format_err!("executed  {} queries", result.len()));
-    }
-    Ok(())
-}
-
-pub async fn check_writability(state: &Arc<RwLock<ComputeState>>) -> Result<()> {
-    let connstr = state.read().unwrap().connstr.clone();
-    let (client, connection) = tokio_postgres::connect(&connstr, NoTls).await?;
-    if client.is_closed() {
-        return Err(anyhow!("connection to postgres closed"));
-    }
-    tokio::spawn(async move {
-        if let Err(e) = connection.await {
-            error!("connection error: {}", e);
-        }
-    });
-
-    let result = client
-        .simple_query("UPDATE health_check SET updated_at = now() WHERE id = 1;")
-        .await?;
-
-    if result.len() != 1 {
-        return Err(anyhow!("statement can't be executed"));
-    }
-    Ok(())
-}

compute_tools/src/http_api.rs

@@ -11,7 +11,7 @@ use log::{error, info};
 use crate::zenith::*;
 
 // Service function to handle all available routes.
-async fn routes(req: Request<Body>, state: Arc<RwLock<ComputeState>>) -> Response<Body> {
+fn routes(req: Request<Body>, state: Arc<RwLock<ComputeState>>) -> Response<Body> {
     match (req.method(), req.uri().path()) {
         // Timestamp of the last Postgres activity in the plain text.
         (&Method::GET, "/last_activity") => {
@@ -29,15 +29,6 @@ async fn routes(req: Request<Body>, state: Arc<RwLock<ComputeState>>) -> Respons
             Response::new(Body::from(format!("{}", state.ready)))
         }
 
-        (&Method::GET, "/check_writability") => {
-            info!("serving /check_writability GET request");
-            let res = crate::checker::check_writability(&state).await;
-            match res {
-                Ok(_) => Response::new(Body::from("true")),
-                Err(e) => Response::new(Body::from(e.to_string())),
-            }
-        }
-
         // Return the `404 Not Found` for any other routes.
         _ => {
             let mut not_found = Response::new(Body::from("404 Not Found"));
@@ -57,7 +48,7 @@ async fn serve(state: Arc<RwLock<ComputeState>>) {
         async move {
             Ok::<_, Infallible>(service_fn(move |req: Request<Body>| {
                 let state = state.clone();
-                async move { Ok::<_, Infallible>(routes(req, state).await) }
+                async move { Ok::<_, Infallible>(routes(req, state)) }
             }))
         }
     });

compute_tools/src/lib.rs

@@ -2,7 +2,6 @@
 //! Various tools and helpers to handle cluster / compute node (Postgres)
 //! configuration.
 //!
-pub mod checker;
 pub mod config;
 pub mod http_api;
 #[macro_use]

compute_tools/src/pg_helpers.rs

@@ -132,14 +132,7 @@ impl Role {
         let mut params: String = "LOGIN".to_string();
 
         if let Some(pass) = &self.encrypted_password {
-            // Some time ago we supported only md5 and treated all encrypted_password as md5.
-            // Now we also support SCRAM-SHA-256 and to preserve compatibility
-            // we treat all encrypted_password as md5 unless they starts with SCRAM-SHA-256.
-            if pass.starts_with("SCRAM-SHA-256") {
-                params.push_str(&format!(" PASSWORD '{}'", pass));
-            } else {
-                params.push_str(&format!(" PASSWORD 'md5{}'", pass));
-            }
+            params.push_str(&format!(" PASSWORD 'md5{}'", pass));
         } else {
             params.push_str(" PASSWORD NULL");
         }

compute_tools/src/spec.rs

@@ -244,24 +244,3 @@ pub fn handle_databases(spec: &ClusterSpec, client: &mut Client) -> Result<()> {
 
     Ok(())
 }
-
-// Grant CREATE ON DATABASE to the database owner
-// to allow clients create trusted extensions.
-pub fn handle_grants(spec: &ClusterSpec, client: &mut Client) -> Result<()> {
-    info!("cluster spec grants:");
-
-    for db in &spec.cluster.databases {
-        let dbname = &db.name;
-
-        let query: String = format!(
-            "GRANT CREATE ON DATABASE {} TO {}",
-            dbname.quote(),
-            db.owner.quote()
-        );
-        info!("grant query {}", &query);
-
-        client.execute(query.as_str(), &[])?;
-    }
-
-    Ok(())
-}

control_plane/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 
 [dependencies]
 tar = "0.4.33"
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
 serde = { version = "1.0", features = ["derive"] }
 serde_with = "1.12.0"
 toml = "0.5"
@@ -18,6 +18,6 @@ url = "2.2.2"
 reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] }
 
 pageserver = { path = "../pageserver" }
-safekeeper = { path = "../safekeeper" }
-utils = { path = "../libs/utils" }
+walkeeper = { path = "../walkeeper" }
+zenith_utils = { path = "../zenith_utils" }
 workspace_hack = { version = "0.1", path = "../workspace_hack" }

control_plane/src/compute.rs

@@ -11,12 +11,11 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use anyhow::{Context, Result};
-use utils::{
-    connstring::connection_host_port,
-    lsn::Lsn,
-    postgres_backend::AuthType,
-    zid::{ZTenantId, ZTimelineId},
-};
+use zenith_utils::connstring::connection_host_port;
+use zenith_utils::lsn::Lsn;
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::ZTenantId;
+use zenith_utils::zid::ZTimelineId;
 
 use crate::local_env::LocalEnv;
 use crate::postgresql_conf::PostgresConf;
@@ -273,7 +272,12 @@ impl PostgresNode {
         conf.append("wal_sender_timeout", "5s");
         conf.append("listen_addresses", &self.address.ip().to_string());
         conf.append("port", &self.address.port().to_string());
-        conf.append("wal_keep_size", "0");
+
+        // Never clean up old WAL. TODO: We should use a replication
+        // slot or something proper, to prevent the compute node
+        // from removing WAL that hasn't been streamed to the safekeeper or
+        // page server yet. (gh issue #349)
+        conf.append("wal_keep_size", "10TB");
 
         // Configure the node to fetch pages from pageserver
         let pageserver_connstr = {
@@ -327,14 +331,14 @@ impl PostgresNode {
             // Configure the node to connect to the safekeepers
             conf.append("synchronous_standby_names", "walproposer");
 
-            let safekeepers = self
+            let wal_acceptors = self
                 .env
                 .safekeepers
                 .iter()
                 .map(|sk| format!("localhost:{}", sk.pg_port))
                 .collect::<Vec<String>>()
                 .join(",");
-            conf.append("wal_acceptors", &safekeepers);
+            conf.append("wal_acceptors", &wal_acceptors);
         } else {
             // We only use setup without safekeepers for tests,
             // and don't care about data durability on pageserver,
@@ -416,15 +420,10 @@ impl PostgresNode {
         if let Some(token) = auth_token {
             cmd.env("ZENITH_AUTH_TOKEN", token);
         }
+        let pg_ctl = cmd.status().context("pg_ctl failed")?;
 
-        let pg_ctl = cmd.output().context("pg_ctl failed")?;
-        if !pg_ctl.status.success() {
-            anyhow::bail!(
-                "pg_ctl failed, exit code: {}, stdout: {}, stderr: {}",
-                pg_ctl.status,
-                String::from_utf8_lossy(&pg_ctl.stdout),
-                String::from_utf8_lossy(&pg_ctl.stderr),
-            );
+        if !pg_ctl.success() {
+            anyhow::bail!("pg_ctl failed");
         }
         Ok(())
     }

control_plane/src/local_env.rs

@@ -11,11 +11,9 @@ use std::env;
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
-use utils::{
-    auth::{encode_from_key_file, Claims, Scope},
-    postgres_backend::AuthType,
-    zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId},
-};
+use zenith_utils::auth::{encode_from_key_file, Claims, Scope};
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId};
 
 use crate::safekeeper::SafekeeperNode;
 

control_plane/src/safekeeper.rs

@@ -13,17 +13,15 @@ use nix::unistd::Pid;
 use postgres::Config;
 use reqwest::blocking::{Client, RequestBuilder, Response};
 use reqwest::{IntoUrl, Method};
-use safekeeper::http::models::TimelineCreateRequest;
 use thiserror::Error;
-use utils::{
-    connstring::connection_address,
-    http::error::HttpErrorBody,
-    zid::{ZNodeId, ZTenantId, ZTimelineId},
-};
+use walkeeper::http::models::TimelineCreateRequest;
+use zenith_utils::http::error::HttpErrorBody;
+use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId};
 
 use crate::local_env::{LocalEnv, SafekeeperConf};
 use crate::storage::PageServerNode;
 use crate::{fill_rust_env_vars, read_pidfile};
+use zenith_utils::connstring::connection_address;
 
 #[derive(Error, Debug)]
 pub enum SafekeeperHttpError {

control_plane/src/storage.rs

@@ -10,23 +10,21 @@ use anyhow::{bail, Context};
 use nix::errno::Errno;
 use nix::sys::signal::{kill, Signal};
 use nix::unistd::Pid;
-use pageserver::http::models::{TenantConfigRequest, TenantCreateRequest, TimelineCreateRequest};
+use pageserver::http::models::{TenantCreateRequest, TimelineCreateRequest};
 use pageserver::timelines::TimelineInfo;
 use postgres::{Config, NoTls};
 use reqwest::blocking::{Client, RequestBuilder, Response};
 use reqwest::{IntoUrl, Method};
 use thiserror::Error;
-use utils::{
-    connstring::connection_address,
-    http::error::HttpErrorBody,
-    lsn::Lsn,
-    postgres_backend::AuthType,
-    zid::{ZTenantId, ZTimelineId},
-};
+use zenith_utils::http::error::HttpErrorBody;
+use zenith_utils::lsn::Lsn;
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
 use crate::local_env::LocalEnv;
 use crate::{fill_rust_env_vars, read_pidfile};
 use pageserver::tenant_mgr::TenantInfo;
+use zenith_utils::connstring::connection_address;
 
 #[derive(Error, Debug)]
 pub enum PageserverHttpError {
@@ -353,21 +351,14 @@ impl PageServerNode {
                 new_tenant_id,
                 checkpoint_distance: settings
                     .get("checkpoint_distance")
-                    .map(|x| x.parse::<u64>())
-                    .transpose()?,
+                    .map(|x| x.parse::<u64>().unwrap()),
                 compaction_target_size: settings
                     .get("compaction_target_size")
-                    .map(|x| x.parse::<u64>())
-                    .transpose()?,
+                    .map(|x| x.parse::<u64>().unwrap()),
                 compaction_period: settings.get("compaction_period").map(|x| x.to_string()),
-                compaction_threshold: settings
-                    .get("compaction_threshold")
-                    .map(|x| x.parse::<usize>())
-                    .transpose()?,
                 gc_horizon: settings
                     .get("gc_horizon")
-                    .map(|x| x.parse::<u64>())
-                    .transpose()?,
+                    .map(|x| x.parse::<u64>().unwrap()),
                 gc_period: settings.get("gc_period").map(|x| x.to_string()),
                 pitr_interval: settings.get("pitr_interval").map(|x| x.to_string()),
             })
@@ -387,32 +378,6 @@ impl PageServerNode {
             .transpose()
     }
 
-    pub fn tenant_config(&self, tenant_id: ZTenantId, settings: HashMap<&str, &str>) -> Result<()> {
-        self.http_request(Method::PUT, format!("{}/tenant/config", self.http_base_url))
-            .json(&TenantConfigRequest {
-                tenant_id,
-                checkpoint_distance: settings
-                    .get("checkpoint_distance")
-                    .map(|x| x.parse::<u64>().unwrap()),
-                compaction_target_size: settings
-                    .get("compaction_target_size")
-                    .map(|x| x.parse::<u64>().unwrap()),
-                compaction_period: settings.get("compaction_period").map(|x| x.to_string()),
-                compaction_threshold: settings
-                    .get("compaction_threshold")
-                    .map(|x| x.parse::<usize>().unwrap()),
-                gc_horizon: settings
-                    .get("gc_horizon")
-                    .map(|x| x.parse::<u64>().unwrap()),
-                gc_period: settings.get("gc_period").map(|x| x.to_string()),
-                pitr_interval: settings.get("pitr_interval").map(|x| x.to_string()),
-            })
-            .send()?
-            .error_from_body()?;
-
-        Ok(())
-    }
-
     pub fn timeline_list(&self, tenant_id: &ZTenantId) -> anyhow::Result<Vec<TimelineInfo>> {
         let timeline_infos: Vec<TimelineInfo> = self
             .http_request(

docs/README.md

@@ -8,7 +8,7 @@
 - [multitenancy.md](multitenancy.md) — how multitenancy is organized in the pageserver and Zenith CLI.
 - [sourcetree.md](sourcetree.md) — Overview of the source tree layeout.
 - [pageserver/README](/pageserver/README) — pageserver overview.
-- [postgres_ffi/README](/libs/postgres_ffi/README) — Postgres FFI overview.
+- [postgres_ffi/README](/postgres_ffi/README) — Postgres FFI overview.
 - [test_runner/README.md](/test_runner/README.md) — tests infrastructure overview.
-- [safekeeper/README](/safekeeper/README) — WAL service overview.
+- [walkeeper/README](/walkeeper/README) — WAL service overview.
 - [core_changes.md](core_changes.md) - Description of Zenith changes in Postgres core

docs/authentication.md

@@ -27,4 +27,4 @@ management_token = jwt.encode({"scope": "pageserverapi"}, auth_keys.priv, algori
 tenant_token = jwt.encode({"scope": "tenant", "tenant_id": ps.initial_tenant}, auth_keys.priv, algorithm="RS256")
 ```
 
-Utility functions to work with jwts in rust are located in libs/utils/src/auth.rs
+Utility functions to work with jwts in rust are located in zenith_utils/src/auth.rs

docs/glossary.md

@@ -29,7 +29,7 @@ Each Branch lives in a corresponding timeline[] and has an ancestor[].
 
 NOTE: This is an overloaded term.
 
-A checkpoint record in the WAL marks a point in the WAL sequence at which it is guaranteed that all data files have been updated with all information from shared memory modified before that checkpoint;
+A checkpoint record in the WAL marks a point in the WAL sequence at which it is guaranteed that all data files have been updated with all information from shared memory modified before that checkpoint; 
 
 ### Checkpoint (Layered repository)
 
@@ -108,10 +108,10 @@ PostgreSQL LSNs and functions to monitor them:
 * `pg_current_wal_lsn()` - Returns the current write-ahead log write location.
 * `pg_current_wal_flush_lsn()` - Returns the current write-ahead log flush location.
 * `pg_last_wal_receive_lsn()` - Returns the last write-ahead log location that has been received and synced to disk by streaming replication. While streaming replication is in progress this will increase monotonically.
-* `pg_last_wal_replay_lsn ()` - Returns the last write-ahead log location that has been replayed during recovery. If recovery is still in progress this will increase monotonically.
+* `pg_last_wal_replay_lsn ()` - Returns the last write-ahead log location that has been replayed during recovery. If recovery is still in progress this will increase monotonically. 
 [source PostgreSQL documentation](https://www.postgresql.org/docs/devel/functions-admin.html):
 
-Zenith safekeeper LSNs. For more check [safekeeper/README_PROTO.md](/safekeeper/README_PROTO.md)
+Zenith safekeeper LSNs. For more check [walkeeper/README_PROTO.md](/walkeeper/README_PROTO.md)
 * `CommitLSN`: position in WAL confirmed by quorum safekeepers.
 * `RestartLSN`: position in WAL confirmed by all safekeepers.
 * `FlushLSN`: part of WAL persisted to the disk by safekeeper.
@@ -190,7 +190,7 @@ or we do not support them in zenith yet (pg_commit_ts).
 Tenant represents a single customer, interacting with Zenith.
 Wal redo[] activity, timelines[], layers[] are managed for each tenant independently.
 One pageserver[] can serve multiple tenants at once.
-One safekeeper
+One safekeeper 
 
 See `docs/multitenancy.md` for more.
 

docs/rfcs/009-snapshot-first-storage-cli.md

@@ -12,7 +12,7 @@ Init empty pageserver using `initdb` in temporary directory.
 
 `--storage_dest=FILE_PREFIX | S3_PREFIX |...` option defines object storage type, all other parameters are passed via env variables. Inspired by WAL-G style naming : https://wal-g.readthedocs.io/STORAGES/.
 
-Save`storage_dest` and other parameters in config.
+Save`storage_dest` and other parameters in config. 
 Push snapshots to `storage_dest` in background.
 
 ```
@@ -21,7 +21,7 @@ zenith start
 ```
 
 #### 2. Restart pageserver (manually or crash-recovery).
-Take `storage_dest` from pageserver config, start pageserver from latest snapshot in `storage_dest`.
+Take `storage_dest` from pageserver config, start pageserver from latest snapshot in `storage_dest`. 
 Push snapshots to `storage_dest` in background.
 
 ```
@@ -32,7 +32,7 @@ zenith start
 Start pageserver from existing snapshot.
 Path to snapshot provided via `--snapshot_path=FILE_PREFIX | S3_PREFIX | ...`
 Do not save `snapshot_path` and `snapshot_format` in config, as it is a one-time operation.
-Save`storage_dest` parameters in config.
+Save`storage_dest` parameters in config. 
 Push snapshots to `storage_dest` in background.
 ```
 //I.e. we want to start zenith on top of existing $PGDATA and use s3 as a persistent storage.
@@ -42,15 +42,15 @@ zenith start
 How to pass credentials needed for `snapshot_path`?
 
 #### 4. Export.
-Manually push snapshot to `snapshot_path` which differs from `storage_dest`
+Manually push snapshot to `snapshot_path` which differs from `storage_dest` 
 Optionally set `snapshot_format`, which can be plain pgdata format or zenith format.
 ```
 zenith export --snapshot_path=FILE_PREFIX --snapshot_format=pgdata
 ```
 
 #### Notes and questions
-- safekeeper s3_offload should use same (similar) syntax for storage. How to set it in UI?
+- walkeeper s3_offload should use same (similar) syntax for storage. How to set it in UI?
 - Why do we need `zenith init` as a separate command? Can't we init everything at first start?
 - We can think of better names for all options.
 - Export to plain postgres format will be useless, if we are not 100% compatible on page level.
-I can recall at least one such difference - PD_WAL_LOGGED flag in pages.
+I can recall at least one such difference - PD_WAL_LOGGED flag in pages.

docs/rfcs/cluster-size-limits.md

@@ -1,79 +0,0 @@
-Cluster size limits
-==================
-
-## Summary
-
-One of the resource consumption limits for free-tier users is a cluster size limit.
-
-To enforce it, we need to calculate the timeline size and check if the limit is reached before relation create/extend operations.
-If the limit is reached, the query must fail with some meaningful error/warning.
-We may want to exempt some operations from the quota to allow users free space to fit back into the limit.
-
-The stateless compute node that performs validation is separate from the storage that calculates the usage, so we need to exchange cluster size information between those components.
-
-## Motivation
-
-Limit the maximum size of a PostgreSQL instance to limit free tier users (and other tiers in the future).
-First of all, this is needed to control our free tier production costs.
-Another reason to limit resources is risk management — we haven't (fully) tested and optimized zenith for big clusters,
-so we don't want to give users access to the functionality that we don't think is ready.
-
-## Components
-
-* pageserver - calculate the size consumed by a timeline and add it to the feedback message.
-* safekeeper - pass feedback message from pageserver to compute.
-* compute - receive feedback message, enforce size limit based on GUC `zenith.max_cluster_size`.
-* console - set and update `zenith.max_cluster_size` setting
-
-## Proposed implementation
-
-First of all, it's necessary to define timeline size.
-
-The current approach is to count all data, including SLRUs. (not including WAL)
-Here we think of it as a physical disk underneath the Postgres cluster.
-This is how the `LOGICAL_TIMELINE_SIZE` metric is implemented in the pageserver.
-
-Alternatively, we could count only relation data. As in pg_database_size().
-This approach is somewhat more user-friendly because it is the data that is really affected by the user.
-On the other hand, it puts us in a weaker position than other services, i.e., RDS.
-We will need to refactor the timeline_size counter or add another counter to implement it. 
-
-Timeline size is updated during wal digestion. It is not versioned and is valid at the last_received_lsn moment.
-Then this size should be reported to compute node.
-
-`current_timeline_size` value is included in the walreceiver's custom feedback message: `ZenithFeedback.`
-
-(PR about protocol changes https://github.com/zenithdb/zenith/pull/1037).
-
-This message is received by the safekeeper and propagated to compute node as a part of `AppendResponse`.
-
-Finally, when compute node receives the `current_timeline_size` from safekeeper (or from pageserver directly), it updates the global variable.
-
-And then every zenith_extend() operation checks if limit is reached `(current_timeline_size > zenith.max_cluster_size)` and throws `ERRCODE_DISK_FULL` error if so.
-(see Postgres error codes [https://www.postgresql.org/docs/devel/errcodes-appendix.html](https://www.postgresql.org/docs/devel/errcodes-appendix.html))
-
-TODO:
-We can allow autovacuum processes to bypass this check, simply checking `IsAutoVacuumWorkerProcess()`.
-It would be nice to allow manual VACUUM and VACUUM FULL to bypass the check, but it's uneasy to distinguish these operations at the low level.
-See issues https://github.com/neondatabase/neon/issues/1245
-https://github.com/zenithdb/zenith/issues/1445
-
-TODO:
-We should warn users if the limit is soon to be reached.
-
-### **Reliability, failure modes and corner cases**
-
-1. `current_timeline_size` is valid at the last received and digested by pageserver lsn.
-    
-    If pageserver lags behind compute node, `current_timeline_size` will lag too. This lag can be tuned using backpressure, but it is not expected to be 0 all the time.
-    
-    So transactions that happen in this lsn range may cause limit overflow. Especially operations that generate (i.e., CREATE DATABASE) or free (i.e., TRUNCATE) a lot of data pages while generating a small amount of WAL. Are there other operations like this?
-    
-    Currently, CREATE DATABASE operations are restricted in the console. So this is not an issue.
-
-
-### **Security implications**
-
-We treat compute as an untrusted component. That's why we try to isolate it with secure container runtime or a VM.
-Malicious users may change the `zenith.max_cluster_size`, so we need an extra size limit check.
-To cover this case, we also monitor the compute node size in the console.

docs/settings.md

@@ -156,9 +156,6 @@ access_key_id = 'SOMEKEYAAAAASADSAH*#'
 
 # Secret access key to connect to the bucket ("password" part of the credentials)
 secret_access_key = 'SOMEsEcReTsd292v'
-
-# S3 API query limit to avoid getting errors/throttling from AWS.
-concurrency_limit = 100
 ```
 
 ###### General remote storage configuration
@@ -170,8 +167,8 @@ Besides, there are parameters common for all types of remote storage that can be
 
 ```toml
 [remote_storage]
-# Max number of concurrent timeline synchronized (layers uploaded or downloaded) with the remote storage at the same time.
-max_concurrent_timelines_sync = 50
+# Max number of concurrent connections to open for uploading to or downloading from the remote storage.
+max_concurrent_sync = 100
 
 # Max number of errors a single task can have before it's considered failed and not attempted to run anymore.
 max_sync_errors = 10

docs/sourcetree.md

@@ -30,6 +30,11 @@ The pageserver has a few different duties:
 
 For more detailed info, see `/pageserver/README`
 
+`/postgres_ffi`:
+
+Utility functions for interacting with PostgreSQL file formats.
+Misc constants, copied from PostgreSQL headers.
+
 `/proxy`:
 
 Postgres protocol proxy/router.
@@ -52,12 +57,12 @@ PostgreSQL extension that implements storage manager API and network communicati
 
 PostgreSQL extension that contains functions needed for testing and debugging.
 
-`/safekeeper`:
+`/walkeeper`:
 
 The zenith WAL service that receives WAL from a primary compute nodes and streams it to the pageserver.
 It acts as a holding area and redistribution center for recently generated WAL.
 
-For more detailed info, see `/safekeeper/README`
+For more detailed info, see `/walkeeper/README`
 
 `/workspace_hack`:
 The workspace_hack crate exists only to pin down some dependencies.
@@ -69,21 +74,14 @@ We use [cargo-hakari](https://crates.io/crates/cargo-hakari) for automation.
 Main entry point for the 'zenith' CLI utility.
 TODO: Doesn't it belong to control_plane?
 
-`/libs`:
-Unites granular neon helper crates under the hood.
+`/zenith_metrics`:
 
-`/libs/postgres_ffi`:
-
-Utility functions for interacting with PostgreSQL file formats.
-Misc constants, copied from PostgreSQL headers.
-
-`/libs/utils`:
-Generic helpers that are shared between other crates in this repository.
-A subject for future modularization.
-
-`/libs/metrics`:
 Helpers for exposing Prometheus metrics from the server.
 
+`/zenith_utils`:
+
+Helpers that are shared between other crates in this repository.
+
 ## Using Python
 Note that Debian/Ubuntu Python packages are stale, as it commonly happens,
 so manual installation of dependencies is not recommended.

libs/utils/scripts/restore_from_wal.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-PG_BIN=$1
-WAL_PATH=$2
-DATA_DIR=$3
-PORT=$4
-SYSID=`od -A n -j 24 -N 8 -t d8 $WAL_PATH/000000010000000000000002* | cut -c 3-`
-rm -fr $DATA_DIR
-env -i LD_LIBRARY_PATH=$PG_BIN/../lib $PG_BIN/initdb -E utf8 -U zenith_admin -D $DATA_DIR --sysid=$SYSID
-echo port=$PORT >> $DATA_DIR/postgresql.conf
-REDO_POS=0x`$PG_BIN/pg_controldata -D $DATA_DIR | fgrep "REDO location"| cut -c 42-`
-declare -i WAL_SIZE=$REDO_POS+114
-$PG_BIN/pg_ctl -D $DATA_DIR -l logfile start
-$PG_BIN/pg_ctl -D $DATA_DIR -l logfile stop -m immediate
-cp $DATA_DIR/pg_wal/000000010000000000000001 .
-cp $WAL_PATH/* $DATA_DIR/pg_wal/
-if [ -f $DATA_DIR/pg_wal/*.partial ]
-then
-	(cd $DATA_DIR/pg_wal ; for partial in \*.partial ; do mv $partial `basename $partial .partial` ; done)
-fi
-dd if=000000010000000000000001 of=$DATA_DIR/pg_wal/000000010000000000000001 bs=$WAL_SIZE count=1 conv=notrunc
-rm -f 000000010000000000000001

libs/utils/scripts/restore_from_wal_archive.sh

@@ -1,20 +0,0 @@
-PG_BIN=$1
-WAL_PATH=$2
-DATA_DIR=$3
-PORT=$4
-SYSID=`od -A n -j 24 -N 8 -t d8 $WAL_PATH/000000010000000000000002* | cut -c 3-`
-rm -fr $DATA_DIR /tmp/pg_wals
-mkdir /tmp/pg_wals
-env -i LD_LIBRARY_PATH=$PG_BIN/../lib $PG_BIN/initdb -E utf8 -U zenith_admin -D $DATA_DIR --sysid=$SYSID
-echo port=$PORT >> $DATA_DIR/postgresql.conf
-REDO_POS=0x`$PG_BIN/pg_controldata -D $DATA_DIR | fgrep "REDO location"| cut -c 42-`
-declare -i WAL_SIZE=$REDO_POS+114
-cp $WAL_PATH/* /tmp/pg_wals
-if [ -f $DATA_DIR/pg_wal/*.partial ]
-then
-	(cd /tmp/pg_wals ; for partial in \*.partial ; do  mv $partial `basename $partial .partial` ; done)
-fi
-dd if=$DATA_DIR/pg_wal/000000010000000000000001 of=/tmp/pg_wals/000000010000000000000001 bs=$WAL_SIZE count=1 conv=notrunc
-echo > $DATA_DIR/recovery.signal
-rm -f $DATA_DIR/pg_wal/*
-echo "restore_command = 'cp /tmp/pg_wals/%f %p'" >> $DATA_DIR/postgresql.conf

pageserver/Cargo.toml

@@ -3,10 +3,6 @@ name = "pageserver"
 version = "0.1.0"
 edition = "2021"
 
-[features]
-default = []
-profiling = ["pprof"]
-
 [dependencies]
 chrono = "0.4.19"
 rand = "0.8.3"
@@ -18,14 +14,14 @@ hex = "0.4.3"
 hyper = "0.14"
 itertools = "0.10.3"
 lazy_static = "1.4.0"
-clap = { version = "3.1.8", features = ["derive"] }
+log = "0.4.14"
+clap = "3.0"
 daemonize = "0.4.1"
 tokio = { version = "1.17", features = ["process", "sync", "macros", "fs", "rt", "io-util", "time"] }
-tokio-util = { version = "0.7", features = ["io"] }
-postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
+postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
+postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
+tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
 tokio-stream = "0.1.8"
 anyhow = { version = "1.0", features = ["backtrace"] }
 crc32c = "0.6.0"
@@ -36,12 +32,12 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 serde_with = "1.12.0"
 
-pprof = { git = "https://github.com/neondatabase/pprof-rs.git", branch = "wallclock-profiling", features = ["flamegraph"], optional = true }
-
 toml_edit = { version = "0.13", features = ["easy"] }
 scopeguard = "1.1.0"
+async-trait = "0.1"
 const_format = "0.2.21"
 tracing = "0.1.27"
+tracing-futures = "0.2"
 signal-hook = "0.3.10"
 url = "2"
 nix = "0.23"
@@ -49,13 +45,12 @@ once_cell = "1.8.0"
 crossbeam-utils = "0.8.5"
 fail = "0.5.0"
 
-rusoto_core = "0.47"
-rusoto_s3 = "0.47"
-async-trait = "0.1"
+rust-s3 = { version = "0.28", default-features = false, features = ["no-verify-ssl", "tokio-rustls-tls"] }
+async-compression = {version = "0.3", features = ["zstd", "tokio"]}
 
-postgres_ffi = { path = "../libs/postgres_ffi" }
-metrics = { path = "../libs/metrics" }
-utils = { path = "../libs/utils" }
+postgres_ffi = { path = "../postgres_ffi" }
+zenith_metrics = { path = "../zenith_metrics" }
+zenith_utils = { path = "../zenith_utils" }
 workspace_hack = { version = "0.1", path = "../workspace_hack" }
 
 [dev-dependencies]

pageserver/src/basebackup.rs

@@ -12,20 +12,20 @@
 //!
 use anyhow::{ensure, Context, Result};
 use bytes::{BufMut, BytesMut};
+use log::*;
 use std::fmt::Write as FmtWrite;
 use std::io;
 use std::io::Write;
 use std::sync::Arc;
 use std::time::SystemTime;
 use tar::{Builder, EntryType, Header};
-use tracing::*;
 
 use crate::reltag::SlruKind;
 use crate::repository::Timeline;
 use crate::DatadirTimelineImpl;
 use postgres_ffi::xlog_utils::*;
 use postgres_ffi::*;
-use utils::lsn::Lsn;
+use zenith_utils::lsn::Lsn;
 
 /// This is short-living object only for the time of tarball creation,
 /// created mostly to avoid passing a lot of parameters between various functions

pageserver/src/bin/dump_layerfile.rs

@@ -2,15 +2,15 @@
 //!
 //! A handy tool for debugging, that's all.
 use anyhow::Result;
-use clap::{Arg, Command};
+use clap::{App, Arg};
 use pageserver::layered_repository::dump_layerfile_from_path;
 use pageserver::page_cache;
 use pageserver::virtual_file;
 use std::path::PathBuf;
-use utils::GIT_VERSION;
+use zenith_utils::GIT_VERSION;
 
 fn main() -> Result<()> {
-    let arg_matches = Command::new("Zenith dump_layerfile utility")
+    let arg_matches = App::new("Zenith dump_layerfile utility")
         .about("Dump contents of one layer file, for debugging")
         .version(GIT_VERSION)
         .arg(

pageserver/src/bin/pageserver.rs

@@ -2,42 +2,38 @@
 
 use std::{env, path::Path, str::FromStr};
 use tracing::*;
+use zenith_utils::{
+    auth::JwtAuth,
+    logging,
+    postgres_backend::AuthType,
+    tcp_listener,
+    zid::{ZTenantId, ZTimelineId},
+    GIT_VERSION,
+};
 
 use anyhow::{bail, Context, Result};
 
-use clap::{Arg, Command};
+use clap::{App, Arg};
 use daemonize::Daemonize;
 
 use pageserver::{
     config::{defaults::*, PageServerConf},
-    http, page_cache, page_service, profiling,
+    http, page_cache, page_service,
     remote_storage::{self, SyncStartupData},
     repository::{Repository, TimelineSyncStatusUpdate},
     tenant_mgr, thread_mgr,
     thread_mgr::ThreadKind,
     timelines, virtual_file, LOG_FILE_NAME,
 };
-use utils::{
-    auth::JwtAuth,
-    http::endpoint,
-    logging,
-    postgres_backend::AuthType,
-    shutdown::exit_now,
-    signals::{self, Signal},
-    tcp_listener,
-    zid::{ZTenantId, ZTimelineId},
-    GIT_VERSION,
-};
-
-fn version() -> String {
-    format!("{} profiling:{}", GIT_VERSION, cfg!(feature = "profiling"))
-}
+use zenith_utils::http::endpoint;
+use zenith_utils::shutdown::exit_now;
+use zenith_utils::signals::{self, Signal};
 
 fn main() -> anyhow::Result<()> {
-    metrics::set_common_metrics_prefix("pageserver");
-    let arg_matches = Command::new("Zenith page server")
+    zenith_metrics::set_common_metrics_prefix("pageserver");
+    let arg_matches = App::new("Zenith page server")
         .about("Materializes WAL stream to pages and serves them to the postgres")
-        .version(&*version())
+        .version(GIT_VERSION)
         .arg(
             Arg::new("daemonize")
                 .short('d')
@@ -189,9 +185,6 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
     // Initialize logger
     let log_file = logging::init(LOG_FILE_NAME, daemonize)?;
 
-    // Initialize wal metadata logger, if necessary
-    pageserver::wal_metadata::init(conf).expect("wal_metadata init failed");
-
     info!("version: {}", GIT_VERSION);
 
     // TODO: Check that it looks like a valid repository before going further
@@ -249,12 +242,11 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
 
     for (tenant_id, local_timeline_init_statuses) in local_timeline_init_statuses {
         // initialize local tenant
-        let repo = tenant_mgr::load_local_repo(conf, tenant_id, &remote_index)
-            .with_context(|| format!("Failed to load repo for tenant {}", tenant_id))?;
+        let repo = tenant_mgr::load_local_repo(conf, tenant_id, &remote_index);
         for (timeline_id, init_status) in local_timeline_init_statuses {
             match init_status {
                 remote_storage::LocalTimelineInitStatus::LocallyComplete => {
-                    debug!("timeline {} for tenant {} is locally complete, registering it in repository", timeline_id, tenant_id);
+                    debug!("timeline {} for tenant {} is locally complete, registering it in repository", tenant_id, timeline_id);
                     // Lets fail here loudly to be on the safe side.
                     // XXX: It may be a better api to actually distinguish between repository startup
                     //   and processing of newly downloaded timelines.
@@ -291,9 +283,6 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
     };
     info!("Using auth: {:#?}", conf.auth_type);
 
-    // start profiler (if enabled)
-    let profiler_guard = profiling::init_profiler(conf);
-
     // Spawn a new thread for the http endpoint
     // bind before launching separate thread so the error reported before startup exits
     let auth_cloned = auth.clone();
@@ -304,7 +293,7 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
         "http_endpoint_thread",
         false,
         move || {
-            let router = http::make_router(conf, auth_cloned, remote_index)?;
+            let router = http::make_router(conf, auth_cloned, remote_index);
             endpoint::serve_thread_main(router, http_listener, thread_mgr::shutdown_watcher())
         },
     )?;
@@ -326,7 +315,6 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
                 "Got {}. Terminating in immediate shutdown mode",
                 signal.name()
             );
-            profiling::exit_profiler(conf, &profiler_guard);
             std::process::exit(111);
         }
 
@@ -335,7 +323,6 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
                 "Got {}. Terminating gracefully in fast shutdown mode",
                 signal.name()
             );
-            profiling::exit_profiler(conf, &profiler_guard);
             pageserver::shutdown_pageserver();
             unreachable!()
         }

pageserver/src/bin/pageserver_zst.rs

@@ -0,0 +1,334 @@
+//! A CLI helper to deal with remote storage (S3, usually) blobs as archives.
+//! See [`compression`] for more details about the archives.
+
+use std::{collections::BTreeSet, path::Path};
+
+use anyhow::{bail, ensure, Context};
+use clap::{App, Arg};
+use pageserver::{
+    layered_repository::metadata::{TimelineMetadata, METADATA_FILE_NAME},
+    remote_storage::compression,
+};
+use tokio::{fs, io};
+use zenith_utils::GIT_VERSION;
+
+const LIST_SUBCOMMAND: &str = "list";
+const ARCHIVE_ARG_NAME: &str = "archive";
+
+const EXTRACT_SUBCOMMAND: &str = "extract";
+const TARGET_DIRECTORY_ARG_NAME: &str = "target_directory";
+
+const CREATE_SUBCOMMAND: &str = "create";
+const SOURCE_DIRECTORY_ARG_NAME: &str = "source_directory";
+
+#[tokio::main(flavor = "current_thread")]
+async fn main() -> anyhow::Result<()> {
+    let arg_matches = App::new("pageserver zst blob [un]compressor utility")
+        .version(GIT_VERSION)
+        .subcommands(vec![
+            App::new(LIST_SUBCOMMAND)
+                .about("List the archive contents")
+                .arg(
+                    Arg::new(ARCHIVE_ARG_NAME)
+                        .required(true)
+                        .takes_value(true)
+                        .help("An archive to list the contents of"),
+                ),
+            App::new(EXTRACT_SUBCOMMAND)
+                .about("Extracts the archive into the directory")
+                .arg(
+                    Arg::new(ARCHIVE_ARG_NAME)
+                        .required(true)
+                        .takes_value(true)
+                        .help("An archive to extract"),
+                )
+                .arg(
+                    Arg::new(TARGET_DIRECTORY_ARG_NAME)
+                        .required(false)
+                        .takes_value(true)
+                        .help("A directory to extract the archive into. Optional, will use the current directory if not specified"),
+                ),
+            App::new(CREATE_SUBCOMMAND)
+                .about("Creates an archive with the contents of a directory (only the first level files are taken, metadata file has to be present in the same directory)")
+                .arg(
+                    Arg::new(SOURCE_DIRECTORY_ARG_NAME)
+                        .required(true)
+                        .takes_value(true)
+                        .help("A directory to use for creating the archive"),
+                )
+                .arg(
+                    Arg::new(TARGET_DIRECTORY_ARG_NAME)
+                        .required(false)
+                        .takes_value(true)
+                        .help("A directory to create the archive in. Optional, will use the current directory if not specified"),
+                ),
+        ])
+        .get_matches();
+
+    let subcommand_name = match arg_matches.subcommand_name() {
+        Some(name) => name,
+        None => bail!("No subcommand specified"),
+    };
+
+    let subcommand_matches = match arg_matches.subcommand_matches(subcommand_name) {
+        Some(matches) => matches,
+        None => bail!(
+            "No subcommand arguments were recognized for subcommand '{}'",
+            subcommand_name
+        ),
+    };
+
+    let target_dir = Path::new(
+        subcommand_matches
+            .value_of(TARGET_DIRECTORY_ARG_NAME)
+            .unwrap_or("./"),
+    );
+
+    match subcommand_name {
+        LIST_SUBCOMMAND => {
+            let archive = match subcommand_matches.value_of(ARCHIVE_ARG_NAME) {
+                Some(archive) => Path::new(archive),
+                None => bail!("No '{}' argument is specified", ARCHIVE_ARG_NAME),
+            };
+            list_archive(archive).await
+        }
+        EXTRACT_SUBCOMMAND => {
+            let archive = match subcommand_matches.value_of(ARCHIVE_ARG_NAME) {
+                Some(archive) => Path::new(archive),
+                None => bail!("No '{}' argument is specified", ARCHIVE_ARG_NAME),
+            };
+            extract_archive(archive, target_dir).await
+        }
+        CREATE_SUBCOMMAND => {
+            let source_dir = match subcommand_matches.value_of(SOURCE_DIRECTORY_ARG_NAME) {
+                Some(source) => Path::new(source),
+                None => bail!("No '{}' argument is specified", SOURCE_DIRECTORY_ARG_NAME),
+            };
+            create_archive(source_dir, target_dir).await
+        }
+        unknown => bail!("Unknown subcommand {}", unknown),
+    }
+}
+
+async fn list_archive(archive: &Path) -> anyhow::Result<()> {
+    let archive = archive.canonicalize().with_context(|| {
+        format!(
+            "Failed to get the absolute path for the archive path '{}'",
+            archive.display()
+        )
+    })?;
+    ensure!(
+        archive.is_file(),
+        "Path '{}' is not an archive file",
+        archive.display()
+    );
+    println!("Listing an archive at path '{}'", archive.display());
+    let archive_name = match archive.file_name().and_then(|name| name.to_str()) {
+        Some(name) => name,
+        None => bail!(
+            "Failed to get the archive name from the path '{}'",
+            archive.display()
+        ),
+    };
+
+    let archive_bytes = fs::read(&archive)
+        .await
+        .context("Failed to read the archive bytes")?;
+
+    let header = compression::read_archive_header(archive_name, &mut archive_bytes.as_slice())
+        .await
+        .context("Failed to read the archive header")?;
+
+    let empty_path = Path::new("");
+    println!("-------------------------------");
+
+    let longest_path_in_archive = header
+        .files
+        .iter()
+        .filter_map(|file| Some(file.subpath.as_path(empty_path).to_str()?.len()))
+        .max()
+        .unwrap_or_default()
+        .max(METADATA_FILE_NAME.len());
+
+    for regular_file in &header.files {
+        println!(
+            "File: {:width$} uncompressed size: {} bytes",
+            regular_file.subpath.as_path(empty_path).display(),
+            regular_file.size,
+            width = longest_path_in_archive,
+        )
+    }
+    println!(
+        "File: {:width$} uncompressed size: {} bytes",
+        METADATA_FILE_NAME,
+        header.metadata_file_size,
+        width = longest_path_in_archive,
+    );
+    println!("-------------------------------");
+
+    Ok(())
+}
+
+async fn extract_archive(archive: &Path, target_dir: &Path) -> anyhow::Result<()> {
+    let archive = archive.canonicalize().with_context(|| {
+        format!(
+            "Failed to get the absolute path for the archive path '{}'",
+            archive.display()
+        )
+    })?;
+    ensure!(
+        archive.is_file(),
+        "Path '{}' is not an archive file",
+        archive.display()
+    );
+    let archive_name = match archive.file_name().and_then(|name| name.to_str()) {
+        Some(name) => name,
+        None => bail!(
+            "Failed to get the archive name from the path '{}'",
+            archive.display()
+        ),
+    };
+
+    if !target_dir.exists() {
+        fs::create_dir_all(target_dir).await.with_context(|| {
+            format!(
+                "Failed to create the target dir at path '{}'",
+                target_dir.display()
+            )
+        })?;
+    }
+    let target_dir = target_dir.canonicalize().with_context(|| {
+        format!(
+            "Failed to get the absolute path for the target dir path '{}'",
+            target_dir.display()
+        )
+    })?;
+    ensure!(
+        target_dir.is_dir(),
+        "Path '{}' is not a directory",
+        target_dir.display()
+    );
+    let mut dir_contents = fs::read_dir(&target_dir)
+        .await
+        .context("Failed to list the target directory contents")?;
+    let dir_entry = dir_contents
+        .next_entry()
+        .await
+        .context("Failed to list the target directory contents")?;
+    ensure!(
+        dir_entry.is_none(),
+        "Target directory '{}' is not empty",
+        target_dir.display()
+    );
+
+    println!(
+        "Extracting an archive at path '{}' into directory '{}'",
+        archive.display(),
+        target_dir.display()
+    );
+
+    let mut archive_file = fs::File::open(&archive).await.with_context(|| {
+        format!(
+            "Failed to get the archive name from the path '{}'",
+            archive.display()
+        )
+    })?;
+    let header = compression::read_archive_header(archive_name, &mut archive_file)
+        .await
+        .context("Failed to read the archive header")?;
+    compression::uncompress_with_header(&BTreeSet::new(), &target_dir, header, &mut archive_file)
+        .await
+        .context("Failed to extract the archive")
+}
+
+async fn create_archive(source_dir: &Path, target_dir: &Path) -> anyhow::Result<()> {
+    let source_dir = source_dir.canonicalize().with_context(|| {
+        format!(
+            "Failed to get the absolute path for the source dir path '{}'",
+            source_dir.display()
+        )
+    })?;
+    ensure!(
+        source_dir.is_dir(),
+        "Path '{}' is not a directory",
+        source_dir.display()
+    );
+
+    if !target_dir.exists() {
+        fs::create_dir_all(target_dir).await.with_context(|| {
+            format!(
+                "Failed to create the target dir at path '{}'",
+                target_dir.display()
+            )
+        })?;
+    }
+    let target_dir = target_dir.canonicalize().with_context(|| {
+        format!(
+            "Failed to get the absolute path for the target dir path '{}'",
+            target_dir.display()
+        )
+    })?;
+    ensure!(
+        target_dir.is_dir(),
+        "Path '{}' is not a directory",
+        target_dir.display()
+    );
+
+    println!(
+        "Compressing directory '{}' and creating resulting archive in directory '{}'",
+        source_dir.display(),
+        target_dir.display()
+    );
+
+    let mut metadata_file_contents = None;
+    let mut files_co_archive = Vec::new();
+
+    let mut source_dir_contents = fs::read_dir(&source_dir)
+        .await
+        .context("Failed to read the source directory contents")?;
+
+    while let Some(source_dir_entry) = source_dir_contents
+        .next_entry()
+        .await
+        .context("Failed to read a source dir entry")?
+    {
+        let entry_path = source_dir_entry.path();
+        if entry_path.is_file() {
+            if entry_path.file_name().and_then(|name| name.to_str()) == Some(METADATA_FILE_NAME) {
+                let metadata_bytes = fs::read(entry_path)
+                    .await
+                    .context("Failed to read metata file bytes in the source dir")?;
+                metadata_file_contents = Some(
+                    TimelineMetadata::from_bytes(&metadata_bytes)
+                        .context("Failed to parse metata file contents in the source dir")?,
+                );
+            } else {
+                files_co_archive.push(entry_path);
+            }
+        }
+    }
+
+    let metadata = match metadata_file_contents {
+        Some(metadata) => metadata,
+        None => bail!(
+            "No metadata file found in the source dir '{}', cannot create the archive",
+            source_dir.display()
+        ),
+    };
+
+    let _ = compression::archive_files_as_stream(
+        &source_dir,
+        files_co_archive.iter(),
+        &metadata,
+        move |mut archive_streamer, archive_name| async move {
+            let archive_target = target_dir.join(&archive_name);
+            let mut archive_file = fs::File::create(&archive_target).await?;
+            io::copy(&mut archive_streamer, &mut archive_file).await?;
+            Ok(archive_target)
+        },
+    )
+    .await
+    .context("Failed to create an archive")?;
+
+    Ok(())
+}

pageserver/src/bin/psbench.rs

@@ -1,265 +0,0 @@
-//! Pageserver benchmark tool
-//!
-//! This tool connects directly to a pageserver, issues queries and measures performance.
-//!
-//! Ideally the tool would be ablle to stream WAL into the pageserver, and (possibly
-//! simultaneously) make read requests. Currently wal streaming is not implemented,
-//! so this tool assumes the pageserver is prepopulated with some data, and only
-//! issues read queries. It also currently assumes that the pageserver writes out some
-//! metadata describing the write access pattern on the workload that was performed on it.
-//! See the python tests that use psbench for usage example.
-//!
-//! This tool runs a variety of workloads. See the PsbenchTest enum below, or run the tool
-//! with --help to see the available workloads.
-//!
-use bytes::{BufMut, BytesMut};
-use clap::{Parser, Subcommand};
-use futures::future;
-use pageserver::wal_metadata::{Page, WalEntryMetadata};
-use postgres_ffi::pg_constants::BLCKSZ;
-use rand::seq::SliceRandom;
-use rand::thread_rng;
-use std::fs::File;
-use std::path::{Path, PathBuf};
-use std::time::Instant;
-use std::{
-    collections::HashSet,
-    io::{BufRead, BufReader, Cursor},
-    time::Duration,
-};
-use tokio::io::{AsyncReadExt, AsyncWriteExt};
-use tokio::net::TcpStream;
-use utils::zid::{ZTenantId, ZTimelineId};
-use utils::{
-    lsn::Lsn,
-    pq_proto::{BeMessage, FeMessage},
-};
-
-use anyhow::Result;
-
-/// Client for the pageserver's pagestream API
-struct PagestreamApi {
-    stream: TcpStream,
-}
-
-/// Good enough implementation for these tests
-impl PagestreamApi {
-    async fn connect(tenant: &ZTenantId, timeline: &ZTimelineId) -> Result<PagestreamApi> {
-        let mut stream = TcpStream::connect("localhost:15000").await?;
-
-        // Connect to pageserver
-        // TODO read host, port, dbname, user from command line
-        let (client, conn) = tokio_postgres::Config::new()
-            .host("127.0.0.1")
-            .port(15000)
-            .dbname("postgres")
-            .user("zenith_admin")
-            .connect_raw(&mut stream, tokio_postgres::NoTls)
-            .await?;
-
-        // Enter pagestream protocol
-        let init_query = format!("pagestream {} {}", tenant, timeline);
-        tokio::select! {
-            _ = conn => panic!("connection closed during pagestream initialization"),
-            _ = client.query(init_query.as_str(), &[]) => (),
-        };
-
-        Ok(PagestreamApi { stream })
-    }
-
-    async fn get_page(&mut self, lsn: &Lsn, page: &Page, latest: bool) -> anyhow::Result<Vec<u8>> {
-        let latest: u8 = if latest { 1 } else { 0 };
-        let msg = {
-            let query = {
-                let mut query = BytesMut::new();
-                query.put_u8(2); // Specifies get_page query
-                query.put_u8(latest);
-                query.put_u64(lsn.0);
-                page.write(&mut query).await?;
-                query.freeze()
-            };
-
-            let mut buf = BytesMut::new();
-            let copy_msg = BeMessage::CopyData(&query);
-            BeMessage::write(&mut buf, &copy_msg)?;
-            buf.freeze()
-        };
-
-        self.stream.write_all(&msg).await?;
-
-        let response = match FeMessage::read_fut(&mut self.stream).await? {
-            Some(FeMessage::CopyData(page)) => page,
-            r => panic!("Expected CopyData message, got: {:?}", r),
-        };
-
-        let page = {
-            let mut cursor = Cursor::new(response);
-            let tag = AsyncReadExt::read_u8(&mut cursor).await?;
-
-            match tag {
-                102 => {
-                    let mut page = Vec::<u8>::new();
-                    cursor.read_to_end(&mut page).await?;
-                    if page.len() != (BLCKSZ as usize) {
-                        panic!("Expected 8kb page, got: {:?}", page.len());
-                    }
-                    page
-                }
-                103 => {
-                    let mut bytes = Vec::<u8>::new();
-                    cursor.read_to_end(&mut bytes).await?;
-                    let message = String::from_utf8(bytes)?;
-                    panic!("Got error message: {}", message);
-                }
-                _ => panic!("Unhandled tag {:?}", tag),
-            }
-        };
-
-        Ok(page)
-    }
-}
-
-/// Parsed wal_metadata file with additional derived
-/// statistics for convenience.
-#[derive(Clone)]
-struct Metadata {
-    // Parsed from metadata file
-    wal_metadata: Vec<WalEntryMetadata>,
-
-    // Derived from wal_metadata
-    total_wal_size: usize,
-    affected_pages: HashSet<Page>,
-    latest_lsn: Lsn,
-}
-
-impl Metadata {
-    /// Construct metadata object from wal_metadata file emitted by pageserver
-    fn build(wal_metadata_path: &Path) -> Result<Metadata> {
-        let wal_metadata_file = File::open(wal_metadata_path).expect("error opening wal_metadata");
-        let wal_metadata: Vec<WalEntryMetadata> = BufReader::new(wal_metadata_file)
-            .lines()
-            .map(|result| result.expect("error reading from file"))
-            .map(|line| serde_json::from_str(&line).expect("corrupt metadata file"))
-            .collect();
-
-        let total_wal_size: usize = wal_metadata.iter().map(|m| m.size).sum();
-        let affected_pages: HashSet<_> = wal_metadata
-            .iter()
-            .flat_map(|m| m.affected_pages.clone())
-            .collect();
-        let latest_lsn = wal_metadata.iter().map(|m| m.lsn).max().unwrap();
-
-        Ok(Metadata {
-            wal_metadata,
-            total_wal_size,
-            affected_pages,
-            latest_lsn,
-        })
-    }
-
-    /// Print results in a format readable by benchmark_fixture.py
-    fn report_latency(&self, latencies: &[Duration]) -> Result<()> {
-        let mut latencies: Vec<&Duration> = latencies.iter().collect();
-        latencies.sort();
-
-        println!("test_param num_pages {}", self.affected_pages.len());
-        println!("test_param num_wal_entries {}", self.wal_metadata.len());
-        println!("test_param total_wal_size {} bytes", self.total_wal_size);
-        println!(
-            "lower_is_better fastest {:?} microseconds",
-            latencies.first().unwrap().as_micros()
-        );
-        println!(
-            "lower_is_better median {:?} microseconds",
-            latencies[latencies.len() / 2].as_micros()
-        );
-        println!(
-            "lower_is_better average {:.2} microseconds",
-            (latencies.iter().map(|l| l.as_micros()).sum::<u128>() as f64)
-                / (latencies.len() as f64)
-        );
-        println!(
-            "lower_is_better p99 {:?} microseconds",
-            latencies[latencies.len() - 1 - latencies.len() / 100].as_micros()
-        );
-        println!(
-            "lower_is_better slowest {:?} microseconds",
-            latencies.last().unwrap().as_micros()
-        );
-        Ok(())
-    }
-}
-
-/// Sequentially get the latest version of each page and report latencies
-async fn test_latest_pages(
-    tenant: &ZTenantId,
-    timeline: &ZTimelineId,
-    metadata: &Metadata,
-) -> Result<Vec<Duration>> {
-    let mut api = PagestreamApi::connect(tenant, timeline).await.unwrap();
-    let mut latencies: Vec<Duration> = vec![];
-    let mut page_order: Vec<&Page> = metadata.affected_pages.iter().collect();
-    page_order.shuffle(&mut thread_rng());
-    for page in page_order {
-        let start = Instant::now();
-        let _page_bytes = api.get_page(&metadata.latest_lsn, page, true).await?;
-        let duration = start.elapsed();
-
-        latencies.push(duration);
-    }
-    Ok(latencies)
-}
-
-#[derive(Parser, Debug, Clone)]
-struct Args {
-    // TODO maybe one metadata file per timeline?
-    wal_metadata_path: PathBuf,
-
-    // TODO get these from wal metadata
-    tenant: ZTenantId,
-    timeline: ZTimelineId,
-
-    // TODO change to `clients_per_timeline`
-    #[clap(long, default_value = "1")]
-    num_clients: usize,
-
-    #[clap(subcommand)]
-    test: PsbenchTest,
-}
-
-#[derive(Subcommand, Debug, Clone)]
-enum PsbenchTest {
-    /// Query the latest version of each page, in a random sequential order.
-    /// If multiple clients are used, all clients will independently query
-    /// every page in a different random order.
-    GetLatestPages,
-    // TODO add more tests:
-    // - Query with realistic read pattern
-    // - Query every page after every change of that page
-    // - Query all pages at given point in time
-    // - etc.
-}
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    let args = Args::parse();
-    let metadata = Metadata::build(&args.wal_metadata_path).unwrap();
-
-    let latencies: Vec<Duration> = match args.test {
-        PsbenchTest::GetLatestPages => {
-            // TODO explicitly spawn a thread for each?
-            future::join_all(
-                (0..args.num_clients)
-                    .map(|_| test_latest_pages(&args.tenant, &args.timeline, &metadata)),
-            )
-            .await
-            .into_iter()
-            .flat_map(|v| v.unwrap())
-            .collect()
-        }
-    };
-
-    println!("test_param num_clients {}", args.num_clients);
-    metadata.report_latency(&latencies).unwrap();
-    Ok(())
-}

pageserver/src/bin/update_metadata.rs

@@ -2,14 +2,15 @@
 //!
 //! A handy tool for debugging, that's all.
 use anyhow::Result;
-use clap::{Arg, Command};
+use clap::{App, Arg};
 use pageserver::layered_repository::metadata::TimelineMetadata;
 use std::path::PathBuf;
 use std::str::FromStr;
-use utils::{lsn::Lsn, GIT_VERSION};
+use zenith_utils::lsn::Lsn;
+use zenith_utils::GIT_VERSION;
 
 fn main() -> Result<()> {
-    let arg_matches = Command::new("Zenith update metadata utility")
+    let arg_matches = App::new("Zenith update metadata utility")
         .about("Dump or update metadata file")
         .version(GIT_VERSION)
         .arg(

pageserver/src/config.rs

@@ -4,24 +4,22 @@
 //! file, or on the command line.
 //! See also `settings.md` for better description on every parameter.
 
-use anyhow::{anyhow, bail, ensure, Context, Result};
+use anyhow::{bail, ensure, Context, Result};
+use toml_edit;
+use toml_edit::{Document, Item};
+use zenith_utils::postgres_backend::AuthType;
+use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId};
+
+use std::convert::TryInto;
 use std::env;
 use std::num::{NonZeroU32, NonZeroUsize};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::time::Duration;
-use toml_edit;
-use toml_edit::{Document, Item};
-use utils::{
-    postgres_backend::AuthType,
-    zid::{ZNodeId, ZTenantId, ZTimelineId},
-};
 
 use crate::layered_repository::TIMELINES_SEGMENT_NAME;
-use crate::tenant_config::{TenantConf, TenantConfOpt};
 
 pub mod defaults {
-    use crate::tenant_config::defaults::*;
     use const_format::formatcp;
 
     pub const DEFAULT_PG_LISTEN_PORT: u16 = 64000;
@@ -33,18 +31,8 @@ pub mod defaults {
     pub const DEFAULT_WAL_REDO_TIMEOUT: &str = "60 s";
 
     pub const DEFAULT_SUPERUSER: &str = "zenith_admin";
-    /// How many different timelines can be processed simultaneously when synchronizing layers with the remote storage.
-    /// During regular work, pageserver produces one layer file per timeline checkpoint, with bursts of concurrency
-    /// during start (where local and remote timelines are compared and initial sync tasks are scheduled) and timeline attach.
-    /// Both cases may trigger timeline download, that might download a lot of layers. This concurrency is limited by the clients internally, if needed.
-    pub const DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_TIMELINES_SYNC: usize = 50;
+    pub const DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNC: usize = 10;
     pub const DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS: u32 = 10;
-    /// Currently, sync happens with AWS S3, that has two limits on requests per second:
-    /// ~200 RPS for IAM services
-    /// https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.html
-    /// ~3500 PUT/COPY/POST/DELETE or 5500 GET/HEAD S3 requests
-    /// https://aws.amazon.com/premiumsupport/knowledge-center/s3-request-limit-avoid-throttling/
-    pub const DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT: usize = 100;
 
     pub const DEFAULT_PAGE_CACHE_SIZE: usize = 8192;
     pub const DEFAULT_MAX_FILE_DESCRIPTORS: usize = 100;
@@ -67,16 +55,6 @@ pub mod defaults {
 # initial superuser role name to use when creating a new tenant
 #initial_superuser_name = '{DEFAULT_SUPERUSER}'
 
-# [tenant_config]
-#checkpoint_distance = {DEFAULT_CHECKPOINT_DISTANCE} # in bytes
-#compaction_target_size = {DEFAULT_COMPACTION_TARGET_SIZE} # in bytes
-#compaction_period = '{DEFAULT_COMPACTION_PERIOD}'
-#compaction_threshold = '{DEFAULT_COMPACTION_THRESHOLD}'
-
-#gc_period = '{DEFAULT_GC_PERIOD}'
-#gc_horizon = {DEFAULT_GC_HORIZON}
-#pitr_interval = '{DEFAULT_PITR_INTERVAL}'
-
 # [remote_storage]
 
 "###
@@ -118,29 +96,6 @@ pub struct PageServerConf {
 
     pub auth_validation_public_key_path: Option<PathBuf>,
     pub remote_storage_config: Option<RemoteStorageConfig>,
-
-    pub emit_wal_metadata: bool,
-    pub profiling: ProfilingConfig,
-    pub default_tenant_conf: TenantConf,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum ProfilingConfig {
-    Disabled,
-    PageRequests,
-}
-
-impl FromStr for ProfilingConfig {
-    type Err = anyhow::Error;
-
-    fn from_str(s: &str) -> Result<ProfilingConfig, Self::Err> {
-        let result = match s {
-            "disabled"  => ProfilingConfig::Disabled,
-            "page_requests"  => ProfilingConfig::PageRequests,
-            _ => bail!("invalid value \"{s}\" for profiling option, valid values are \"disabled\" and \"page_requests\""),
-        };
-        Ok(result)
-    }
 }
 
 // use dedicated enum for builder to better indicate the intention
@@ -184,9 +139,6 @@ struct PageServerConfigBuilder {
     remote_storage_config: BuilderValue<Option<RemoteStorageConfig>>,
 
     id: BuilderValue<ZNodeId>,
-
-    emit_wal_metadata: BuilderValue<bool>,
-    profiling: BuilderValue<ProfilingConfig>,
 }
 
 impl Default for PageServerConfigBuilder {
@@ -211,8 +163,6 @@ impl Default for PageServerConfigBuilder {
             auth_validation_public_key_path: Set(None),
             remote_storage_config: Set(None),
             id: NotSet,
-            emit_wal_metadata: Set(false),
-            profiling: Set(ProfilingConfig::Disabled),
         }
     }
 }
@@ -273,53 +223,39 @@ impl PageServerConfigBuilder {
         self.id = BuilderValue::Set(node_id)
     }
 
-    pub fn emit_wal_metadata(&mut self, value: bool) {
-        self.emit_wal_metadata = BuilderValue::Set(value)
-    }
-
-    pub fn profiling(&mut self, profiling: ProfilingConfig) {
-        self.profiling = BuilderValue::Set(profiling)
-    }
-
     pub fn build(self) -> Result<PageServerConf> {
         Ok(PageServerConf {
             listen_pg_addr: self
                 .listen_pg_addr
-                .ok_or(anyhow!("missing listen_pg_addr"))?,
+                .ok_or(anyhow::anyhow!("missing listen_pg_addr"))?,
             listen_http_addr: self
                 .listen_http_addr
-                .ok_or(anyhow!("missing listen_http_addr"))?,
+                .ok_or(anyhow::anyhow!("missing listen_http_addr"))?,
             wait_lsn_timeout: self
                 .wait_lsn_timeout
-                .ok_or(anyhow!("missing wait_lsn_timeout"))?,
+                .ok_or(anyhow::anyhow!("missing wait_lsn_timeout"))?,
             wal_redo_timeout: self
                 .wal_redo_timeout
-                .ok_or(anyhow!("missing wal_redo_timeout"))?,
-            superuser: self.superuser.ok_or(anyhow!("missing superuser"))?,
+                .ok_or(anyhow::anyhow!("missing wal_redo_timeout"))?,
+            superuser: self.superuser.ok_or(anyhow::anyhow!("missing superuser"))?,
             page_cache_size: self
                 .page_cache_size
-                .ok_or(anyhow!("missing page_cache_size"))?,
+                .ok_or(anyhow::anyhow!("missing page_cache_size"))?,
             max_file_descriptors: self
                 .max_file_descriptors
-                .ok_or(anyhow!("missing max_file_descriptors"))?,
-            workdir: self.workdir.ok_or(anyhow!("missing workdir"))?,
+                .ok_or(anyhow::anyhow!("missing max_file_descriptors"))?,
+            workdir: self.workdir.ok_or(anyhow::anyhow!("missing workdir"))?,
             pg_distrib_dir: self
                 .pg_distrib_dir
-                .ok_or(anyhow!("missing pg_distrib_dir"))?,
-            auth_type: self.auth_type.ok_or(anyhow!("missing auth_type"))?,
+                .ok_or(anyhow::anyhow!("missing pg_distrib_dir"))?,
+            auth_type: self.auth_type.ok_or(anyhow::anyhow!("missing auth_type"))?,
             auth_validation_public_key_path: self
                 .auth_validation_public_key_path
-                .ok_or(anyhow!("missing auth_validation_public_key_path"))?,
+                .ok_or(anyhow::anyhow!("missing auth_validation_public_key_path"))?,
             remote_storage_config: self
                 .remote_storage_config
-                .ok_or(anyhow!("missing remote_storage_config"))?,
-            id: self.id.ok_or(anyhow!("missing id"))?,
-            emit_wal_metadata: self
-                .emit_wal_metadata
-                .ok_or(anyhow!("emit_wal_metadata not specifiec"))?,
-            profiling: self.profiling.ok_or(anyhow!("missing profiling"))?,
-            // TenantConf is handled separately
-            default_tenant_conf: TenantConf::default(),
+                .ok_or(anyhow::anyhow!("missing remote_storage_config"))?,
+            id: self.id.ok_or(anyhow::anyhow!("missing id"))?,
         })
     }
 }
@@ -328,7 +264,7 @@ impl PageServerConfigBuilder {
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct RemoteStorageConfig {
     /// Max allowed number of concurrent sync operations between pageserver and the remote storage.
-    pub max_concurrent_timelines_sync: NonZeroUsize,
+    pub max_concurrent_sync: NonZeroUsize,
     /// Max allowed errors before the sync task is considered failed and evicted.
     pub max_sync_errors: NonZeroU32,
     /// The storage connection configuration.
@@ -369,9 +305,6 @@ pub struct S3Config {
     ///
     /// Example: `http://127.0.0.1:5000`
     pub endpoint: Option<String>,
-    /// AWS S3 has various limits on its API calls, we need not to exceed those.
-    /// See [`defaults::DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT`] for more details.
-    pub concurrency_limit: NonZeroUsize,
 }
 
 impl std::fmt::Debug for S3Config {
@@ -380,7 +313,6 @@ impl std::fmt::Debug for S3Config {
             .field("bucket_name", &self.bucket_name)
             .field("bucket_region", &self.bucket_region)
             .field("prefix_in_bucket", &self.prefix_in_bucket)
-            .field("concurrency_limit", &self.concurrency_limit)
             .finish()
     }
 }
@@ -426,8 +358,6 @@ impl PageServerConf {
         let mut builder = PageServerConfigBuilder::default();
         builder.workdir(workdir.to_owned());
 
-        let mut t_conf: TenantConfOpt = Default::default();
-
         for (key, item) in toml.iter() {
             match key {
                 "listen_pg_addr" => builder.listen_pg_addr(parse_toml_string(key, item)?),
@@ -445,17 +375,12 @@ impl PageServerConf {
                 "auth_validation_public_key_path" => builder.auth_validation_public_key_path(Some(
                     PathBuf::from(parse_toml_string(key, item)?),
                 )),
-                "auth_type" => builder.auth_type(parse_toml_from_str(key, item)?),
+                "auth_type" => builder.auth_type(parse_toml_auth_type(key, item)?),
                 "remote_storage" => {
                     builder.remote_storage_config(Some(Self::parse_remote_storage_config(item)?))
                 }
-                "tenant_conf" => {
-                    t_conf = Self::parse_toml_tenant_conf(item)?;
-                }
                 "id" => builder.id(ZNodeId(parse_toml_u64(key, item)?)),
-                "emit_wal_metadata" => builder.emit_wal_metadata(true),
-                "profiling" => builder.profiling(parse_toml_from_str(key, item)?),
-                _ => bail!("unrecognized pageserver option '{key}'"),
+                _ => bail!("unrecognized pageserver option '{}'", key),
             }
         }
 
@@ -481,75 +406,41 @@ impl PageServerConf {
             );
         }
 
-        conf.default_tenant_conf = t_conf.merge(TenantConf::default());
-
         Ok(conf)
     }
 
-    // subroutine of parse_and_validate to parse `[tenant_conf]` section
-
-    pub fn parse_toml_tenant_conf(item: &toml_edit::Item) -> Result<TenantConfOpt> {
-        let mut t_conf: TenantConfOpt = Default::default();
-        if let Some(checkpoint_distance) = item.get("checkpoint_distance") {
-            t_conf.checkpoint_distance =
-                Some(parse_toml_u64("checkpoint_distance", checkpoint_distance)?);
-        }
-
-        if let Some(compaction_target_size) = item.get("compaction_target_size") {
-            t_conf.compaction_target_size = Some(parse_toml_u64(
-                "compaction_target_size",
-                compaction_target_size,
-            )?);
-        }
-
-        if let Some(compaction_period) = item.get("compaction_period") {
-            t_conf.compaction_period =
-                Some(parse_toml_duration("compaction_period", compaction_period)?);
-        }
-
-        if let Some(compaction_threshold) = item.get("compaction_threshold") {
-            t_conf.compaction_threshold =
-                Some(parse_toml_u64("compaction_threshold", compaction_threshold)?.try_into()?);
-        }
-
-        if let Some(gc_horizon) = item.get("gc_horizon") {
-            t_conf.gc_horizon = Some(parse_toml_u64("gc_horizon", gc_horizon)?);
-        }
-
-        if let Some(gc_period) = item.get("gc_period") {
-            t_conf.gc_period = Some(parse_toml_duration("gc_period", gc_period)?);
-        }
-
-        if let Some(pitr_interval) = item.get("pitr_interval") {
-            t_conf.pitr_interval = Some(parse_toml_duration("pitr_interval", pitr_interval)?);
-        }
-
-        Ok(t_conf)
-    }
-
     /// subroutine of parse_config(), to parse the `[remote_storage]` table.
     fn parse_remote_storage_config(toml: &toml_edit::Item) -> anyhow::Result<RemoteStorageConfig> {
         let local_path = toml.get("local_path");
         let bucket_name = toml.get("bucket_name");
         let bucket_region = toml.get("bucket_region");
 
-        let max_concurrent_timelines_sync = NonZeroUsize::new(
-            parse_optional_integer("max_concurrent_timelines_sync", toml)?
-                .unwrap_or(defaults::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_TIMELINES_SYNC),
-        )
-        .context("Failed to parse 'max_concurrent_timelines_sync' as a positive integer")?;
-
-        let max_sync_errors = NonZeroU32::new(
-            parse_optional_integer("max_sync_errors", toml)?
-                .unwrap_or(defaults::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS),
-        )
-        .context("Failed to parse 'max_sync_errors' as a positive integer")?;
-
-        let concurrency_limit = NonZeroUsize::new(
-            parse_optional_integer("concurrency_limit", toml)?
-                .unwrap_or(defaults::DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT),
-        )
-        .context("Failed to parse 'concurrency_limit' as a positive integer")?;
+        let max_concurrent_sync: NonZeroUsize = if let Some(s) = toml.get("max_concurrent_sync") {
+            parse_toml_u64("max_concurrent_sync", s)
+                .and_then(|toml_u64| {
+                    toml_u64.try_into().with_context(|| {
+                        format!("'max_concurrent_sync' value {} is too large", toml_u64)
+                    })
+                })
+                .ok()
+                .and_then(NonZeroUsize::new)
+                .context("'max_concurrent_sync' must be a non-zero positive integer")?
+        } else {
+            NonZeroUsize::new(defaults::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNC).unwrap()
+        };
+        let max_sync_errors: NonZeroU32 = if let Some(s) = toml.get("max_sync_errors") {
+            parse_toml_u64("max_sync_errors", s)
+                .and_then(|toml_u64| {
+                    toml_u64.try_into().with_context(|| {
+                        format!("'max_sync_errors' value {} is too large", toml_u64)
+                    })
+                })
+                .ok()
+                .and_then(NonZeroU32::new)
+                .context("'max_sync_errors' must be a non-zero positive integer")?
+        } else {
+            NonZeroU32::new(defaults::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS).unwrap()
+        };
 
         let storage = match (local_path, bucket_name, bucket_region) {
             (None, None, None) => bail!("no 'local_path' nor 'bucket_name' option"),
@@ -580,7 +471,6 @@ impl PageServerConf {
                     .get("endpoint")
                     .map(|endpoint| parse_toml_string("endpoint", endpoint))
                     .transpose()?,
-                concurrency_limit,
             }),
             (Some(local_path), None, None) => RemoteStorageKind::LocalFs(PathBuf::from(
                 parse_toml_string("local_path", local_path)?,
@@ -589,7 +479,7 @@ impl PageServerConf {
         };
 
         Ok(RemoteStorageConfig {
-            max_concurrent_timelines_sync,
+            max_concurrent_sync,
             max_sync_errors,
             storage,
         })
@@ -597,7 +487,7 @@ impl PageServerConf {
 
     #[cfg(test)]
     pub fn test_repo_dir(test_name: &str) -> PathBuf {
-        PathBuf::from(format!("../tmp_check/test_{test_name}"))
+        PathBuf::from(format!("../tmp_check/test_{}", test_name))
     }
 
     #[cfg(test)]
@@ -616,9 +506,6 @@ impl PageServerConf {
             auth_type: AuthType::Trust,
             auth_validation_public_key_path: None,
             remote_storage_config: None,
-            emit_wal_metadata: false,
-            profiling: ProfilingConfig::Disabled,
-            default_tenant_conf: TenantConf::dummy_conf(),
         }
     }
 }
@@ -628,7 +515,7 @@ impl PageServerConf {
 fn parse_toml_string(name: &str, item: &Item) -> Result<String> {
     let s = item
         .as_str()
-        .with_context(|| format!("configure option {name} is not a string"))?;
+        .with_context(|| format!("configure option {} is not a string", name))?;
     Ok(s.to_string())
 }
 
@@ -637,46 +524,26 @@ fn parse_toml_u64(name: &str, item: &Item) -> Result<u64> {
     // for our use, though.
     let i: i64 = item
         .as_integer()
-        .with_context(|| format!("configure option {name} is not an integer"))?;
+        .with_context(|| format!("configure option {} is not an integer", name))?;
     if i < 0 {
-        bail!("configure option {name} cannot be negative");
+        bail!("configure option {} cannot be negative", name);
     }
     Ok(i as u64)
 }
 
-fn parse_optional_integer<I, E>(name: &str, item: &toml_edit::Item) -> anyhow::Result<Option<I>>
-where
-    I: TryFrom<i64, Error = E>,
-    E: std::error::Error + Send + Sync + 'static,
-{
-    let toml_integer = match item.get(name) {
-        Some(item) => item
-            .as_integer()
-            .with_context(|| format!("configure option {name} is not an integer"))?,
-        None => return Ok(None),
-    };
-
-    I::try_from(toml_integer)
-        .map(Some)
-        .with_context(|| format!("configure option {name} is too large"))
-}
-
 fn parse_toml_duration(name: &str, item: &Item) -> Result<Duration> {
     let s = item
         .as_str()
-        .with_context(|| format!("configure option {name} is not a string"))?;
+        .with_context(|| format!("configure option {} is not a string", name))?;
 
     Ok(humantime::parse_duration(s)?)
 }
 
-fn parse_toml_from_str<T>(name: &str, item: &Item) -> Result<T>
-where
-    T: FromStr<Err = anyhow::Error>,
-{
+fn parse_toml_auth_type(name: &str, item: &Item) -> Result<AuthType> {
     let v = item
         .as_str()
-        .with_context(|| format!("configure option {name} is not a string"))?;
-    T::from_str(v)
+        .with_context(|| format!("configure option {} is not a string", name))?;
+    AuthType::from_str(v)
 }
 
 #[cfg(test)]
@@ -713,8 +580,10 @@ id = 10
         let config_string = format!("pg_distrib_dir='{}'\nid=10", pg_distrib_dir.display());
         let toml = config_string.parse()?;
 
-        let parsed_config = PageServerConf::parse_and_validate(&toml, &workdir)
-            .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e}"));
+        let parsed_config =
+            PageServerConf::parse_and_validate(&toml, &workdir).unwrap_or_else(|e| {
+                panic!("Failed to parse config '{}', reason: {}", config_string, e)
+            });
 
         assert_eq!(
             parsed_config,
@@ -732,9 +601,6 @@ id = 10
                 auth_type: AuthType::Trust,
                 auth_validation_public_key_path: None,
                 remote_storage_config: None,
-                emit_wal_metadata: false,
-                profiling: ProfilingConfig::Disabled,
-                default_tenant_conf: TenantConf::default(),
             },
             "Correct defaults should be used when no config values are provided"
         );
@@ -748,13 +614,16 @@ id = 10
         let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?;
 
         let config_string = format!(
-            "{ALL_BASE_VALUES_TOML}pg_distrib_dir='{}'",
+            "{}pg_distrib_dir='{}'",
+            ALL_BASE_VALUES_TOML,
             pg_distrib_dir.display()
         );
         let toml = config_string.parse()?;
 
-        let parsed_config = PageServerConf::parse_and_validate(&toml, &workdir)
-            .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e}"));
+        let parsed_config =
+            PageServerConf::parse_and_validate(&toml, &workdir).unwrap_or_else(|e| {
+                panic!("Failed to parse config '{}', reason: {}", config_string, e)
+            });
 
         assert_eq!(
             parsed_config,
@@ -772,9 +641,6 @@ id = 10
                 auth_type: AuthType::Trust,
                 auth_validation_public_key_path: None,
                 remote_storage_config: None,
-                emit_wal_metadata: false,
-                profiling: ProfilingConfig::Disabled,
-                default_tenant_conf: TenantConf::default(),
             },
             "Should be able to parse all basic config values correctly"
         );
@@ -803,33 +669,37 @@ local_path = '{}'"#,
 
         for remote_storage_config_str in identical_toml_declarations {
             let config_string = format!(
-                r#"{ALL_BASE_VALUES_TOML}
+                r#"{}
 pg_distrib_dir='{}'
 
-{remote_storage_config_str}"#,
+{}"#,
+                ALL_BASE_VALUES_TOML,
                 pg_distrib_dir.display(),
+                remote_storage_config_str,
             );
 
             let toml = config_string.parse()?;
 
             let parsed_remote_storage_config = PageServerConf::parse_and_validate(&toml, &workdir)
-                .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e}"))
+                .unwrap_or_else(|e| {
+                    panic!("Failed to parse config '{}', reason: {}", config_string, e)
+                })
                 .remote_storage_config
                 .expect("Should have remote storage config for the local FS");
 
             assert_eq!(
-                parsed_remote_storage_config,
-                RemoteStorageConfig {
-                    max_concurrent_timelines_sync: NonZeroUsize::new(
-                        defaults::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_TIMELINES_SYNC
-                    )
+            parsed_remote_storage_config,
+            RemoteStorageConfig {
+                max_concurrent_sync: NonZeroUsize::new(
+                    defaults::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNC
+                )
+                .unwrap(),
+                max_sync_errors: NonZeroU32::new(defaults::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS)
                     .unwrap(),
-                    max_sync_errors: NonZeroU32::new(defaults::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS)
-                        .unwrap(),
-                    storage: RemoteStorageKind::LocalFs(local_storage_path.clone()),
-                },
-                "Remote storage config should correctly parse the local FS config and fill other storage defaults"
-            );
+                storage: RemoteStorageKind::LocalFs(local_storage_path.clone()),
+            },
+            "Remote storage config should correctly parse the local FS config and fill other storage defaults"
+        );
         }
         Ok(())
     }
@@ -845,49 +715,52 @@ pg_distrib_dir='{}'
         let access_key_id = "SOMEKEYAAAAASADSAH*#".to_string();
         let secret_access_key = "SOMEsEcReTsd292v".to_string();
         let endpoint = "http://localhost:5000".to_string();
-        let max_concurrent_timelines_sync = NonZeroUsize::new(111).unwrap();
+        let max_concurrent_sync = NonZeroUsize::new(111).unwrap();
         let max_sync_errors = NonZeroU32::new(222).unwrap();
-        let s3_concurrency_limit = NonZeroUsize::new(333).unwrap();
 
         let identical_toml_declarations = &[
             format!(
                 r#"[remote_storage]
-max_concurrent_timelines_sync = {max_concurrent_timelines_sync}
-max_sync_errors = {max_sync_errors}
-bucket_name = '{bucket_name}'
-bucket_region = '{bucket_region}'
-prefix_in_bucket = '{prefix_in_bucket}'
-access_key_id = '{access_key_id}'
-secret_access_key = '{secret_access_key}'
-endpoint = '{endpoint}'
-concurrency_limit = {s3_concurrency_limit}"#
+max_concurrent_sync = {}
+max_sync_errors = {}
+bucket_name = '{}'
+bucket_region = '{}'
+prefix_in_bucket = '{}'
+access_key_id = '{}'
+secret_access_key = '{}'
+endpoint = '{}'"#,
+                max_concurrent_sync, max_sync_errors, bucket_name, bucket_region, prefix_in_bucket, access_key_id, secret_access_key, endpoint
             ),
             format!(
-                "remote_storage={{max_concurrent_timelines_sync={max_concurrent_timelines_sync}, max_sync_errors={max_sync_errors}, bucket_name='{bucket_name}',\
-                bucket_region='{bucket_region}', prefix_in_bucket='{prefix_in_bucket}', access_key_id='{access_key_id}', secret_access_key='{secret_access_key}', endpoint='{endpoint}', concurrency_limit={s3_concurrency_limit}}}",
+                "remote_storage={{max_concurrent_sync={}, max_sync_errors={}, bucket_name='{}', bucket_region='{}', prefix_in_bucket='{}', access_key_id='{}', secret_access_key='{}', endpoint='{}'}}",
+                max_concurrent_sync, max_sync_errors, bucket_name, bucket_region, prefix_in_bucket, access_key_id, secret_access_key, endpoint
             ),
         ];
 
         for remote_storage_config_str in identical_toml_declarations {
             let config_string = format!(
-                r#"{ALL_BASE_VALUES_TOML}
+                r#"{}
 pg_distrib_dir='{}'
 
-{remote_storage_config_str}"#,
+{}"#,
+                ALL_BASE_VALUES_TOML,
                 pg_distrib_dir.display(),
+                remote_storage_config_str,
             );
 
             let toml = config_string.parse()?;
 
             let parsed_remote_storage_config = PageServerConf::parse_and_validate(&toml, &workdir)
-                .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e}"))
+                .unwrap_or_else(|e| {
+                    panic!("Failed to parse config '{}', reason: {}", config_string, e)
+                })
                 .remote_storage_config
                 .expect("Should have remote storage config for S3");
 
             assert_eq!(
                 parsed_remote_storage_config,
                 RemoteStorageConfig {
-                    max_concurrent_timelines_sync,
+                    max_concurrent_sync,
                     max_sync_errors,
                     storage: RemoteStorageKind::AwsS3(S3Config {
                         bucket_name: bucket_name.clone(),
@@ -895,8 +768,7 @@ pg_distrib_dir='{}'
                         access_key_id: Some(access_key_id.clone()),
                         secret_access_key: Some(secret_access_key.clone()),
                         prefix_in_bucket: Some(prefix_in_bucket.clone()),
-                        endpoint: Some(endpoint.clone()),
-                        concurrency_limit: s3_concurrency_limit,
+                        endpoint: Some(endpoint.clone())
                     }),
                 },
                 "Remote storage config should correctly parse the S3 config"

pageserver/src/http/models.rs

@@ -1,6 +1,6 @@
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
-use utils::{
+use zenith_utils::{
     lsn::Lsn,
     zid::{ZNodeId, ZTenantId, ZTimelineId},
 };
@@ -20,7 +20,7 @@ pub struct TimelineCreateRequest {
 }
 
 #[serde_as]
-#[derive(Serialize, Deserialize, Default)]
+#[derive(Serialize, Deserialize)]
 pub struct TenantCreateRequest {
     #[serde(default)]
     #[serde_as(as = "Option<DisplayFromStr>")]
@@ -28,7 +28,6 @@ pub struct TenantCreateRequest {
     pub checkpoint_distance: Option<u64>,
     pub compaction_target_size: Option<u64>,
     pub compaction_period: Option<String>,
-    pub compaction_threshold: Option<usize>,
     pub gc_horizon: Option<u64>,
     pub gc_period: Option<String>,
     pub pitr_interval: Option<String>,
@@ -48,34 +47,9 @@ impl TenantCreateRequest {
     pub fn new(new_tenant_id: Option<ZTenantId>) -> TenantCreateRequest {
         TenantCreateRequest {
             new_tenant_id,
-            ..Default::default()
-        }
-    }
-}
-
-#[serde_as]
-#[derive(Serialize, Deserialize)]
-pub struct TenantConfigRequest {
-    pub tenant_id: ZTenantId,
-    #[serde(default)]
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub checkpoint_distance: Option<u64>,
-    pub compaction_target_size: Option<u64>,
-    pub compaction_period: Option<String>,
-    pub compaction_threshold: Option<usize>,
-    pub gc_horizon: Option<u64>,
-    pub gc_period: Option<String>,
-    pub pitr_interval: Option<String>,
-}
-
-impl TenantConfigRequest {
-    pub fn new(tenant_id: ZTenantId) -> TenantConfigRequest {
-        TenantConfigRequest {
-            tenant_id,
             checkpoint_distance: None,
             compaction_target_size: None,
             compaction_period: None,
-            compaction_threshold: None,
             gc_horizon: None,
             gc_period: None,
             pitr_interval: None,

pageserver/src/http/openapi_spec.yml

@@ -328,7 +328,21 @@ paths:
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/TenantCreateInfo"
+              type: object
+              properties:
+                new_tenant_id:
+                  type: string
+                  format: hex
+                gc_period:
+                  type: string
+                gc_horizon:
+                  type: integer
+                pitr_interval:
+                  type: string
+                checkpoint_distance:
+                  type: integer
+                compaction_period:
+                  type: string
       responses:
         "201":
           description: New tenant created successfully
@@ -367,48 +381,7 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/Error"
-  /v1/tenant/config:
-    put:
-      description: |
-        Update tenant's config.
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: "#/components/schemas/TenantConfigInfo"
-      responses:
-        "200":
-          description: OK
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  $ref: "#/components/schemas/TenantInfo"
-        "400":
-          description: Malformed tenant config request
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
+
 components:
   securitySchemes:
     JWT:
@@ -426,45 +399,6 @@ components:
           type: string
         state:
           type: string
-    TenantCreateInfo:
-      type: object
-      properties:
-        new_tenant_id:
-          type: string
-          format: hex
-        tenant_id:
-          type: string
-          format: hex
-        gc_period:
-          type: string
-        gc_horizon:
-          type: integer
-        pitr_interval:
-          type: string
-        checkpoint_distance:
-          type: integer
-        compaction_period:
-          type: string
-        compaction_threshold:
-          type: string
-    TenantConfigInfo:
-      type: object
-      properties:
-        tenant_id:
-          type: string
-          format: hex
-        gc_period:
-          type: string
-        gc_horizon:
-          type: integer
-        pitr_interval:
-          type: string
-        checkpoint_distance:
-          type: integer
-        compaction_period:
-          type: string
-        compaction_threshold:
-          type: string
     TimelineInfo:
       type: object
       required:
@@ -485,7 +419,6 @@ components:
       type: object
       required:
         - awaits_download
-        - remote_consistent_lsn
       properties:
         awaits_download:
           type: boolean

pageserver/src/http/routes.rs

@@ -1,45 +1,37 @@
 use std::sync::Arc;
 
-use anyhow::{Context, Result};
+use anyhow::Result;
 use hyper::StatusCode;
 use hyper::{Body, Request, Response, Uri};
 use tracing::*;
+use zenith_utils::auth::JwtAuth;
+use zenith_utils::http::endpoint::attach_openapi_ui;
+use zenith_utils::http::endpoint::auth_middleware;
+use zenith_utils::http::endpoint::check_permission;
+use zenith_utils::http::error::ApiError;
+use zenith_utils::http::{
+    endpoint,
+    error::HttpErrorBody,
+    json::{json_request, json_response},
+    request::parse_request_param,
+};
+use zenith_utils::http::{RequestExt, RouterBuilder};
+use zenith_utils::zid::{ZTenantTimelineId, ZTimelineId};
 
 use super::models::{
-    StatusResponse, TenantConfigRequest, TenantCreateRequest, TenantCreateResponse,
-    TimelineCreateRequest,
-};
-use crate::config::RemoteStorageKind;
-use crate::remote_storage::{
-    download_index_part, schedule_timeline_download, LocalFs, RemoteIndex, RemoteTimeline, S3Bucket,
+    StatusResponse, TenantCreateRequest, TenantCreateResponse, TimelineCreateRequest,
 };
+use crate::remote_storage::{schedule_timeline_download, RemoteIndex};
 use crate::repository::Repository;
-use crate::tenant_config::TenantConfOpt;
+use crate::tenant_config::TenantConf;
 use crate::timelines::{LocalTimelineInfo, RemoteTimelineInfo, TimelineInfo};
-use crate::{config::PageServerConf, tenant_mgr, timelines};
-use utils::{
-    auth::JwtAuth,
-    http::{
-        endpoint::{self, attach_openapi_ui, auth_middleware, check_permission},
-        error::{ApiError, HttpErrorBody},
-        json::{json_request, json_response},
-        request::parse_request_param,
-        RequestExt, RouterBuilder,
-    },
-    zid::{ZTenantId, ZTenantTimelineId, ZTimelineId},
-};
+use crate::{config::PageServerConf, tenant_mgr, timelines, ZTenantId};
 
 struct State {
     conf: &'static PageServerConf,
     auth: Option<Arc<JwtAuth>>,
     remote_index: RemoteIndex,
     allowlist_routes: Vec<Uri>,
-    remote_storage: Option<GenericRemoteStorage>,
-}
-
-enum GenericRemoteStorage {
-    Local(LocalFs),
-    S3(S3Bucket),
 }
 
 impl State {
@@ -47,34 +39,17 @@ impl State {
         conf: &'static PageServerConf,
         auth: Option<Arc<JwtAuth>>,
         remote_index: RemoteIndex,
-    ) -> anyhow::Result<Self> {
+    ) -> Self {
         let allowlist_routes = ["/v1/status", "/v1/doc", "/swagger.yml"]
             .iter()
             .map(|v| v.parse().unwrap())
             .collect::<Vec<_>>();
-        // Note that this remote storage is created separately from the main one in the sync_loop.
-        // It's fine since it's stateless and some code duplication saves us from bloating the code around with generics.
-        let remote_storage = conf
-            .remote_storage_config
-            .as_ref()
-            .map(|storage_config| match &storage_config.storage {
-                RemoteStorageKind::LocalFs(root) => {
-                    LocalFs::new(root.clone(), &conf.workdir).map(GenericRemoteStorage::Local)
-                }
-                RemoteStorageKind::AwsS3(s3_config) => {
-                    S3Bucket::new(s3_config, &conf.workdir).map(GenericRemoteStorage::S3)
-                }
-            })
-            .transpose()
-            .context("Failed to init generic remote storage")?;
-
-        Ok(Self {
+        Self {
             conf,
             auth,
             allowlist_routes,
             remote_index,
-            remote_storage,
-        })
+        }
     }
 }
 
@@ -94,7 +69,10 @@ fn get_config(request: &Request<Body>) -> &'static PageServerConf {
 // healthcheck handler
 async fn status_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let config = get_config(&request);
-    json_response(StatusCode::OK, StatusResponse { id: config.id })
+    Ok(json_response(
+        StatusCode::OK,
+        StatusResponse { id: config.id },
+    )?)
 }
 
 async fn timeline_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -148,13 +126,13 @@ async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>,
                     timeline_id,
                 })
                 .map(|remote_entry| RemoteTimelineInfo {
-                    remote_consistent_lsn: remote_entry.metadata.disk_consistent_lsn(),
-                    awaits_download: remote_entry.awaits_download,
+                    remote_consistent_lsn: remote_entry.disk_consistent_lsn(),
+                    awaits_download: remote_entry.get_awaits_download(),
                 }),
         })
     }
 
-    json_response(StatusCode::OK, response_data)
+    Ok(json_response(StatusCode::OK, response_data)?)
 }
 
 // Gate non incremental logical size calculation behind a flag
@@ -210,8 +188,8 @@ async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body
                 timeline_id,
             })
             .map(|remote_entry| RemoteTimelineInfo {
-                remote_consistent_lsn: remote_entry.metadata.disk_consistent_lsn(),
-                awaits_download: remote_entry.awaits_download,
+                remote_consistent_lsn: remote_entry.disk_consistent_lsn(),
+                awaits_download: remote_entry.get_awaits_download(),
             })
     };
 
@@ -230,7 +208,7 @@ async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body
         remote: remote_timeline_info,
     };
 
-    json_response(StatusCode::OK, timeline_info)
+    Ok(json_response(StatusCode::OK, timeline_info)?)
 }
 
 async fn timeline_attach_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -238,103 +216,39 @@ async fn timeline_attach_handler(request: Request<Body>) -> Result<Response<Body
     check_permission(&request, Some(tenant_id))?;
 
     let timeline_id: ZTimelineId = parse_request_param(&request, "timeline_id")?;
-    info!(
-        "Handling timeline {} attach for tenant: {}",
-        timeline_id, tenant_id,
-    );
+    let span = info_span!("timeline_attach_handler", tenant = %tenant_id, timeline = %timeline_id);
 
-    tokio::task::spawn_blocking(move || {
+    let span = tokio::task::spawn_blocking(move || {
+        let entered = span.entered();
         if tenant_mgr::get_timeline_for_tenant_load(tenant_id, timeline_id).is_ok() {
             // TODO: maybe answer with 309 Not Modified here?
             anyhow::bail!("Timeline is already present locally")
         };
-        Ok(())
+        Ok(entered.exit())
     })
     .await
     .map_err(ApiError::from_err)??;
 
-    let sync_id = ZTenantTimelineId {
-        tenant_id,
-        timeline_id,
-    };
-    let state = get_state(&request);
-    let remote_index = &state.remote_index;
+    let mut remote_index_write = get_state(&request).remote_index.write().await;
 
-    let mut index_accessor = remote_index.write().await;
-    if let Some(remote_timeline) = index_accessor.timeline_entry_mut(&sync_id) {
-        if remote_timeline.awaits_download {
-            return Err(ApiError::Conflict(
-                "Timeline download is already in progress".to_string(),
-            ));
-        }
-
-        remote_timeline.awaits_download = true;
-        schedule_timeline_download(tenant_id, timeline_id);
-        return json_response(StatusCode::ACCEPTED, ());
-    } else {
-        // no timeline in the index, release the lock to make the potentially lengthy download opetation
-        drop(index_accessor);
-    }
-
-    let new_timeline = match try_download_shard_data(state, sync_id).await {
-        Ok(Some(mut new_timeline)) => {
-            tokio::fs::create_dir_all(state.conf.timeline_path(&timeline_id, &tenant_id))
-                .await
-                .context("Failed to create new timeline directory")?;
-            new_timeline.awaits_download = true;
-            new_timeline
-        }
-        Ok(None) => return Err(ApiError::NotFound("Unknown remote timeline".to_string())),
-        Err(e) => {
-            error!("Failed to retrieve remote timeline data: {:?}", e);
-            return Err(ApiError::NotFound(
-                "Failed to retrieve remote timeline".to_string(),
-            ));
-        }
-    };
-
-    let mut index_accessor = remote_index.write().await;
-    match index_accessor.timeline_entry_mut(&sync_id) {
-        Some(remote_timeline) => {
-            if remote_timeline.awaits_download {
-                return Err(ApiError::Conflict(
-                    "Timeline download is already in progress".to_string(),
-                ));
-            }
-            remote_timeline.awaits_download = true;
-        }
-        None => index_accessor.add_timeline_entry(sync_id, new_timeline),
-    }
-    schedule_timeline_download(tenant_id, timeline_id);
-    json_response(StatusCode::ACCEPTED, ())
-}
-
-async fn try_download_shard_data(
-    state: &State,
-    sync_id: ZTenantTimelineId,
-) -> anyhow::Result<Option<RemoteTimeline>> {
-    let shard = match state.remote_storage.as_ref() {
-        Some(GenericRemoteStorage::Local(local_storage)) => {
-            download_index_part(state.conf, local_storage, sync_id).await
-        }
-        Some(GenericRemoteStorage::S3(s3_storage)) => {
-            download_index_part(state.conf, s3_storage, sync_id).await
-        }
-        None => return Ok(None),
-    }
-    .with_context(|| format!("Failed to download index shard for timeline {}", sync_id))?;
-
-    let timeline_path = state
-        .conf
-        .timeline_path(&sync_id.timeline_id, &sync_id.tenant_id);
-    RemoteTimeline::from_index_part(&timeline_path, shard)
-        .map(Some)
-        .with_context(|| {
-            format!(
-                "Failed to convert index shard into remote timeline for timeline {}",
-                sync_id
-            )
+    let _enter = span.entered(); // entered guard cannot live across awaits (non Send)
+    let index_entry = remote_index_write
+        .timeline_entry_mut(&ZTenantTimelineId {
+            tenant_id,
+            timeline_id,
         })
+        .ok_or_else(|| ApiError::NotFound("Unknown remote timeline".to_string()))?;
+
+    if index_entry.get_awaits_download() {
+        return Err(ApiError::Conflict(
+            "Timeline download is already in progress".to_string(),
+        ));
+    }
+
+    index_entry.set_awaits_download(true);
+    schedule_timeline_download(tenant_id, timeline_id);
+
+    Ok(json_response(StatusCode::ACCEPTED, ())?)
 }
 
 async fn timeline_detach_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -353,7 +267,7 @@ async fn timeline_detach_handler(request: Request<Body>) -> Result<Response<Body
     .await
     .map_err(ApiError::from_err)??;
 
-    json_response(StatusCode::OK, ())
+    Ok(json_response(StatusCode::OK, ())?)
 }
 
 async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -367,7 +281,7 @@ async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, A
     .await
     .map_err(ApiError::from_err)??;
 
-    json_response(StatusCode::OK, response_data)
+    Ok(json_response(StatusCode::OK, response_data)?)
 }
 
 async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -377,27 +291,28 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
     let request_data: TenantCreateRequest = json_request(&mut request).await?;
     let remote_index = get_state(&request).remote_index.clone();
 
-    let mut tenant_conf: TenantConfOpt = Default::default();
+    let mut tenant_conf = TenantConf::default();
     if let Some(gc_period) = request_data.gc_period {
         tenant_conf.gc_period =
-            Some(humantime::parse_duration(&gc_period).map_err(ApiError::from_err)?);
+            humantime::parse_duration(&gc_period).map_err(ApiError::from_err)?;
+    }
+    if let Some(gc_horizon) = request_data.gc_horizon {
+        tenant_conf.gc_horizon = gc_horizon;
     }
-    tenant_conf.gc_horizon = request_data.gc_horizon;
-
     if let Some(pitr_interval) = request_data.pitr_interval {
         tenant_conf.pitr_interval =
-            Some(humantime::parse_duration(&pitr_interval).map_err(ApiError::from_err)?);
+            humantime::parse_duration(&pitr_interval).map_err(ApiError::from_err)?;
+    }
+    if let Some(checkpoint_distance) = request_data.checkpoint_distance {
+        tenant_conf.checkpoint_distance = checkpoint_distance;
+    }
+    if let Some(compaction_target_size) = request_data.compaction_target_size {
+        tenant_conf.compaction_target_size = compaction_target_size;
     }
-
-    tenant_conf.checkpoint_distance = request_data.checkpoint_distance;
-    tenant_conf.compaction_target_size = request_data.compaction_target_size;
-    tenant_conf.compaction_threshold = request_data.compaction_threshold;
-
     if let Some(compaction_period) = request_data.compaction_period {
         tenant_conf.compaction_period =
-            Some(humantime::parse_duration(&compaction_period).map_err(ApiError::from_err)?);
+            humantime::parse_duration(&compaction_period).map_err(ApiError::from_err)?;
     }
-
     let target_tenant_id = request_data
         .new_tenant_id
         .map(ZTenantId::from)
@@ -418,44 +333,6 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
     })
 }
 
-async fn tenant_config_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let request_data: TenantConfigRequest = json_request(&mut request).await?;
-    let tenant_id = request_data.tenant_id;
-    // check for management permission
-    check_permission(&request, Some(tenant_id))?;
-
-    let mut tenant_conf: TenantConfOpt = Default::default();
-    if let Some(gc_period) = request_data.gc_period {
-        tenant_conf.gc_period =
-            Some(humantime::parse_duration(&gc_period).map_err(ApiError::from_err)?);
-    }
-    tenant_conf.gc_horizon = request_data.gc_horizon;
-
-    if let Some(pitr_interval) = request_data.pitr_interval {
-        tenant_conf.pitr_interval =
-            Some(humantime::parse_duration(&pitr_interval).map_err(ApiError::from_err)?);
-    }
-
-    tenant_conf.checkpoint_distance = request_data.checkpoint_distance;
-    tenant_conf.compaction_target_size = request_data.compaction_target_size;
-    tenant_conf.compaction_threshold = request_data.compaction_threshold;
-
-    if let Some(compaction_period) = request_data.compaction_period {
-        tenant_conf.compaction_period =
-            Some(humantime::parse_duration(&compaction_period).map_err(ApiError::from_err)?);
-    }
-
-    tokio::task::spawn_blocking(move || {
-        let _enter = info_span!("tenant_config", tenant = ?tenant_id).entered();
-
-        tenant_mgr::update_tenant_config(tenant_conf, tenant_id)
-    })
-    .await
-    .map_err(ApiError::from_err)??;
-
-    json_response(StatusCode::OK, ())
-}
-
 async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
     json_response(
         StatusCode::NOT_FOUND,
@@ -467,7 +344,7 @@ pub fn make_router(
     conf: &'static PageServerConf,
     auth: Option<Arc<JwtAuth>>,
     remote_index: RemoteIndex,
-) -> anyhow::Result<RouterBuilder<hyper::Body, ApiError>> {
+) -> RouterBuilder<hyper::Body, ApiError> {
     let spec = include_bytes!("openapi_spec.yml");
     let mut router = attach_openapi_ui(endpoint::make_router(), spec, "/swagger.yml", "/v1/doc");
     if auth.is_some() {
@@ -481,14 +358,11 @@ pub fn make_router(
         }))
     }
 
-    Ok(router
-        .data(Arc::new(
-            State::new(conf, auth, remote_index).context("Failed to initialize router state")?,
-        ))
+    router
+        .data(Arc::new(State::new(conf, auth, remote_index)))
         .get("/v1/status", status_handler)
         .get("/v1/tenant", tenant_list_handler)
         .post("/v1/tenant", tenant_create_handler)
-        .put("/v1/tenant/config", tenant_config_handler)
         .get("/v1/tenant/:tenant_id/timeline", timeline_list_handler)
         .post("/v1/tenant/:tenant_id/timeline", timeline_create_handler)
         .get(
@@ -503,5 +377,5 @@ pub fn make_router(
             "/v1/tenant/:tenant_id/timeline/:timeline_id/detach",
             timeline_detach_handler,
         )
-        .any(handler_404))
+        .any(handler_404)
 }

pageserver/src/import_datadir.rs

@@ -20,7 +20,7 @@ use postgres_ffi::waldecoder::*;
 use postgres_ffi::xlog_utils::*;
 use postgres_ffi::{pg_constants, ControlFileData, DBState_DB_SHUTDOWNED};
 use postgres_ffi::{Oid, TransactionId};
-use utils::lsn::Lsn;
+use zenith_utils::lsn::Lsn;
 
 ///
 /// Import all relation data pages from local disk into the repository.

pageserver/src/layered_repository.rs

@@ -34,7 +34,7 @@ use std::time::{Duration, Instant, SystemTime};
 use self::metadata::{metadata_path, TimelineMetadata, METADATA_FILE_NAME};
 use crate::config::PageServerConf;
 use crate::keyspace::KeySpace;
-use crate::tenant_config::{TenantConf, TenantConfOpt};
+use crate::tenant_config::{TenantConf, TenantConfFile};
 
 use crate::page_cache;
 use crate::remote_storage::{schedule_timeline_checkpoint_upload, RemoteIndex};
@@ -48,18 +48,14 @@ use crate::virtual_file::VirtualFile;
 use crate::walreceiver::IS_WAL_RECEIVER;
 use crate::walredo::WalRedoManager;
 use crate::CheckpointConfig;
+use crate::{ZTenantId, ZTimelineId};
 
-use metrics::{
-    register_histogram_vec, register_int_counter, register_int_counter_vec, register_int_gauge_vec,
-    Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec,
-};
-use toml_edit;
-use utils::{
-    crashsafe_dir,
-    lsn::{AtomicLsn, Lsn, RecordLsn},
-    seqwait::SeqWait,
-    zid::{ZTenantId, ZTimelineId},
+use zenith_metrics::{
+    register_histogram_vec, register_int_gauge_vec, Histogram, HistogramVec, IntGauge, IntGaugeVec,
 };
+use zenith_utils::crashsafe_dir;
+use zenith_utils::lsn::{AtomicLsn, Lsn, RecordLsn};
+use zenith_utils::seqwait::SeqWait;
 
 mod blob_io;
 pub mod block_io;
@@ -106,21 +102,6 @@ lazy_static! {
     .expect("failed to define a metric");
 }
 
-lazy_static! {
-    static ref MATERIALIZED_PAGE_CACHE_HIT: IntCounterVec = register_int_counter_vec!(
-        "materialize_page_cache_hits",
-        "Number of cache hits from materialized page cache",
-        &["tenant_id", "timeline_id"]
-    )
-    .expect("failed to define a metric");
-    static ref WAIT_LSN_TIME: HistogramVec = register_histogram_vec!(
-        "wait_lsn_time",
-        "Time spent waiting for WAL to arrive",
-        &["tenant_id", "timeline_id"]
-    )
-    .expect("failed to define a metric");
-}
-
 lazy_static! {
     static ref LAST_RECORD_LSN: IntGaugeVec = register_int_gauge_vec!(
         "pageserver_last_record_lsn",
@@ -130,21 +111,6 @@ lazy_static! {
     .expect("failed to define a metric");
 }
 
-// Metrics for cloud upload. These metrics reflect data uploaded to cloud storage,
-// or in testing they estimate how much we would upload if we did.
-lazy_static! {
-    static ref NUM_PERSISTENT_FILES_CREATED: IntCounter = register_int_counter!(
-        "pageserver_num_persistent_files_created",
-        "Number of files created that are meant to be uploaded to cloud storage",
-    )
-    .expect("failed to define a metric");
-    static ref PERSISTENT_BYTES_WRITTEN: IntCounter = register_int_counter!(
-        "pageserver_persistent_bytes_written",
-        "Total bytes written that are meant to be uploaded to cloud storage",
-    )
-    .expect("failed to define a metric");
-}
-
 /// Parts of the `.zenith/tenants/<tenantid>/timelines/<timelineid>` directory prefix.
 pub const TIMELINES_SEGMENT_NAME: &str = "timelines";
 
@@ -152,14 +118,8 @@ pub const TIMELINES_SEGMENT_NAME: &str = "timelines";
 /// Repository consists of multiple timelines. Keep them in a hash table.
 ///
 pub struct LayeredRepository {
-    // Global pageserver config parameters
     pub conf: &'static PageServerConf,
-
-    // Overridden tenant-specific config parameters.
-    // We keep TenantConfOpt sturct here to preserve the information
-    // about parameters that are not set.
-    // This is necessary to allow global config updates.
-    tenant_conf: Arc<RwLock<TenantConfOpt>>,
+    tenant_conf: TenantConf,
 
     tenantid: ZTenantId,
     timelines: Mutex<HashMap<ZTimelineId, LayeredTimelineEntry>>,
@@ -184,6 +144,10 @@ pub struct LayeredRepository {
 impl Repository for LayeredRepository {
     type Timeline = LayeredTimeline;
 
+    fn get_tenant_conf(&self) -> TenantConf {
+        self.tenant_conf
+    }
+
     fn get_timeline(&self, timelineid: ZTimelineId) -> Option<RepositoryTimeline<Self::Timeline>> {
         let timelines = self.timelines.lock().unwrap();
         self.get_timeline_internal(timelineid, &timelines)
@@ -230,7 +194,6 @@ impl Repository for LayeredRepository {
 
         let timeline = LayeredTimeline::new(
             self.conf,
-            Arc::clone(&self.tenant_conf),
             metadata,
             None,
             timelineid,
@@ -238,7 +201,7 @@ impl Repository for LayeredRepository {
             Arc::clone(&self.walredo_mgr),
             self.upload_layers,
         );
-        timeline.layers.write().unwrap().next_open_layer_at = Some(initdb_lsn);
+        timeline.layers.lock().unwrap().next_open_layer_at = Some(initdb_lsn);
 
         let timeline = Arc::new(timeline);
         let r = timelines.insert(
@@ -417,6 +380,8 @@ impl Repository for LayeredRepository {
             timeline_id, timeline_sync_status_update
         );
         match timeline_sync_status_update {
+            TimelineSyncStatusUpdate::Uploaded => { /* nothing to do, remote consistent lsn is managed by the remote storage */
+            }
             TimelineSyncStatusUpdate::Downloaded => {
                 match self.timelines.lock().unwrap().entry(timeline_id) {
                     Entry::Occupied(_) => bail!("We completed a download for a timeline that already exists in repository. This is a bug."),
@@ -493,64 +458,6 @@ impl From<LayeredTimelineEntry> for RepositoryTimeline<LayeredTimeline> {
 
 /// Private functions
 impl LayeredRepository {
-    pub fn get_checkpoint_distance(&self) -> u64 {
-        let tenant_conf = self.tenant_conf.read().unwrap();
-        tenant_conf
-            .checkpoint_distance
-            .unwrap_or(self.conf.default_tenant_conf.checkpoint_distance)
-    }
-
-    pub fn get_compaction_target_size(&self) -> u64 {
-        let tenant_conf = self.tenant_conf.read().unwrap();
-        tenant_conf
-            .compaction_target_size
-            .unwrap_or(self.conf.default_tenant_conf.compaction_target_size)
-    }
-
-    pub fn get_compaction_period(&self) -> Duration {
-        let tenant_conf = self.tenant_conf.read().unwrap();
-        tenant_conf
-            .compaction_period
-            .unwrap_or(self.conf.default_tenant_conf.compaction_period)
-    }
-
-    pub fn get_compaction_threshold(&self) -> usize {
-        let tenant_conf = self.tenant_conf.read().unwrap();
-        tenant_conf
-            .compaction_threshold
-            .unwrap_or(self.conf.default_tenant_conf.compaction_threshold)
-    }
-
-    pub fn get_gc_horizon(&self) -> u64 {
-        let tenant_conf = self.tenant_conf.read().unwrap();
-        tenant_conf
-            .gc_horizon
-            .unwrap_or(self.conf.default_tenant_conf.gc_horizon)
-    }
-
-    pub fn get_gc_period(&self) -> Duration {
-        let tenant_conf = self.tenant_conf.read().unwrap();
-        tenant_conf
-            .gc_period
-            .unwrap_or(self.conf.default_tenant_conf.gc_period)
-    }
-
-    pub fn get_pitr_interval(&self) -> Duration {
-        let tenant_conf = self.tenant_conf.read().unwrap();
-        tenant_conf
-            .pitr_interval
-            .unwrap_or(self.conf.default_tenant_conf.pitr_interval)
-    }
-
-    pub fn update_tenant_config(&self, new_tenant_conf: TenantConfOpt) -> Result<()> {
-        let mut tenant_conf = self.tenant_conf.write().unwrap();
-
-        tenant_conf.update(&new_tenant_conf);
-
-        LayeredRepository::persist_tenant_config(self.conf, self.tenantid, *tenant_conf)?;
-        Ok(())
-    }
-
     // Implementation of the public `get_timeline` function.
     // Differences from the public:
     //  * interface in that the caller must already hold the mutex on the 'timelines' hashmap.
@@ -624,10 +531,8 @@ impl LayeredRepository {
             .flatten()
             .map(LayeredTimelineEntry::Loaded);
         let _enter = info_span!("loading local timeline").entered();
-
         let timeline = LayeredTimeline::new(
             self.conf,
-            Arc::clone(&self.tenant_conf),
             metadata,
             ancestor,
             timelineid,
@@ -644,7 +549,7 @@ impl LayeredRepository {
 
     pub fn new(
         conf: &'static PageServerConf,
-        tenant_conf: TenantConfOpt,
+        tenant_conf: TenantConf,
         walredo_mgr: Arc<dyn WalRedoManager + Send + Sync>,
         tenantid: ZTenantId,
         remote_index: RemoteIndex,
@@ -653,7 +558,7 @@ impl LayeredRepository {
         LayeredRepository {
             tenantid,
             conf,
-            tenant_conf: Arc::new(RwLock::new(tenant_conf)),
+            tenant_conf,
             timelines: Mutex::new(HashMap::new()),
             gc_cs: Mutex::new(()),
             walredo_mgr,
@@ -662,69 +567,56 @@ impl LayeredRepository {
         }
     }
 
-    /// Locate and load config
-    pub fn load_tenant_config(
+    /// Save tenant's config to file
+    pub fn save_tenantconf(
         conf: &'static PageServerConf,
         tenantid: ZTenantId,
-    ) -> anyhow::Result<TenantConfOpt> {
-        let target_config_path = TenantConf::path(conf, tenantid);
+        tenant_conf: TenantConf,
+        first_save: bool,
+    ) -> Result<()> {
+        let _enter = info_span!("saving tenantconf").entered();
+        let path = TenantConf::tenantconf_path(conf, tenantid);
+        info!("save tenantconf to {}", path.display());
 
-        info!("load tenantconf from {}", target_config_path.display());
+        // use OpenOptions to ensure file presence is consistent with first_save
+        let mut file = VirtualFile::open_with_options(
+            &path,
+            OpenOptions::new().write(true).create_new(first_save),
+        )?;
 
-        // FIXME If the config file is not found, assume that we're attaching
-        // a detached tenant and config is passed via attach command.
-        // https://github.com/neondatabase/neon/issues/1555
-        if !target_config_path.exists() {
-            info!(
-                "Zenith tenant config is not found in {}",
-                target_config_path.display()
-            );
-            return Ok(Default::default());
+        let data = TenantConfFile::from(tenant_conf);
+        let tenantconf_bytes = data.to_bytes().context("Failed to get tenantconf bytes")?;
+
+        if file.write(&tenantconf_bytes)? != tenantconf_bytes.len() {
+            bail!("Could not write all the tenantconf bytes in a single call");
+        }
+        file.sync_all()?;
+
+        // fsync the parent directory to ensure the directory entry is durable
+        if first_save {
+            let tenant_dir = File::open(
+                &path
+                    .parent()
+                    .expect("Tenantconf should always have a parent dir"),
+            )?;
+            tenant_dir.sync_all()?;
         }
 
-        // load and parse file
-        let config = fs::read_to_string(target_config_path)?;
-
-        let toml = config.parse::<toml_edit::Document>()?;
-
-        let mut tenant_conf: TenantConfOpt = Default::default();
-        for (key, item) in toml.iter() {
-            match key {
-                "tenant_conf" => {
-                    tenant_conf = PageServerConf::parse_toml_tenant_conf(item)?;
-                }
-                _ => bail!("unrecognized pageserver option '{}'", key),
-            }
-        }
-
-        Ok(tenant_conf)
+        Ok(())
     }
 
-    pub fn persist_tenant_config(
+    pub fn load_tenantconf(
         conf: &'static PageServerConf,
         tenantid: ZTenantId,
-        tenant_conf: TenantConfOpt,
-    ) -> anyhow::Result<()> {
-        let _enter = info_span!("saving tenantconf").entered();
-        let target_config_path = TenantConf::path(conf, tenantid);
-        info!("save tenantconf to {}", target_config_path.display());
-
-        let mut conf_content = r#"# This file contains a specific per-tenant's config.
-#  It is read in case of pageserver restart.
-
-# [tenant_config]
-"#
-        .to_string();
-
-        // Convert the config to a toml file.
-        conf_content += &toml_edit::easy::to_string(&tenant_conf)?;
-
-        fs::write(&target_config_path, conf_content).with_context(|| {
-            format!(
-                "Failed to write config file into path '{}'",
-                target_config_path.display()
-            )
-        })
+    ) -> Result<TenantConf> {
+        let path = TenantConf::tenantconf_path(conf, tenantid);
+        info!("loading tenantconf from {}", path.display());
+        let tenantconf_bytes = std::fs::read(&path)?;
+        let tenant_conf_file = TenantConfFile::from_bytes(&tenantconf_bytes);
+        match tenant_conf_file {
+            Ok(tenant_conf) => return Ok(tenant_conf.body),
+            Err(err) => return Err(err),
+        };
     }
 
     /// Save timeline metadata to file
@@ -806,8 +698,7 @@ impl LayeredRepository {
         checkpoint_before_gc: bool,
     ) -> Result<GcResult> {
         let _span_guard =
-            info_span!("gc iteration", tenant = %self.tenantid, timeline = ?target_timelineid)
-                .entered();
+            info_span!("gc iteration", tenant = %self.tenantid, timeline = ?target_timelineid);
         let mut totals: GcResult = Default::default();
         let now = Instant::now();
 
@@ -894,12 +785,11 @@ impl LayeredRepository {
 
 pub struct LayeredTimeline {
     conf: &'static PageServerConf,
-    tenant_conf: Arc<RwLock<TenantConfOpt>>,
 
     tenantid: ZTenantId,
     timelineid: ZTimelineId,
 
-    layers: RwLock<LayerMap>,
+    layers: Mutex<LayerMap>,
 
     last_freeze_at: AtomicLsn,
 
@@ -937,12 +827,10 @@ pub struct LayeredTimeline {
 
     // Metrics
     reconstruct_time_histo: Histogram,
-    materialized_page_cache_hit_counter: IntCounter,
     flush_time_histo: Histogram,
     compact_time_histo: Histogram,
     create_images_time_histo: Histogram,
     last_record_gauge: IntGauge,
-    wait_lsn_time_histo: Histogram,
 
     /// If `true`, will backup its files that appear after each checkpointing to the remote storage.
     upload_layers: AtomicBool,
@@ -1027,15 +915,14 @@ impl Timeline for LayeredTimeline {
             "wait_lsn called by WAL receiver thread"
         );
 
-        self.wait_lsn_time_histo.observe_closure_duration(
-            || self.last_record_lsn
-                .wait_for_timeout(lsn, self.conf.wait_lsn_timeout)
-                .with_context(|| {
-                    format!(
-                        "Timed out while waiting for WAL record at LSN {} to arrive, last_record_lsn {} disk consistent LSN={}",
-                        lsn, self.get_last_record_lsn(), self.get_disk_consistent_lsn()
-                    )
-                }))?;
+        self.last_record_lsn
+            .wait_for_timeout(lsn, self.conf.wait_lsn_timeout)
+            .with_context(|| {
+                format!(
+                    "Timed out while waiting for WAL record at LSN {} to arrive, last_record_lsn {} disk consistent LSN={}",
+                    lsn, self.get_last_record_lsn(), self.get_disk_consistent_lsn()
+                )
+            })?;
 
         Ok(())
     }
@@ -1134,34 +1021,12 @@ impl Timeline for LayeredTimeline {
 }
 
 impl LayeredTimeline {
-    fn get_checkpoint_distance(&self) -> u64 {
-        let tenant_conf = self.tenant_conf.read().unwrap();
-        tenant_conf
-            .checkpoint_distance
-            .unwrap_or(self.conf.default_tenant_conf.checkpoint_distance)
-    }
-
-    fn get_compaction_target_size(&self) -> u64 {
-        let tenant_conf = self.tenant_conf.read().unwrap();
-        tenant_conf
-            .compaction_target_size
-            .unwrap_or(self.conf.default_tenant_conf.compaction_target_size)
-    }
-
-    fn get_compaction_threshold(&self) -> usize {
-        let tenant_conf = self.tenant_conf.read().unwrap();
-        tenant_conf
-            .compaction_threshold
-            .unwrap_or(self.conf.default_tenant_conf.compaction_threshold)
-    }
-
     /// Open a Timeline handle.
     ///
     /// Loads the metadata for the timeline into memory, but not the layer map.
     #[allow(clippy::too_many_arguments)]
     fn new(
         conf: &'static PageServerConf,
-        tenant_conf: Arc<RwLock<TenantConfOpt>>,
         metadata: TimelineMetadata,
         ancestor: Option<LayeredTimelineEntry>,
         timelineid: ZTimelineId,
@@ -1172,9 +1037,6 @@ impl LayeredTimeline {
         let reconstruct_time_histo = RECONSTRUCT_TIME
             .get_metric_with_label_values(&[&tenantid.to_string(), &timelineid.to_string()])
             .unwrap();
-        let materialized_page_cache_hit_counter = MATERIALIZED_PAGE_CACHE_HIT
-            .get_metric_with_label_values(&[&tenantid.to_string(), &timelineid.to_string()])
-            .unwrap();
         let flush_time_histo = STORAGE_TIME
             .get_metric_with_label_values(&[
                 "layer flush",
@@ -1199,16 +1061,12 @@ impl LayeredTimeline {
         let last_record_gauge = LAST_RECORD_LSN
             .get_metric_with_label_values(&[&tenantid.to_string(), &timelineid.to_string()])
             .unwrap();
-        let wait_lsn_time_histo = WAIT_LSN_TIME
-            .get_metric_with_label_values(&[&tenantid.to_string(), &timelineid.to_string()])
-            .unwrap();
 
         LayeredTimeline {
             conf,
-            tenant_conf,
             timelineid,
             tenantid,
-            layers: RwLock::new(LayerMap::default()),
+            layers: Mutex::new(LayerMap::default()),
 
             walredo_mgr,
 
@@ -1225,12 +1083,10 @@ impl LayeredTimeline {
             ancestor_lsn: metadata.ancestor_lsn(),
 
             reconstruct_time_histo,
-            materialized_page_cache_hit_counter,
             flush_time_histo,
             compact_time_histo,
             create_images_time_histo,
             last_record_gauge,
-            wait_lsn_time_histo,
 
             upload_layers: AtomicBool::new(upload_layers),
 
@@ -1254,7 +1110,7 @@ impl LayeredTimeline {
     /// Returns all timeline-related files that were found and loaded.
     ///
     fn load_layer_map(&self, disk_consistent_lsn: Lsn) -> anyhow::Result<()> {
-        let mut layers = self.layers.write().unwrap();
+        let mut layers = self.layers.lock().unwrap();
         let mut num_layers = 0;
 
         // Scan timeline directory and create ImageFileName and DeltaFilename
@@ -1348,12 +1204,6 @@ impl LayeredTimeline {
 
         let mut path: Vec<(ValueReconstructResult, Lsn, Arc<dyn Layer>)> = Vec::new();
 
-        let cached_lsn = if let Some((cached_lsn, _)) = &reconstruct_state.img {
-            *cached_lsn
-        } else {
-            Lsn(0)
-        };
-
         // 'prev_lsn' tracks the last LSN that we were at in our search. It's used
         // to check that each iteration make some progress, to break infinite
         // looping if something goes wrong.
@@ -1364,15 +1214,10 @@ impl LayeredTimeline {
 
         'outer: loop {
             // The function should have updated 'state'
-            //info!("CALLED for {} at {}: {:?} with {} records, cached {}", key, cont_lsn, result, reconstruct_state.records.len(), cached_lsn);
+            //info!("CALLED for {} at {}: {:?} with {} records", reconstruct_state.key, reconstruct_state.lsn, result, reconstruct_state.records.len());
             match result {
                 ValueReconstructResult::Complete => return Ok(()),
                 ValueReconstructResult::Continue => {
-                    // If we reached an earlier cached page image, we're done.
-                    if cont_lsn == cached_lsn + 1 {
-                        self.materialized_page_cache_hit_counter.inc_by(1);
-                        return Ok(());
-                    }
                     if prev_lsn <= cont_lsn {
                         // Didn't make any progress in last iteration. Error out to avoid
                         // getting stuck in the loop.
@@ -1419,22 +1264,19 @@ impl LayeredTimeline {
                 continue;
             }
 
-            let layers = timeline.layers.read().unwrap();
+            let layers = timeline.layers.lock().unwrap();
 
             // Check the open and frozen in-memory layers first
             if let Some(open_layer) = &layers.open_layer {
                 let start_lsn = open_layer.get_lsn_range().start;
                 if cont_lsn > start_lsn {
                     //info!("CHECKING for {} at {} on open layer {}", key, cont_lsn, open_layer.filename().display());
-                    // Get all the data needed to reconstruct the page version from this layer.
-                    // But if we have an older cached page image, no need to go past that.
-                    let lsn_floor = max(cached_lsn + 1, start_lsn);
                     result = open_layer.get_value_reconstruct_data(
                         key,
-                        lsn_floor..cont_lsn,
+                        open_layer.get_lsn_range().start..cont_lsn,
                         reconstruct_state,
                     )?;
-                    cont_lsn = lsn_floor;
+                    cont_lsn = start_lsn;
                     path.push((result, cont_lsn, open_layer.clone()));
                     continue;
                 }
@@ -1443,13 +1285,12 @@ impl LayeredTimeline {
                 let start_lsn = frozen_layer.get_lsn_range().start;
                 if cont_lsn > start_lsn {
                     //info!("CHECKING for {} at {} on frozen layer {}", key, cont_lsn, frozen_layer.filename().display());
-                    let lsn_floor = max(cached_lsn + 1, start_lsn);
                     result = frozen_layer.get_value_reconstruct_data(
                         key,
-                        lsn_floor..cont_lsn,
+                        frozen_layer.get_lsn_range().start..cont_lsn,
                         reconstruct_state,
                     )?;
-                    cont_lsn = lsn_floor;
+                    cont_lsn = start_lsn;
                     path.push((result, cont_lsn, frozen_layer.clone()));
                     continue 'outer;
                 }
@@ -1458,7 +1299,6 @@ impl LayeredTimeline {
             if let Some(SearchResult { lsn_floor, layer }) = layers.search(key, cont_lsn)? {
                 //info!("CHECKING for {} at {} on historic layer {}", key, cont_lsn, layer.filename().display());
 
-                let lsn_floor = max(cached_lsn + 1, lsn_floor);
                 result = layer.get_value_reconstruct_data(
                     key,
                     lsn_floor..cont_lsn,
@@ -1466,10 +1306,10 @@ impl LayeredTimeline {
                 )?;
                 cont_lsn = lsn_floor;
                 path.push((result, cont_lsn, layer));
-            } else if timeline.ancestor_timeline.is_some() {
+            } else if self.ancestor_timeline.is_some() {
                 // Nothing on this timeline. Traverse to parent
                 result = ValueReconstructResult::Continue;
-                cont_lsn = Lsn(timeline.ancestor_lsn.0 + 1);
+                cont_lsn = Lsn(self.ancestor_lsn.0 + 1);
             } else {
                 // Nothing found
                 result = ValueReconstructResult::Missing;
@@ -1506,7 +1346,7 @@ impl LayeredTimeline {
     /// Get a handle to the latest layer for appending.
     ///
     fn get_layer_for_write(&self, lsn: Lsn) -> anyhow::Result<Arc<InMemoryLayer>> {
-        let mut layers = self.layers.write().unwrap();
+        let mut layers = self.layers.lock().unwrap();
 
         ensure!(lsn.is_aligned());
 
@@ -1577,7 +1417,7 @@ impl LayeredTimeline {
         } else {
             Some(self.write_lock.lock().unwrap())
         };
-        let mut layers = self.layers.write().unwrap();
+        let mut layers = self.layers.lock().unwrap();
         if let Some(open_layer) = &layers.open_layer {
             let open_layer_rc = Arc::clone(open_layer);
             // Does this layer need freezing?
@@ -1600,9 +1440,11 @@ impl LayeredTimeline {
     ///
     pub fn check_checkpoint_distance(self: &Arc<LayeredTimeline>) -> Result<()> {
         let last_lsn = self.get_last_record_lsn();
+        let repo = tenant_mgr::get_repository_for_tenant(self.tenantid)?;
+        let tenant_conf = repo.get_tenant_conf();
 
         let distance = last_lsn.widening_sub(self.last_freeze_at.load());
-        if distance >= self.get_checkpoint_distance().into() {
+        if distance >= tenant_conf.checkpoint_distance.into() {
             self.freeze_inmem_layer(true);
             self.last_freeze_at.store(last_lsn);
         }
@@ -1642,7 +1484,7 @@ impl LayeredTimeline {
         let timer = self.flush_time_histo.start_timer();
 
         loop {
-            let layers = self.layers.read().unwrap();
+            let layers = self.layers.lock().unwrap();
             if let Some(frozen_layer) = layers.frozen_layers.front() {
                 let frozen_layer = Arc::clone(frozen_layer);
                 drop(layers); // to allow concurrent reads and writes
@@ -1686,7 +1528,7 @@ impl LayeredTimeline {
 
         // Finally, replace the frozen in-memory layer with the new on-disk layers
         {
-            let mut layers = self.layers.write().unwrap();
+            let mut layers = self.layers.lock().unwrap();
             let l = layers.frozen_layers.pop_front();
 
             // Only one thread may call this function at a time (for this
@@ -1704,7 +1546,8 @@ impl LayeredTimeline {
         //
         // TODO: This perhaps should be done in 'flush_frozen_layers', after flushing
         // *all* the layers, to avoid fsyncing the file multiple times.
-        let disk_consistent_lsn = Lsn(frozen_layer.get_lsn_range().end.0 - 1);
+        let disk_consistent_lsn;
+        disk_consistent_lsn = Lsn(frozen_layer.get_lsn_range().end.0 - 1);
 
         // If we were able to advance 'disk_consistent_lsn', save it the metadata file.
         // After crash, we will restart WAL streaming and processing from that point.
@@ -1754,15 +1597,11 @@ impl LayeredTimeline {
                 &metadata,
                 false,
             )?;
-
-            NUM_PERSISTENT_FILES_CREATED.inc_by(1);
-            PERSISTENT_BYTES_WRITTEN.inc_by(new_delta_path.metadata()?.len());
-
             if self.upload_layers.load(atomic::Ordering::Relaxed) {
                 schedule_timeline_checkpoint_upload(
                     self.tenantid,
                     self.timelineid,
-                    new_delta_path,
+                    vec![new_delta_path],
                     metadata,
                 );
             }
@@ -1811,14 +1650,17 @@ impl LayeredTimeline {
         // above. Rewrite it.
         let _compaction_cs = self.compaction_cs.lock().unwrap();
 
-        let target_file_size = self.get_checkpoint_distance();
+        let repo = tenant_mgr::get_repository_for_tenant(self.tenantid)?;
+        let tenant_conf = repo.get_tenant_conf();
+
+        let target_file_size = tenant_conf.checkpoint_distance;
 
         // Define partitioning schema if needed
         if let Ok(pgdir) = tenant_mgr::get_timeline_for_tenant_load(self.tenantid, self.timelineid)
         {
             let (partitioning, lsn) = pgdir.repartition(
                 self.get_last_record_lsn(),
-                self.get_compaction_target_size(),
+                tenant_conf.compaction_target_size,
             )?;
             let timer = self.create_images_time_histo.start_timer();
             // 2. Create new image layers for partitions that have been modified
@@ -1848,7 +1690,7 @@ impl LayeredTimeline {
         lsn: Lsn,
         threshold: usize,
     ) -> Result<bool> {
-        let layers = self.layers.read().unwrap();
+        let layers = self.layers.lock().unwrap();
 
         for part_range in &partition.ranges {
             let image_coverage = layers.image_coverage(part_range, lsn)?;
@@ -1906,7 +1748,7 @@ impl LayeredTimeline {
 
         // FIXME: Do we need to do something to upload it to remote storage here?
 
-        let mut layers = self.layers.write().unwrap();
+        let mut layers = self.layers.lock().unwrap();
         layers.insert_historic(Arc::new(image_layer));
         drop(layers);
 
@@ -1914,13 +1756,15 @@ impl LayeredTimeline {
     }
 
     fn compact_level0(&self, target_file_size: u64) -> Result<()> {
-        let layers = self.layers.read().unwrap();
+        let layers = self.layers.lock().unwrap();
+
+        // We compact or "shuffle" the level-0 delta layers when 10 have
+        // accumulated.
+        static COMPACT_THRESHOLD: usize = 10;
 
         let level0_deltas = layers.get_level0_deltas()?;
 
-        // We compact or "shuffle" the level-0 delta layers when they've
-        // accumulated over the compaction threshold.
-        if level0_deltas.len() < self.get_compaction_threshold() {
+        if level0_deltas.len() < COMPACT_THRESHOLD {
             return Ok(());
         }
         drop(layers);
@@ -2004,7 +1848,7 @@ impl LayeredTimeline {
             layer_paths.pop().unwrap();
         }
 
-        let mut layers = self.layers.write().unwrap();
+        let mut layers = self.layers.lock().unwrap();
         for l in new_layers {
             layers.insert_historic(Arc::new(l));
         }
@@ -2089,7 +1933,7 @@ impl LayeredTimeline {
         // 3. it doesn't need to be retained for 'retain_lsns';
         // 4. newer on-disk image layers cover the layer's whole key range
         //
-        let mut layers = self.layers.write().unwrap();
+        let mut layers = self.layers.lock().unwrap();
         'outer: for l in layers.iter_historic_layers() {
             // This layer is in the process of being flushed to disk.
             // It will be swapped out of the layer map, replaced with
@@ -2119,10 +1963,6 @@ impl LayeredTimeline {
             // It is not expected that users will need high precision here. And this estimation
             // is conservative: modification time of file is always newer than actual time of version
             // creation. So it is safe for users.
-            // TODO A possible "bloat" issue still persists here.
-            // If modification time changes because of layer upload/download, we will keep these files
-            // longer than necessary.
-            // https://github.com/neondatabase/neon/issues/1554
             //
             if let Ok(metadata) = fs::metadata(&l.filename()) {
                 let last_modified = metadata.modified()?;
@@ -2630,61 +2470,4 @@ pub mod tests {
 
         Ok(())
     }
-
-    #[test]
-    fn test_traverse_ancestors() -> Result<()> {
-        let repo = RepoHarness::create("test_traverse_ancestors")?.load();
-        let mut tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?;
-
-        const NUM_KEYS: usize = 100;
-        const NUM_TLINES: usize = 50;
-
-        let mut test_key = Key::from_hex("012222222233333333444444445500000000").unwrap();
-        // Track page mutation lsns across different timelines.
-        let mut updated = [[Lsn(0); NUM_KEYS]; NUM_TLINES];
-
-        let mut lsn = Lsn(0);
-        let mut tline_id = TIMELINE_ID;
-
-        #[allow(clippy::needless_range_loop)]
-        for idx in 0..NUM_TLINES {
-            let new_tline_id = ZTimelineId::generate();
-            repo.branch_timeline(tline_id, new_tline_id, lsn)?;
-            tline = repo.get_timeline_load(new_tline_id)?;
-            tline_id = new_tline_id;
-
-            for _ in 0..NUM_KEYS {
-                lsn = Lsn(lsn.0 + 0x10);
-                let blknum = thread_rng().gen_range(0..NUM_KEYS);
-                test_key.field6 = blknum as u32;
-                let writer = tline.writer();
-                writer.put(
-                    test_key,
-                    lsn,
-                    Value::Image(TEST_IMG(&format!("{} {} at {}", idx, blknum, lsn))),
-                )?;
-                println!("updating [{}][{}] at {}", idx, blknum, lsn);
-                writer.finish_write(lsn);
-                drop(writer);
-                updated[idx][blknum] = lsn;
-            }
-        }
-
-        // Read pages from leaf timeline across all ancestors.
-        for (idx, lsns) in updated.iter().enumerate() {
-            for (blknum, lsn) in lsns.iter().enumerate() {
-                // Skip empty mutations.
-                if lsn.0 == 0 {
-                    continue;
-                }
-                println!("chekcking [{}][{}] at {}", idx, blknum, lsn);
-                test_key.field6 = blknum as u32;
-                assert_eq!(
-                    tline.get(test_key, *lsn)?,
-                    TEST_IMG(&format!("{} {} at {}", idx, blknum, lsn))
-                );
-            }
-        }
-        Ok(())
-    }
 }

pageserver/src/layered_repository/blob_io.rs

@@ -1,20 +1,12 @@
 //!
 //! Functions for reading and writing variable-sized "blobs".
 //!
-//! Each blob begins with a 1- or 4-byte length field, followed by the
-//! actual data. If the length is smaller than 128 bytes, the length
-//! is written as a one byte. If it's larger than that, the length
-//! is written as a four-byte integer, in big-endian, with the high
-//! bit set. This way, we can detect whether it's 1- or 4-byte header
-//! by peeking at the first byte.
-//!
-//! len <  128: 0XXXXXXX
-//! len >= 128: 1XXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX
+//! Each blob begins with a 4-byte length, followed by the actual data.
 //!
 use crate::layered_repository::block_io::{BlockCursor, BlockReader};
 use crate::page_cache::PAGE_SZ;
 use std::cmp::min;
-use std::io::{Error, ErrorKind};
+use std::io::Error;
 
 /// For reading
 pub trait BlobCursor {
@@ -48,30 +40,21 @@ where
 
         let mut buf = self.read_blk(blknum)?;
 
-        // peek at the first byte, to determine if it's a 1- or 4-byte length
-        let first_len_byte = buf[off];
-        let len: usize = if first_len_byte < 0x80 {
-            // 1-byte length header
-            off += 1;
-            first_len_byte as usize
+        // read length
+        let mut len_buf = [0u8; 4];
+        let thislen = PAGE_SZ - off;
+        if thislen < 4 {
+            // it is split across two pages
+            len_buf[..thislen].copy_from_slice(&buf[off..PAGE_SZ]);
+            blknum += 1;
+            buf = self.read_blk(blknum)?;
+            len_buf[thislen..].copy_from_slice(&buf[0..4 - thislen]);
+            off = 4 - thislen;
         } else {
-            // 4-byte length header
-            let mut len_buf = [0u8; 4];
-            let thislen = PAGE_SZ - off;
-            if thislen < 4 {
-                // it is split across two pages
-                len_buf[..thislen].copy_from_slice(&buf[off..PAGE_SZ]);
-                blknum += 1;
-                buf = self.read_blk(blknum)?;
-                len_buf[thislen..].copy_from_slice(&buf[0..4 - thislen]);
-                off = 4 - thislen;
-            } else {
-                len_buf.copy_from_slice(&buf[off..off + 4]);
-                off += 4;
-            }
-            len_buf[0] &= 0x7f;
-            u32::from_be_bytes(len_buf) as usize
-        };
+            len_buf.copy_from_slice(&buf[off..off + 4]);
+            off += 4;
+        }
+        let len = u32::from_ne_bytes(len_buf) as usize;
 
         dstbuf.clear();
 
@@ -147,27 +130,10 @@ where
 {
     fn write_blob(&mut self, srcbuf: &[u8]) -> Result<u64, Error> {
         let offset = self.offset;
-
-        if srcbuf.len() < 128 {
-            // Short blob. Write a 1-byte length header
-            let len_buf = srcbuf.len() as u8;
-            self.inner.write_all(&[len_buf])?;
-            self.offset += 1;
-        } else {
-            // Write a 4-byte length header
-            if srcbuf.len() > 0x7fff_ffff {
-                return Err(Error::new(
-                    ErrorKind::Other,
-                    format!("blob too large ({} bytes)", srcbuf.len()),
-                ));
-            }
-            let mut len_buf = ((srcbuf.len()) as u32).to_be_bytes();
-            len_buf[0] |= 0x80;
-            self.inner.write_all(&len_buf)?;
-            self.offset += 4;
-        }
+        self.inner
+            .write_all(&((srcbuf.len()) as u32).to_ne_bytes())?;
         self.inner.write_all(srcbuf)?;
-        self.offset += srcbuf.len() as u64;
+        self.offset += 4 + srcbuf.len() as u64;
         Ok(offset)
     }
 }

pageserver/src/layered_repository/block_io.rs

@@ -198,6 +198,7 @@ impl BlockWriter for BlockBuf {
         assert!(buf.len() == PAGE_SZ);
         let blknum = self.blocks.len();
         self.blocks.push(buf);
+        tracing::info!("buffered block {}", blknum);
         Ok(blknum as u32)
     }
 }

pageserver/src/layered_repository/delta_layer.rs

@@ -35,10 +35,11 @@ use crate::page_cache::{PageReadGuard, PAGE_SZ};
 use crate::repository::{Key, Value, KEY_SIZE};
 use crate::virtual_file::VirtualFile;
 use crate::walrecord;
+use crate::{ZTenantId, ZTimelineId};
 use crate::{DELTA_FILE_MAGIC, STORAGE_FORMAT_VERSION};
 use anyhow::{bail, ensure, Context, Result};
+use log::*;
 use serde::{Deserialize, Serialize};
-use tracing::*;
 // avoid binding to Write (conflicts with std::io::Write)
 // while being able to use std::fmt::Write's methods
 use std::fmt::Write as _;
@@ -50,11 +51,8 @@ use std::os::unix::fs::FileExt;
 use std::path::{Path, PathBuf};
 use std::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
 
-use utils::{
-    bin_ser::BeSer,
-    lsn::Lsn,
-    zid::{ZTenantId, ZTimelineId},
-};
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
 
 ///
 /// Header stored in the beginning of the file
@@ -224,7 +222,6 @@ impl Layer for DeltaLayer {
         lsn_range: Range<Lsn>,
         reconstruct_state: &mut ValueReconstructState,
     ) -> anyhow::Result<ValueReconstructResult> {
-        ensure!(lsn_range.start >= self.lsn_range.start);
         let mut need_image = true;
 
         ensure!(self.key_range.contains(&key));
@@ -290,10 +287,7 @@ impl Layer for DeltaLayer {
     }
 
     fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = anyhow::Result<(Key, Lsn, Value)>> + 'a> {
-        let inner = match self.load() {
-            Ok(inner) => inner,
-            Err(e) => panic!("Failed to load a delta layer: {e:?}"),
-        };
+        let inner = self.load().unwrap();
 
         match DeltaValueIter::new(inner) {
             Ok(iter) => Box::new(iter),
@@ -425,9 +419,7 @@ impl DeltaLayer {
             drop(inner);
             let inner = self.inner.write().unwrap();
             if !inner.loaded {
-                self.load_inner(inner).with_context(|| {
-                    format!("Failed to load delta layer {}", self.path().display())
-                })?;
+                self.load_inner(inner)?;
             } else {
                 // Another thread loaded it while we were not holding the lock.
             }

pageserver/src/layered_repository/ephemeral_file.rs

@@ -16,8 +16,8 @@ use std::io::{Error, ErrorKind};
 use std::ops::DerefMut;
 use std::path::PathBuf;
 use std::sync::{Arc, RwLock};
-use tracing::*;
-use utils::zid::{ZTenantId, ZTimelineId};
+use zenith_utils::zid::ZTenantId;
+use zenith_utils::zid::ZTimelineId;
 
 use std::os::unix::fs::FileExt;
 
@@ -199,24 +199,18 @@ impl BlobWriter for EphemeralFile {
         let mut buf = self.get_buf_for_write(blknum)?;
 
         // Write the length field
-        if srcbuf.len() < 0x80 {
-            buf[off] = srcbuf.len() as u8;
-            off += 1;
+        let len_buf = u32::to_ne_bytes(srcbuf.len() as u32);
+        let thislen = PAGE_SZ - off;
+        if thislen < 4 {
+            // it needs to be split across pages
+            buf[off..(off + thislen)].copy_from_slice(&len_buf[..thislen]);
+            blknum += 1;
+            buf = self.get_buf_for_write(blknum)?;
+            buf[0..4 - thislen].copy_from_slice(&len_buf[thislen..]);
+            off = 4 - thislen;
         } else {
-            let mut len_buf = u32::to_be_bytes(srcbuf.len() as u32);
-            len_buf[0] |= 0x80;
-            let thislen = PAGE_SZ - off;
-            if thislen < 4 {
-                // it needs to be split across pages
-                buf[off..(off + thislen)].copy_from_slice(&len_buf[..thislen]);
-                blknum += 1;
-                buf = self.get_buf_for_write(blknum)?;
-                buf[0..4 - thislen].copy_from_slice(&len_buf[thislen..]);
-                off = 4 - thislen;
-            } else {
-                buf[off..off + 4].copy_from_slice(&len_buf);
-                off += 4;
-            }
+            buf[off..off + 4].copy_from_slice(&len_buf);
+            off += 4;
         }
 
         // Write the payload
@@ -235,13 +229,7 @@ impl BlobWriter for EphemeralFile {
             buf_remain = &buf_remain[this_blk_len..];
         }
         drop(buf);
-
-        if srcbuf.len() < 0x80 {
-            self.size += 1;
-        } else {
-            self.size += 4;
-        }
-        self.size += srcbuf.len() as u64;
+        self.size += 4 + srcbuf.len() as u64;
 
         Ok(pos)
     }
@@ -256,31 +244,16 @@ impl Drop for EphemeralFile {
         // remove entry from the hash map
         EPHEMERAL_FILES.write().unwrap().files.remove(&self.file_id);
 
-        // unlink the file
-        let res = std::fs::remove_file(&self.file.path);
-        if let Err(e) = res {
-            warn!(
-                "could not remove ephemeral file '{}': {}",
-                self.file.path.display(),
-                e
-            );
-        }
+        // unlink file
+        // FIXME: print error
+        let _ = std::fs::remove_file(&self.file.path);
     }
 }
 
 pub fn writeback(file_id: u64, blkno: u32, buf: &[u8]) -> Result<(), std::io::Error> {
     if let Some(file) = EPHEMERAL_FILES.read().unwrap().files.get(&file_id) {
-        match file.write_all_at(buf, blkno as u64 * PAGE_SZ as u64) {
-            Ok(_) => Ok(()),
-            Err(e) => Err(std::io::Error::new(
-                ErrorKind::Other,
-                format!(
-                    "failed to write back to ephemeral file at {} error: {}",
-                    file.path.display(),
-                    e
-                ),
-            )),
-        }
+        file.write_all_at(buf, blkno as u64 * PAGE_SZ as u64)?;
+        Ok(())
     } else {
         Err(std::io::Error::new(
             ErrorKind::Other,
@@ -399,12 +372,6 @@ mod tests {
             let pos = file.write_blob(&data)?;
             blobs.push((pos, data));
         }
-        // also test with a large blobs
-        for i in 0..100 {
-            let data = format!("blob{}", i).as_bytes().repeat(100);
-            let pos = file.write_blob(&data)?;
-            blobs.push((pos, data));
-        }
 
         let mut cursor = BlockCursor::new(&file);
         for (pos, expected) in blobs {

pageserver/src/layered_repository/filename.rs

@@ -8,7 +8,7 @@ use std::fmt;
 use std::ops::Range;
 use std::path::PathBuf;
 
-use utils::lsn::Lsn;
+use zenith_utils::lsn::Lsn;
 
 // Note: LayeredTimeline::load_layer_map() relies on this sort order
 #[derive(Debug, PartialEq, Eq, Clone)]
@@ -25,7 +25,9 @@ impl PartialOrd for DeltaFileName {
 
 impl Ord for DeltaFileName {
     fn cmp(&self, other: &Self) -> Ordering {
-        let mut cmp = self.key_range.start.cmp(&other.key_range.start);
+        let mut cmp;
+
+        cmp = self.key_range.start.cmp(&other.key_range.start);
         if cmp != Ordering::Equal {
             return cmp;
         }
@@ -115,7 +117,9 @@ impl PartialOrd for ImageFileName {
 
 impl Ord for ImageFileName {
     fn cmp(&self, other: &Self) -> Ordering {
-        let mut cmp = self.key_range.start.cmp(&other.key_range.start);
+        let mut cmp;
+
+        cmp = self.key_range.start.cmp(&other.key_range.start);
         if cmp != Ordering::Equal {
             return cmp;
         }

pageserver/src/layered_repository/image_layer.rs

@@ -30,10 +30,12 @@ use crate::layered_repository::storage_layer::{
 use crate::page_cache::PAGE_SZ;
 use crate::repository::{Key, Value, KEY_SIZE};
 use crate::virtual_file::VirtualFile;
+use crate::{ZTenantId, ZTimelineId};
 use crate::{IMAGE_FILE_MAGIC, STORAGE_FORMAT_VERSION};
 use anyhow::{bail, ensure, Context, Result};
 use bytes::Bytes;
 use hex;
+use log::*;
 use serde::{Deserialize, Serialize};
 use std::fs;
 use std::io::Write;
@@ -41,13 +43,9 @@ use std::io::{Seek, SeekFrom};
 use std::ops::Range;
 use std::path::{Path, PathBuf};
 use std::sync::{RwLock, RwLockReadGuard};
-use tracing::*;
 
-use utils::{
-    bin_ser::BeSer,
-    lsn::Lsn,
-    zid::{ZTenantId, ZTimelineId},
-};
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
 
 ///
 /// Header stored in the beginning of the file
@@ -150,7 +148,6 @@ impl Layer for ImageLayer {
         reconstruct_state: &mut ValueReconstructState,
     ) -> anyhow::Result<ValueReconstructResult> {
         assert!(self.key_range.contains(&key));
-        assert!(lsn_range.start >= self.lsn);
         assert!(lsn_range.end >= self.lsn);
 
         let inner = self.load()?;
@@ -254,9 +251,7 @@ impl ImageLayer {
             drop(inner);
             let mut inner = self.inner.write().unwrap();
             if !inner.loaded {
-                self.load_inner(&mut inner).with_context(|| {
-                    format!("Failed to load image layer {}", self.path().display())
-                })?
+                self.load_inner(&mut inner)?;
             } else {
                 // Another thread loaded it while we were not holding the lock.
             }

pageserver/src/layered_repository/inmemory_layer.rs

@@ -14,21 +14,19 @@ use crate::layered_repository::storage_layer::{
 };
 use crate::repository::{Key, Value};
 use crate::walrecord;
+use crate::{ZTenantId, ZTimelineId};
 use anyhow::{bail, ensure, Result};
+use log::*;
 use std::collections::HashMap;
-use tracing::*;
-use utils::{
-    bin_ser::BeSer,
-    lsn::Lsn,
-    vec_map::VecMap,
-    zid::{ZTenantId, ZTimelineId},
-};
 // avoid binding to Write (conflicts with std::io::Write)
 // while being able to use std::fmt::Write's methods
 use std::fmt::Write as _;
 use std::ops::Range;
 use std::path::PathBuf;
 use std::sync::RwLock;
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
+use zenith_utils::vec_map::VecMap;
 
 pub struct InMemoryLayer {
     conf: &'static PageServerConf,
@@ -115,7 +113,7 @@ impl Layer for InMemoryLayer {
         lsn_range: Range<Lsn>,
         reconstruct_state: &mut ValueReconstructState,
     ) -> anyhow::Result<ValueReconstructResult> {
-        ensure!(lsn_range.start >= self.start_lsn);
+        ensure!(lsn_range.start <= self.start_lsn);
         let mut need_image = true;
 
         let inner = self.inner.read().unwrap();
@@ -126,6 +124,13 @@ impl Layer for InMemoryLayer {
         if let Some(vec_map) = inner.index.get(&key) {
             let slice = vec_map.slice_range(lsn_range);
             for (entry_lsn, pos) in slice.iter().rev() {
+                match &reconstruct_state.img {
+                    Some((cached_lsn, _)) if entry_lsn <= cached_lsn => {
+                        return Ok(ValueReconstructResult::Complete)
+                    }
+                    _ => {}
+                }
+
                 let buf = reader.read_blob(*pos)?;
                 let value = Value::des(&buf)?;
                 match value {

pageserver/src/layered_repository/layer_map.rs

@@ -16,12 +16,12 @@ use crate::layered_repository::InMemoryLayer;
 use crate::repository::Key;
 use anyhow::Result;
 use lazy_static::lazy_static;
-use metrics::{register_int_gauge, IntGauge};
 use std::collections::VecDeque;
 use std::ops::Range;
 use std::sync::Arc;
 use tracing::*;
-use utils::lsn::Lsn;
+use zenith_metrics::{register_int_gauge, IntGauge};
+use zenith_utils::lsn::Lsn;
 
 lazy_static! {
     static ref NUM_ONDISK_LAYERS: IntGauge =
@@ -296,7 +296,9 @@ impl LayerMap {
         key_range: &Range<Key>,
         lsn: Lsn,
     ) -> Result<Vec<(Range<Key>, Option<Arc<dyn Layer>>)>> {
-        let mut points = vec![key_range.start];
+        let mut points: Vec<Key>;
+
+        points = vec![key_range.start];
         for l in self.historic_layers.iter() {
             if l.get_lsn_range().start > lsn {
                 continue;

pageserver/src/layered_repository/metadata.rs

@@ -10,7 +10,7 @@ use std::path::PathBuf;
 
 use anyhow::ensure;
 use serde::{Deserialize, Serialize};
-use utils::{
+use zenith_utils::{
     bin_ser::BeSer,
     lsn::Lsn,
     zid::{ZTenantId, ZTimelineId},

pageserver/src/layered_repository/storage_layer.rs

@@ -4,15 +4,13 @@
 
 use crate::repository::{Key, Value};
 use crate::walrecord::ZenithWalRecord;
+use crate::{ZTenantId, ZTimelineId};
 use anyhow::Result;
 use bytes::Bytes;
 use std::ops::Range;
 use std::path::PathBuf;
 
-use utils::{
-    lsn::Lsn,
-    zid::{ZTenantId, ZTimelineId},
-};
+use zenith_utils::lsn::Lsn;
 
 pub fn range_overlaps<T>(a: &Range<T>, b: &Range<T>) -> bool
 where

pageserver/src/lib.rs

@@ -7,7 +7,6 @@ pub mod layered_repository;
 pub mod page_cache;
 pub mod page_service;
 pub mod pgdatadir_mapping;
-pub mod profiling;
 pub mod reltag;
 pub mod remote_storage;
 pub mod repository;
@@ -17,7 +16,6 @@ pub mod tenant_threads;
 pub mod thread_mgr;
 pub mod timelines;
 pub mod virtual_file;
-pub mod wal_metadata;
 pub mod walingest;
 pub mod walreceiver;
 pub mod walrecord;
@@ -25,10 +23,13 @@ pub mod walredo;
 
 use lazy_static::lazy_static;
 use tracing::info;
-use utils::postgres_backend;
+use zenith_metrics::{register_int_gauge_vec, IntGaugeVec};
+use zenith_utils::{
+    postgres_backend,
+    zid::{ZTenantId, ZTimelineId},
+};
 
 use crate::thread_mgr::ThreadKind;
-use metrics::{register_int_gauge_vec, IntGaugeVec};
 
 use layered_repository::LayeredRepository;
 use pgdatadir_mapping::DatadirTimeline;

pageserver/src/page_cache.rs

@@ -47,7 +47,7 @@ use std::{
 
 use once_cell::sync::OnceCell;
 use tracing::error;
-use utils::{
+use zenith_utils::{
     lsn::Lsn,
     zid::{ZTenantId, ZTimelineId},
 };

pageserver/src/page_service.rs

@@ -21,18 +21,19 @@ use std::str::FromStr;
 use std::sync::{Arc, RwLockReadGuard};
 use std::time::Duration;
 use tracing::*;
-use utils::{
-    auth::{self, Claims, JwtAuth, Scope},
-    lsn::Lsn,
-    postgres_backend::{self, is_socket_read_timed_out, AuthType, PostgresBackend},
-    pq_proto::{BeMessage, FeMessage, RowDescriptor, SINGLE_COL_ROWDESC},
-    zid::{ZTenantId, ZTimelineId},
-};
+use zenith_metrics::{register_histogram_vec, HistogramVec};
+use zenith_utils::auth::{self, JwtAuth};
+use zenith_utils::auth::{Claims, Scope};
+use zenith_utils::lsn::Lsn;
+use zenith_utils::postgres_backend::is_socket_read_timed_out;
+use zenith_utils::postgres_backend::PostgresBackend;
+use zenith_utils::postgres_backend::{self, AuthType};
+use zenith_utils::pq_proto::{BeMessage, FeMessage, RowDescriptor, SINGLE_COL_ROWDESC};
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
 use crate::basebackup;
-use crate::config::{PageServerConf, ProfilingConfig};
+use crate::config::PageServerConf;
 use crate::pgdatadir_mapping::DatadirTimeline;
-use crate::profiling::profpoint_start;
 use crate::reltag::RelTag;
 use crate::repository::Repository;
 use crate::repository::Timeline;
@@ -41,7 +42,6 @@ use crate::thread_mgr;
 use crate::thread_mgr::ThreadKind;
 use crate::walreceiver;
 use crate::CheckpointConfig;
-use metrics::{register_histogram_vec, HistogramVec};
 
 // Wrapped in libpq CopyData
 enum PagestreamFeMessage {
@@ -333,10 +333,7 @@ impl PageServerHandler {
         pgb.write_message(&BeMessage::CopyBothResponse)?;
 
         while !thread_mgr::is_shutdown_requested() {
-            let msg = pgb.read_message();
-
-            let profiling_guard = profpoint_start(self.conf, ProfilingConfig::PageRequests);
-            match msg {
+            match pgb.read_message() {
                 Ok(message) => {
                     if let Some(message) = message {
                         trace!("query: {:?}", message);
@@ -388,7 +385,6 @@ impl PageServerHandler {
                     }
                 }
             }
-            drop(profiling_guard);
         }
         Ok(())
     }
@@ -684,28 +680,28 @@ impl postgres_backend::Handler for PageServerHandler {
             ensure!(params.len() == 1, "invalid param number for config command");
             let tenantid = ZTenantId::from_str(params[0])?;
             let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
+            let tenant_conf = repo.get_tenant_conf();
             pgb.write_message_noflush(&BeMessage::RowDescription(&[
                 RowDescriptor::int8_col(b"checkpoint_distance"),
                 RowDescriptor::int8_col(b"compaction_target_size"),
                 RowDescriptor::int8_col(b"compaction_period"),
-                RowDescriptor::int8_col(b"compaction_threshold"),
                 RowDescriptor::int8_col(b"gc_horizon"),
                 RowDescriptor::int8_col(b"gc_period"),
                 RowDescriptor::int8_col(b"pitr_interval"),
             ]))?
             .write_message_noflush(&BeMessage::DataRow(&[
-                Some(repo.get_checkpoint_distance().to_string().as_bytes()),
-                Some(repo.get_compaction_target_size().to_string().as_bytes()),
+                Some(tenant_conf.checkpoint_distance.to_string().as_bytes()),
+                Some(tenant_conf.compaction_target_size.to_string().as_bytes()),
                 Some(
-                    repo.get_compaction_period()
+                    tenant_conf
+                        .compaction_period
                         .as_secs()
                         .to_string()
                         .as_bytes(),
                 ),
-                Some(repo.get_compaction_threshold().to_string().as_bytes()),
-                Some(repo.get_gc_horizon().to_string().as_bytes()),
-                Some(repo.get_gc_period().as_secs().to_string().as_bytes()),
-                Some(repo.get_pitr_interval().as_secs().to_string().as_bytes()),
+                Some(tenant_conf.gc_horizon.to_string().as_bytes()),
+                Some(tenant_conf.gc_period.as_secs().to_string().as_bytes()),
+                Some(tenant_conf.pitr_interval.as_secs().to_string().as_bytes()),
             ]))?
             .write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;
         } else if query_string.starts_with("do_gc ") {
@@ -727,11 +723,12 @@ impl postgres_backend::Handler for PageServerHandler {
             let timelineid = ZTimelineId::from_str(caps.get(2).unwrap().as_str())?;
 
             let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
+            let tenant_conf = repo.get_tenant_conf();
 
             let gc_horizon: u64 = caps
                 .get(4)
                 .map(|h| h.as_str().parse())
-                .unwrap_or_else(|| Ok(repo.get_gc_horizon()))?;
+                .unwrap_or(Ok(tenant_conf.gc_horizon))?;
 
             let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
             let result = repo.gc_iteration(Some(timelineid), gc_horizon, Duration::ZERO, true)?;
@@ -754,26 +751,6 @@ impl postgres_backend::Handler for PageServerHandler {
                 Some(result.elapsed.as_millis().to_string().as_bytes()),
             ]))?
             .write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;
-        } else if query_string.starts_with("compact ") {
-            // Run compaction immediately on given timeline.
-            // FIXME This is just for tests. Don't expect this to be exposed to
-            // the users or the api.
-
-            // compact <tenant_id> <timeline_id>
-            let re = Regex::new(r"^compact ([[:xdigit:]]+)\s([[:xdigit:]]+)($|\s)?").unwrap();
-
-            let caps = re
-                .captures(query_string)
-                .with_context(|| format!("Invalid compact: '{}'", query_string))?;
-
-            let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?;
-            let timelineid = ZTimelineId::from_str(caps.get(2).unwrap().as_str())?;
-            let timeline = tenant_mgr::get_timeline_for_tenant_load(tenantid, timelineid)
-                .context("Couldn't load timeline")?;
-            timeline.tline.compact()?;
-
-            pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
-                .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
         } else if query_string.starts_with("checkpoint ") {
             // Run checkpoint immediately on given timeline.
 

pageserver/src/pgdatadir_mapping.rs

@@ -20,7 +20,8 @@ use std::ops::Range;
 use std::sync::atomic::{AtomicIsize, Ordering};
 use std::sync::{Arc, Mutex, RwLockReadGuard};
 use tracing::{debug, error, trace, warn};
-use utils::{bin_ser::BeSer, lsn::Lsn};
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
 
 /// Block number within a relation or SLRU. This matches PostgreSQL's BlockNumber type.
 pub type BlockNumber = u32;
@@ -1211,7 +1212,7 @@ pub fn key_to_slru_block(key: Key) -> Result<(SlruKind, u32, BlockNumber)> {
 #[cfg(test)]
 pub fn create_test_timeline<R: Repository>(
     repo: R,
-    timeline_id: utils::zid::ZTimelineId,
+    timeline_id: zenith_utils::zid::ZTimelineId,
 ) -> Result<Arc<crate::DatadirTimeline<R>>> {
     let tline = repo.create_empty_timeline(timeline_id, Lsn(8))?;
     let tline = DatadirTimeline::new(tline, 256 * 1024);

pageserver/src/profiling.rs

@@ -1,101 +0,0 @@
-//!
-//! Support for profiling
-//!
-//! This relies on a modified version of the 'pprof-rs' crate. That's not very
-//! nice, so to avoid a hard dependency on that, this is an optional feature.
-//!
-use crate::config::{PageServerConf, ProfilingConfig};
-
-/// The actual implementation is in the `profiling_impl` submodule. If the profiling
-/// feature is not enabled, it's just a dummy implementation that panics if you
-/// try to enabled profiling in the configuration.
-pub use profiling_impl::*;
-
-#[cfg(feature = "profiling")]
-mod profiling_impl {
-    use super::*;
-    use pprof;
-    use std::marker::PhantomData;
-
-    /// Start profiling the current thread. Returns a guard object;
-    /// the profiling continues until the guard is dropped.
-    ///
-    /// Note: profiling is not re-entrant. If you call 'profpoint_start' while
-    /// profiling is already started, nothing happens, and the profiling will be
-    /// stopped when either guard object is dropped.
-    #[inline]
-    pub fn profpoint_start(
-        conf: &crate::config::PageServerConf,
-        point: ProfilingConfig,
-    ) -> Option<ProfilingGuard> {
-        if conf.profiling == point {
-            pprof::start_profiling();
-            Some(ProfilingGuard(PhantomData))
-        } else {
-            None
-        }
-    }
-
-    /// A hack to remove Send and Sync from the ProfilingGuard. Because the
-    /// profiling is attached to current thread.
-    ////
-    /// See comments in https://github.com/rust-lang/rust/issues/68318
-    type PhantomUnsend = std::marker::PhantomData<*mut u8>;
-
-    pub struct ProfilingGuard(PhantomUnsend);
-
-    impl Drop for ProfilingGuard {
-        fn drop(&mut self) {
-            pprof::stop_profiling();
-        }
-    }
-
-    /// Initialize the profiler. This must be called before any 'profpoint_start' calls.
-    pub fn init_profiler(conf: &PageServerConf) -> Option<pprof::ProfilerGuard> {
-        if conf.profiling != ProfilingConfig::Disabled {
-            Some(pprof::ProfilerGuardBuilder::default().build().unwrap())
-        } else {
-            None
-        }
-    }
-
-    /// Exit the profiler. Writes the flamegraph to current workdir.
-    pub fn exit_profiler(_conf: &PageServerConf, profiler_guard: &Option<pprof::ProfilerGuard>) {
-        // Write out the flamegraph
-        if let Some(profiler_guard) = profiler_guard {
-            if let Ok(report) = profiler_guard.report().build() {
-                // this gets written under the workdir
-                let file = std::fs::File::create("flamegraph.svg").unwrap();
-                let mut options = pprof::flamegraph::Options::default();
-                options.image_width = Some(2500);
-                report.flamegraph_with_options(file, &mut options).unwrap();
-            }
-        }
-    }
-}
-
-/// Dummy implementation when compiling without profiling feature or for non-linux OSes.
-#[cfg(not(feature = "profiling"))]
-mod profiling_impl {
-    use super::*;
-
-    pub struct DummyProfilerGuard;
-
-    pub fn profpoint_start(
-        _conf: &PageServerConf,
-        _point: ProfilingConfig,
-    ) -> Option<DummyProfilerGuard> {
-        None
-    }
-
-    pub fn init_profiler(conf: &PageServerConf) -> Option<DummyProfilerGuard> {
-        if conf.profiling != ProfilingConfig::Disabled {
-            // shouldn't happen, we don't allow profiling in the config if the support
-            // for it is disabled.
-            panic!("profiling enabled but the binary was compiled without profiling support");
-        }
-        None
-    }
-
-    pub fn exit_profiler(_conf: &PageServerConf, _guard: &Option<DummyProfilerGuard>) {}
-}

pageserver/src/reltag.rs

@@ -39,7 +39,9 @@ impl PartialOrd for RelTag {
 
 impl Ord for RelTag {
     fn cmp(&self, other: &Self) -> Ordering {
-        let mut cmp = self.spcnode.cmp(&other.spcnode);
+        let mut cmp;
+
+        cmp = self.spcnode.cmp(&other.spcnode);
         if cmp != Ordering::Equal {
             return cmp;
         }

pageserver/src/remote_storage.rs

@@ -5,10 +5,11 @@
 //! There are a few components the storage machinery consists of:
 //! * [`RemoteStorage`] trait a CRUD-like generic abstraction to use for adapting external storages with a few implementations:
 //!     * [`local_fs`] allows to use local file system as an external storage
-//!     * [`s3_bucket`] uses AWS S3 bucket as an external storage
+//!     * [`rust_s3`] uses AWS S3 bucket as an external storage
 //!
 //! * synchronization logic at [`storage_sync`] module that keeps pageserver state (both runtime one and the workdir files) and storage state in sync.
 //! Synchronization internals are split into submodules
+//!     * [`storage_sync::compression`] for a custom remote storage format used to store timeline files in archives
 //!     * [`storage_sync::index`] to keep track of remote tenant files, the metadata and their mappings to local files
 //!     * [`storage_sync::upload`] and [`storage_sync::download`] to manage archive creation and upload; download and extraction, respectively
 //!
@@ -53,32 +54,25 @@
 //! The checkpoint uploads are disabled, if no remote storage configuration is provided (no sync loop is started this way either).
 //! See [`crate::layered_repository`] for the upload calls and the adjacent logic.
 //!
-//! Synchronization logic is able to communicate back with updated timeline sync states, [`crate::repository::TimelineSyncStatusUpdate`],
-//! submitted via [`crate::tenant_mgr::apply_timeline_sync_status_updates`] function. Tenant manager applies corresponding timeline updates in pageserver's in-memory state.
+//! Synchronization logic is able to communicate back with updated timeline sync states, [`TimelineSyncState`],
+//! submitted via [`crate::tenant_mgr::set_timeline_states`] function. Tenant manager applies corresponding timeline updates in pageserver's in-memory state.
 //! Such submissions happen in two cases:
 //! * once after the sync loop startup, to signal pageserver which timelines will be synchronized in the near future
 //! * after every loop step, in case a timeline needs to be reloaded or evicted from pageserver's memory
 //!
-//! When the pageserver terminates, the sync loop finishes a current sync task (if any) and exits.
+//! When the pageserver terminates, the upload loop finishes a current sync task (if any) and exits.
 //!
-//! The storage logic considers `image` as a set of local files (layers), fully representing a certain timeline at given moment (identified with `disk_consistent_lsn` from the corresponding `metadata` file).
+//! The storage logic considers `image` as a set of local files, fully representing a certain timeline at given moment (identified with `disk_consistent_lsn`).
 //! Timeline can change its state, by adding more files on disk and advancing its `disk_consistent_lsn`: this happens after pageserver checkpointing and is followed
 //! by the storage upload, if enabled.
-//! Yet timeline cannot alter already existing files, and cannot remove those too: only a GC process is capable of removing unused files.
+//! Yet timeline cannot alter already existing files, and normally cannot remote those too: only a GC process is capable of removing unused files.
 //! This way, remote storage synchronization relies on the fact that every checkpoint is incremental and local files are "immutable":
 //! * when a certain checkpoint gets uploaded, the sync loop remembers the fact, preventing further reuploads of the same state
 //! * no files are deleted from either local or remote storage, only the missing ones locally/remotely get downloaded/uploaded, local metadata file will be overwritten
 //! when the newer image is downloaded
 //!
-//! Pageserver maintains similar to the local file structure remotely: all layer files are uploaded with the same names under the same directory structure.
-//! Yet instead of keeping the `metadata` file remotely, we wrap it with more data in [`IndexShard`], containing the list of remote files.
-//! This file gets read to populate the cache, if the remote timeline data is missing from it and gets updated after every successful download.
-//! This way, we optimize S3 storage access by not running the `S3 list` command that could be expencive and slow: knowing both [`ZTenantId`] and [`ZTimelineId`],
-//! we can always reconstruct the path to the timeline, use this to get the same path on the remote storage and retrive its shard contents, if needed, same as any layer files.
-//!
-//! By default, pageserver reads the remote storage index data only for timelines located locally, to synchronize those, if needed.
-//! Bulk index data download happens only initially, on pageserer startup. The rest of the remote storage stays unknown to pageserver and loaded on demand only,
-//! when a new timeline is scheduled for the download.
+//! To optimize S3 storage (and access), the sync loop compresses the checkpoint files before placing them to S3, and uncompresses them back, keeping track of timeline files and metadata.
+//! Also, the remote file list is queried once only, at startup, to avoid possible extra costs and latency issues.
 //!
 //! NOTES:
 //! * pageserver assumes it has exclusive write access to the remote storage. If supported, the way multiple pageservers can be separated in the same storage
@@ -88,11 +82,11 @@
 //! The sync queue processing also happens in batches, so the sync tasks can wait in the queue for some time.
 
 mod local_fs;
-mod s3_bucket;
+mod rust_s3;
 mod storage_sync;
 
 use std::{
-    collections::{HashMap, HashSet},
+    collections::HashMap,
     ffi, fs,
     path::{Path, PathBuf},
 };
@@ -100,36 +94,22 @@ use std::{
 use anyhow::{bail, Context};
 use tokio::io;
 use tracing::{debug, error, info};
+use zenith_utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId};
 
-pub use self::{
-    local_fs::LocalFs,
-    s3_bucket::S3Bucket,
-    storage_sync::{
-        download_index_part,
-        index::{IndexPart, RemoteIndex, RemoteTimeline},
-        schedule_timeline_checkpoint_upload, schedule_timeline_download,
-    },
-};
+pub use self::storage_sync::index::{RemoteIndex, TimelineIndexEntry};
+pub use self::storage_sync::{schedule_timeline_checkpoint_upload, schedule_timeline_download};
+use self::{local_fs::LocalFs, rust_s3::S3};
+use crate::layered_repository::ephemeral_file::is_ephemeral_file;
 use crate::{
     config::{PageServerConf, RemoteStorageKind},
-    layered_repository::{
-        ephemeral_file::is_ephemeral_file,
-        metadata::{TimelineMetadata, METADATA_FILE_NAME},
-    },
+    layered_repository::metadata::{TimelineMetadata, METADATA_FILE_NAME},
 };
-use utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId};
 
-/// A timeline status to share with pageserver's sync counterpart,
-/// after comparing local and remote timeline state.
+pub use storage_sync::compression;
+
 #[derive(Clone, Copy, Debug)]
 pub enum LocalTimelineInitStatus {
-    /// The timeline has every remote layer present locally.
-    /// There could be some layers requiring uploading,
-    /// but this does not block the timeline from any user interaction.
     LocallyComplete,
-    /// A timeline has some files remotely, that are not present locally and need downloading.
-    /// Downloading might update timeline's metadata locally and current pageserver logic deals with local layers only,
-    /// so the data needs to be downloaded first before the timeline can be used.
     NeedsSync,
 }
 
@@ -161,7 +141,7 @@ pub fn start_local_timeline_sync(
                     config,
                     local_timeline_files,
                     LocalFs::new(root.clone(), &config.workdir)?,
-                    storage_config.max_concurrent_timelines_sync,
+                    storage_config.max_concurrent_sync,
                     storage_config.max_sync_errors,
                 )
             },
@@ -171,8 +151,8 @@ pub fn start_local_timeline_sync(
                 storage_sync::spawn_storage_sync_thread(
                     config,
                     local_timeline_files,
-                    S3Bucket::new(s3_config, &config.workdir)?,
-                    storage_config.max_concurrent_timelines_sync,
+                    S3::new(s3_config, &config.workdir)?,
+                    storage_config.max_concurrent_sync,
                     storage_config.max_sync_errors,
                 )
             },
@@ -199,7 +179,7 @@ pub fn start_local_timeline_sync(
 
 fn local_tenant_timeline_files(
     config: &'static PageServerConf,
-) -> anyhow::Result<HashMap<ZTenantTimelineId, (TimelineMetadata, HashSet<PathBuf>)>> {
+) -> anyhow::Result<HashMap<ZTenantTimelineId, (TimelineMetadata, Vec<PathBuf>)>> {
     let mut local_tenant_timeline_files = HashMap::new();
     let tenants_dir = config.tenants_path();
     for tenants_dir_entry in fs::read_dir(&tenants_dir)
@@ -234,8 +214,9 @@ fn local_tenant_timeline_files(
 fn collect_timelines_for_tenant(
     config: &'static PageServerConf,
     tenant_path: &Path,
-) -> anyhow::Result<HashMap<ZTenantTimelineId, (TimelineMetadata, HashSet<PathBuf>)>> {
-    let mut timelines = HashMap::new();
+) -> anyhow::Result<HashMap<ZTenantTimelineId, (TimelineMetadata, Vec<PathBuf>)>> {
+    let mut timelines: HashMap<ZTenantTimelineId, (TimelineMetadata, Vec<PathBuf>)> =
+        HashMap::new();
     let tenant_id = tenant_path
         .file_name()
         .and_then(ffi::OsStr::to_str)
@@ -284,8 +265,8 @@ fn collect_timelines_for_tenant(
 //  NOTE: ephemeral files are excluded from the list
 fn collect_timeline_files(
     timeline_dir: &Path,
-) -> anyhow::Result<(ZTimelineId, TimelineMetadata, HashSet<PathBuf>)> {
-    let mut timeline_files = HashSet::new();
+) -> anyhow::Result<(ZTimelineId, TimelineMetadata, Vec<PathBuf>)> {
+    let mut timeline_files = Vec::new();
     let mut timeline_metadata_path = None;
 
     let timeline_id = timeline_dir
@@ -305,7 +286,7 @@ fn collect_timeline_files(
                 debug!("skipping ephemeral file {}", entry_path.display());
                 continue;
             } else {
-                timeline_files.insert(entry_path);
+                timeline_files.push(entry_path);
             }
         }
     }
@@ -326,7 +307,7 @@ fn collect_timeline_files(
 /// This storage tries to be unaware of any layered repository context,
 /// providing basic CRUD operations for storage files.
 #[async_trait::async_trait]
-pub trait RemoteStorage: Send + Sync {
+trait RemoteStorage: Send + Sync {
     /// A way to uniquely reference a file in the remote storage.
     type StoragePath;
 
@@ -343,39 +324,28 @@ pub trait RemoteStorage: Send + Sync {
     async fn upload(
         &self,
         from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
-        // S3 PUT request requires the content length to be specified,
-        // otherwise it starts to fail with the concurrent connection count increasing.
-        from_size_bytes: usize,
         to: &Self::StoragePath,
-        metadata: Option<StorageMetadata>,
     ) -> anyhow::Result<()>;
 
     /// Streams the remote storage entry contents into the buffered writer given, returns the filled writer.
-    /// Returns the metadata, if any was stored with the file previously.
     async fn download(
         &self,
         from: &Self::StoragePath,
         to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>>;
+    ) -> anyhow::Result<()>;
 
     /// Streams a given byte range of the remote storage entry contents into the buffered writer given, returns the filled writer.
-    /// Returns the metadata, if any was stored with the file previously.
     async fn download_range(
         &self,
         from: &Self::StoragePath,
         start_inclusive: u64,
         end_exclusive: Option<u64>,
         to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>>;
+    ) -> anyhow::Result<()>;
 
     async fn delete(&self, path: &Self::StoragePath) -> anyhow::Result<()>;
 }
 
-/// Extra set of key-value pairs that contain arbitrary metadata about the storage entry.
-/// Immutable, cannot be changed once the file is created.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct StorageMetadata(HashMap<String, String>);
-
 fn strip_path_prefix<'a>(prefix: &'a Path, path: &'a Path) -> anyhow::Result<&'a Path> {
     if prefix == path {
         anyhow::bail!(

pageserver/src/remote_storage/README.md

@@ -0,0 +1,64 @@
+# Non-implementation details
+
+This document describes the current state of the backup system in pageserver, existing limitations and concerns, why some things are done the way they are the future development plans.
+Detailed description on how the synchronization works and how it fits into the rest of the pageserver can be found in the [storage module](./../remote_storage.rs) and its submodules.
+Ideally, this document should disappear after current implementation concerns are mitigated, with the remaining useful knowledge bits moved into rustdocs.
+
+## Approach
+
+Backup functionality is a new component, appeared way after the core DB functionality was implemented.
+Pageserver layer functionality is also quite volatile at the moment, there's a risk its local file management changes over time.
+
+To avoid adding more chaos into that, backup functionality is currently designed as a relatively standalone component, with the majority of its logic placed in a standalone async loop.
+This way, the backups are managed in background, not affecting directly other pageserver parts: this way the backup and restoration process may lag behind, but eventually keep up with the reality. To track that, a set of prometheus metrics is exposed from pageserver.
+
+## What's done
+
+Current implementation
+* provides remote storage wrappers for AWS S3 and local FS
+* synchronizes the differences with local timelines and remote states as fast as possible
+* uploads new layer files
+* downloads and registers timelines, found on the remote storage, but missing locally, if those are requested somehow via pageserver (e.g. http api, gc)
+* uses compression when deals with files, for better S3 usage
+* maintains an index of what's stored remotely
+* evicts failing tasks and stops the corresponding timelines
+
+The tasks are delayed with every retry and the retries are capped, to avoid poisonous tasks.
+After any task eviction, or any error at startup checks (e.g. obviously different and wrong local and remote states fot the same timeline),
+the timeline has to be stopped from submitting further checkpoint upload tasks, which is done along the corresponding timeline status change.
+
+No good optimisations or performance testing is done, the feature is disabled by default and gets polished over time.
+It's planned to deal with all questions that are currently on and prepare the feature to be enabled by default in cloud environments.
+
+### Peculiarities
+
+As mentioned, the backup component is rather new and under development currently, so not all things are done properly from the start.
+Here's the list of known compromises with comments:
+
+* Remote storage file model is currently a custom archive format, that's not possible to deserialize without a particular Rust code of ours (including `serde`).
+We also don't optimize the archivation and pack every timeline checkpoint separately, so the resulting blob's size that gets on S3 could be arbitrary.
+But, it's a single blob, which is way better than storing ~780 small files separately.
+
+* Archive index restoration requires reading every blob's head.
+This could be avoided by a background thread/future storing the serialized index in the remote storage.
+
+* no proper file comparison
+
+No file checksum assertion is done currently, but should be (AWS S3 returns file checksums during the `list` operation)
+
+* sad rust-s3 api
+
+rust-s3 is not very pleasant to use:
+1. it returns `anyhow::Result` and it's hard to distinguish "missing file" cases from "no connection" one, for instance
+2. at least one function it its API that we need (`get_object_stream`) has `async` keyword and blocks (!), see details [here](https://github.com/zenithdb/zenith/pull/752#discussion_r728373091)
+3. it's a prerelease library with unclear maintenance status
+4. noisy on debug level
+
+But it's already used in the project, so for now it's reused to avoid bloating the dependency tree.
+Based on previous evaluation, even `rusoto-s3` could be a better choice over this library, but needs further benchmarking.
+
+
+* gc is ignored
+
+So far, we don't adjust the remote storage based on GC thread loop results, only checkpointer loop affects the remote storage.
+Index module could be used as a base to implement a deferred GC mechanism, a "defragmentation" that repacks archives into new ones after GC is done removing the files from the archives.

pageserver/src/remote_storage/local_fs.rs

@@ -5,6 +5,7 @@
 //! volume is mounted to the local FS.
 
 use std::{
+    ffi::OsString,
     future::Future,
     path::{Path, PathBuf},
     pin::Pin,
@@ -17,7 +18,7 @@ use tokio::{
 };
 use tracing::*;
 
-use super::{strip_path_prefix, RemoteStorage, StorageMetadata};
+use super::{strip_path_prefix, RemoteStorage};
 
 pub struct LocalFs {
     pageserver_workdir: &'static Path,
@@ -53,32 +54,6 @@ impl LocalFs {
             )
         }
     }
-
-    async fn read_storage_metadata(
-        &self,
-        file_path: &Path,
-    ) -> anyhow::Result<Option<StorageMetadata>> {
-        let metadata_path = storage_metadata_path(file_path);
-        if metadata_path.exists() && metadata_path.is_file() {
-            let metadata_string = fs::read_to_string(&metadata_path).await.with_context(|| {
-                format!(
-                    "Failed to read metadata from the local storage at '{}'",
-                    metadata_path.display()
-                )
-            })?;
-
-            serde_json::from_str(&metadata_string)
-                .with_context(|| {
-                    format!(
-                        "Failed to deserialize metadata from the local storage at '{}'",
-                        metadata_path.display()
-                    )
-                })
-                .map(|metadata| Some(StorageMetadata(metadata)))
-        } else {
-            Ok(None)
-        }
-    }
 }
 
 #[async_trait::async_trait]
@@ -104,17 +79,21 @@ impl RemoteStorage for LocalFs {
 
     async fn upload(
         &self,
-        from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
-        from_size_bytes: usize,
+        mut from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
         to: &Self::StoragePath,
-        metadata: Option<StorageMetadata>,
     ) -> anyhow::Result<()> {
         let target_file_path = self.resolve_in_storage(to)?;
         create_target_directory(&target_file_path).await?;
         // We need this dance with sort of durable rename (without fsyncs)
         // to prevent partial uploads. This was really hit when pageserver shutdown
         // cancelled the upload and partial file was left on the fs
-        let temp_file_path = path_with_suffix_extension(&target_file_path, ".temp");
+        let mut temp_extension = target_file_path
+            .extension()
+            .unwrap_or_default()
+            .to_os_string();
+
+        temp_extension.push(OsString::from(".temp"));
+        let temp_file_path = target_file_path.with_extension(temp_extension);
         let mut destination = io::BufWriter::new(
             fs::OpenOptions::new()
                 .write(true)
@@ -129,11 +108,7 @@ impl RemoteStorage for LocalFs {
                 })?,
         );
 
-        let from_size_bytes = from_size_bytes as u64;
-        // Require to read 1 byte more than the expected to check later, that the stream and its size match.
-        let mut buffer_to_read = from.take(from_size_bytes + 1);
-
-        let bytes_read = io::copy(&mut buffer_to_read, &mut destination)
+        io::copy(&mut from, &mut destination)
             .await
             .with_context(|| {
                 format!(
@@ -142,19 +117,6 @@ impl RemoteStorage for LocalFs {
                 )
             })?;
 
-        ensure!(
-            bytes_read == from_size_bytes,
-            "Provided stream has actual size {} fthat is smaller than the given stream size {}",
-            bytes_read,
-            from_size_bytes
-        );
-
-        ensure!(
-            buffer_to_read.read(&mut [0]).await? == 0,
-            "Provided stream has bigger size than the given stream size {}",
-            from_size_bytes
-        );
-
         destination.flush().await.with_context(|| {
             format!(
                 "Failed to upload (flush temp) file to the local storage at '{}'",
@@ -170,23 +132,6 @@ impl RemoteStorage for LocalFs {
                     target_file_path.display()
                 )
             })?;
-
-        if let Some(storage_metadata) = metadata {
-            let storage_metadata_path = storage_metadata_path(&target_file_path);
-            fs::write(
-                &storage_metadata_path,
-                serde_json::to_string(&storage_metadata.0)
-                    .context("Failed to serialize storage metadata as json")?,
-            )
-            .await
-            .with_context(|| {
-                format!(
-                    "Failed to write metadata to the local storage at '{}'",
-                    storage_metadata_path.display()
-                )
-            })?;
-        }
-
         Ok(())
     }
 
@@ -194,7 +139,7 @@ impl RemoteStorage for LocalFs {
         &self,
         from: &Self::StoragePath,
         to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>> {
+    ) -> anyhow::Result<()> {
         let file_path = self.resolve_in_storage(from)?;
 
         if file_path.exists() && file_path.is_file() {
@@ -217,8 +162,7 @@ impl RemoteStorage for LocalFs {
                 )
             })?;
             source.flush().await?;
-
-            self.read_storage_metadata(&file_path).await
+            Ok(())
         } else {
             bail!(
                 "File '{}' either does not exist or is not a file",
@@ -233,7 +177,7 @@ impl RemoteStorage for LocalFs {
         start_inclusive: u64,
         end_exclusive: Option<u64>,
         to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>> {
+    ) -> anyhow::Result<()> {
         if let Some(end_exclusive) = end_exclusive {
             ensure!(
                 end_exclusive > start_inclusive,
@@ -242,7 +186,7 @@ impl RemoteStorage for LocalFs {
                 end_exclusive
             );
             if start_inclusive == end_exclusive.saturating_sub(1) {
-                return Ok(None);
+                return Ok(());
             }
         }
         let file_path = self.resolve_in_storage(from)?;
@@ -276,8 +220,7 @@ impl RemoteStorage for LocalFs {
                     file_path.display()
                 )
             })?;
-
-            self.read_storage_metadata(&file_path).await
+            Ok(())
         } else {
             bail!(
                 "File '{}' either does not exist or is not a file",
@@ -299,17 +242,6 @@ impl RemoteStorage for LocalFs {
     }
 }
 
-fn path_with_suffix_extension(original_path: &Path, suffix: &str) -> PathBuf {
-    let mut extension_with_suffix = original_path.extension().unwrap_or_default().to_os_string();
-    extension_with_suffix.push(suffix);
-
-    original_path.with_extension(extension_with_suffix)
-}
-
-fn storage_metadata_path(original_path: &Path) -> PathBuf {
-    path_with_suffix_extension(original_path, ".metadata")
-}
-
 fn get_all_files<'a, P>(
     directory_path: P,
 ) -> Pin<Box<dyn Future<Output = anyhow::Result<Vec<PathBuf>>> + Send + Sync + 'a>>
@@ -519,7 +451,7 @@ mod fs_tests {
     use super::*;
     use crate::repository::repo_harness::{RepoHarness, TIMELINE_ID};
 
-    use std::{collections::HashMap, io::Write};
+    use std::io::Write;
     use tempfile::tempdir;
 
     #[tokio::test]
@@ -527,13 +459,13 @@ mod fs_tests {
         let repo_harness = RepoHarness::create("upload_file")?;
         let storage = create_storage()?;
 
-        let (file, size) = create_file_for_upload(
+        let source = create_file_for_upload(
             &storage.pageserver_workdir.join("whatever"),
             "whatever_contents",
         )
         .await?;
         let target_path = PathBuf::from("/").join("somewhere").join("else");
-        match storage.upload(file, size, &target_path, None).await {
+        match storage.upload(source, &target_path).await {
             Ok(()) => panic!("Should not allow storing files with wrong target path"),
             Err(e) => {
                 let message = format!("{:?}", e);
@@ -543,14 +475,14 @@ mod fs_tests {
         }
         assert!(storage.list().await?.is_empty());
 
-        let target_path_1 = upload_dummy_file(&repo_harness, &storage, "upload_1", None).await?;
+        let target_path_1 = upload_dummy_file(&repo_harness, &storage, "upload_1").await?;
         assert_eq!(
             storage.list().await?,
             vec![target_path_1.clone()],
             "Should list a single file after first upload"
         );
 
-        let target_path_2 = upload_dummy_file(&repo_harness, &storage, "upload_2", None).await?;
+        let target_path_2 = upload_dummy_file(&repo_harness, &storage, "upload_2").await?;
         assert_eq!(
             list_files_sorted(&storage).await?,
             vec![target_path_1.clone(), target_path_2.clone()],
@@ -571,16 +503,12 @@ mod fs_tests {
         let repo_harness = RepoHarness::create("download_file")?;
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;
 
         let mut content_bytes = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let metadata = storage.download(&upload_target, &mut content_bytes).await?;
-        assert!(
-            metadata.is_none(),
-            "No metadata should be returned for no metadata upload"
-        );
-
+        storage.download(&upload_target, &mut content_bytes).await?;
         content_bytes.flush().await?;
+
         let contents = String::from_utf8(content_bytes.into_inner().into_inner())?;
         assert_eq!(
             dummy_contents(upload_name),
@@ -605,16 +533,12 @@ mod fs_tests {
         let repo_harness = RepoHarness::create("download_file_range_positive")?;
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;
 
         let mut full_range_bytes = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let metadata = storage
+        storage
             .download_range(&upload_target, 0, None, &mut full_range_bytes)
             .await?;
-        assert!(
-            metadata.is_none(),
-            "No metadata should be returned for no metadata upload"
-        );
         full_range_bytes.flush().await?;
         assert_eq!(
             dummy_contents(upload_name),
@@ -624,7 +548,7 @@ mod fs_tests {
 
         let mut zero_range_bytes = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
         let same_byte = 1_000_000_000;
-        let metadata = storage
+        storage
             .download_range(
                 &upload_target,
                 same_byte,
@@ -632,10 +556,6 @@ mod fs_tests {
                 &mut zero_range_bytes,
             )
             .await?;
-        assert!(
-            metadata.is_none(),
-            "No metadata should be returned for no metadata upload"
-        );
         zero_range_bytes.flush().await?;
         assert!(
             zero_range_bytes.into_inner().into_inner().is_empty(),
@@ -646,7 +566,7 @@ mod fs_tests {
         let (first_part_local, second_part_local) = uploaded_bytes.split_at(3);
 
         let mut first_part_remote = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let metadata = storage
+        storage
             .download_range(
                 &upload_target,
                 0,
@@ -654,11 +574,6 @@ mod fs_tests {
                 &mut first_part_remote,
             )
             .await?;
-        assert!(
-            metadata.is_none(),
-            "No metadata should be returned for no metadata upload"
-        );
-
         first_part_remote.flush().await?;
         let first_part_remote = first_part_remote.into_inner().into_inner();
         assert_eq!(
@@ -668,7 +583,7 @@ mod fs_tests {
         );
 
         let mut second_part_remote = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let metadata = storage
+        storage
             .download_range(
                 &upload_target,
                 first_part_local.len() as u64,
@@ -676,11 +591,6 @@ mod fs_tests {
                 &mut second_part_remote,
             )
             .await?;
-        assert!(
-            metadata.is_none(),
-            "No metadata should be returned for no metadata upload"
-        );
-
         second_part_remote.flush().await?;
         let second_part_remote = second_part_remote.into_inner().into_inner();
         assert_eq!(
@@ -697,7 +607,7 @@ mod fs_tests {
         let repo_harness = RepoHarness::create("download_file_range_negative")?;
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;
 
         let start = 10000;
         let end = 234;
@@ -735,7 +645,7 @@ mod fs_tests {
         let repo_harness = RepoHarness::create("delete_file")?;
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;
 
         storage.delete(&upload_target).await?;
         assert!(storage.list().await?.is_empty());
@@ -751,84 +661,31 @@ mod fs_tests {
         Ok(())
     }
 
-    #[tokio::test]
-    async fn file_with_metadata() -> anyhow::Result<()> {
-        let repo_harness = RepoHarness::create("download_file")?;
-        let storage = create_storage()?;
-        let upload_name = "upload_1";
-        let metadata = StorageMetadata(HashMap::from([
-            ("one".to_string(), "1".to_string()),
-            ("two".to_string(), "2".to_string()),
-        ]));
-        let upload_target =
-            upload_dummy_file(&repo_harness, &storage, upload_name, Some(metadata.clone())).await?;
-
-        let mut content_bytes = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let full_download_metadata = storage.download(&upload_target, &mut content_bytes).await?;
-
-        content_bytes.flush().await?;
-        let contents = String::from_utf8(content_bytes.into_inner().into_inner())?;
-        assert_eq!(
-            dummy_contents(upload_name),
-            contents,
-            "We should upload and download the same contents"
-        );
-
-        assert_eq!(
-            full_download_metadata.as_ref(),
-            Some(&metadata),
-            "We should get the same metadata back for full download"
-        );
-
-        let uploaded_bytes = dummy_contents(upload_name).into_bytes();
-        let (first_part_local, _) = uploaded_bytes.split_at(3);
-
-        let mut first_part_remote = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let partial_download_metadata = storage
-            .download_range(
-                &upload_target,
-                0,
-                Some(first_part_local.len() as u64),
-                &mut first_part_remote,
-            )
-            .await?;
-        first_part_remote.flush().await?;
-        let first_part_remote = first_part_remote.into_inner().into_inner();
-        assert_eq!(
-            first_part_local,
-            first_part_remote.as_slice(),
-            "First part bytes should be returned when requested"
-        );
-
-        assert_eq!(
-            partial_download_metadata.as_ref(),
-            Some(&metadata),
-            "We should get the same metadata back for partial download"
-        );
-
-        Ok(())
-    }
-
     async fn upload_dummy_file(
         harness: &RepoHarness<'_>,
         storage: &LocalFs,
         name: &str,
-        metadata: Option<StorageMetadata>,
     ) -> anyhow::Result<PathBuf> {
         let timeline_path = harness.timeline_path(&TIMELINE_ID);
         let relative_timeline_path = timeline_path.strip_prefix(&harness.conf.workdir)?;
         let storage_path = storage.root.join(relative_timeline_path).join(name);
-
-        let from_path = storage.pageserver_workdir.join(name);
-        let (file, size) = create_file_for_upload(&from_path, &dummy_contents(name)).await?;
-        storage.upload(file, size, &storage_path, metadata).await?;
+        storage
+            .upload(
+                create_file_for_upload(
+                    &storage.pageserver_workdir.join(name),
+                    &dummy_contents(name),
+                )
+                .await?,
+                &storage_path,
+            )
+            .await?;
         Ok(storage_path)
     }
 
     async fn create_file_for_upload(
         path: &Path,
         contents: &str,
-    ) -> anyhow::Result<(io::BufReader<fs::File>, usize)> {
+    ) -> anyhow::Result<io::BufReader<fs::File>> {
         std::fs::create_dir_all(path.parent().unwrap())?;
         let mut file_for_writing = std::fs::OpenOptions::new()
             .write(true)
@@ -836,10 +693,8 @@ mod fs_tests {
             .open(path)?;
         write!(file_for_writing, "{}", contents)?;
         drop(file_for_writing);
-        let file_size = path.metadata()?.len() as usize;
-        Ok((
-            io::BufReader::new(fs::OpenOptions::new().read(true).open(&path).await?),
-            file_size,
+        Ok(io::BufReader::new(
+            fs::OpenOptions::new().read(true).open(&path).await?,
         ))
     }
 

pageserver/src/remote_storage/rust_s3.rs

@@ -1,4 +1,4 @@
-//! AWS S3 storage wrapper around `rusoto` library.
+//! AWS S3 storage wrapper around `rust_s3` library.
 //!
 //! Respects `prefix_in_bucket` property from [`S3Config`],
 //! allowing multiple pageservers to independently work with the same S3 bucket, if
@@ -7,16 +7,8 @@
 use std::path::{Path, PathBuf};
 
 use anyhow::Context;
-use rusoto_core::{
-    credential::{InstanceMetadataProvider, StaticProvider},
-    HttpClient, Region,
-};
-use rusoto_s3::{
-    DeleteObjectRequest, GetObjectRequest, ListObjectsV2Request, PutObjectRequest, S3Client,
-    StreamingBody, S3,
-};
-use tokio::{io, sync::Semaphore};
-use tokio_util::io::ReaderStream;
+use s3::{bucket::Bucket, creds::Credentials, region::Region};
+use tokio::io::{self, AsyncWriteExt};
 use tracing::debug;
 
 use crate::{
@@ -24,8 +16,6 @@ use crate::{
     remote_storage::{strip_path_prefix, RemoteStorage},
 };
 
-use super::StorageMetadata;
-
 const S3_FILE_SEPARATOR: char = '/';
 
 #[derive(Debug, Eq, PartialEq)]
@@ -60,50 +50,38 @@ impl S3ObjectKey {
 }
 
 /// AWS S3 storage.
-pub struct S3Bucket {
+pub struct S3 {
     pageserver_workdir: &'static Path,
-    client: S3Client,
-    bucket_name: String,
+    bucket: Bucket,
     prefix_in_bucket: Option<String>,
-    // Every request to S3 can be throttled or cancelled, if a certain number of requests per second is exceeded.
-    // Same goes to IAM, which is queried before every S3 request, if enabled. IAM has even lower RPS threshold.
-    // The helps to ensure we don't exceed the thresholds.
-    concurrency_limiter: Semaphore,
 }
 
-impl S3Bucket {
-    /// Creates the S3 storage, errors if incorrect AWS S3 configuration provided.
+impl S3 {
+    /// Creates the storage, errors if incorrect AWS S3 configuration provided.
     pub fn new(aws_config: &S3Config, pageserver_workdir: &'static Path) -> anyhow::Result<Self> {
         debug!(
-            "Creating s3 remote storage for S3 bucket {}",
+            "Creating s3 remote storage around bucket {}",
             aws_config.bucket_name
         );
         let region = match aws_config.endpoint.clone() {
-            Some(custom_endpoint) => Region::Custom {
-                name: aws_config.bucket_region.clone(),
-                endpoint: custom_endpoint,
+            Some(endpoint) => Region::Custom {
+                endpoint,
+                region: aws_config.bucket_region.clone(),
             },
             None => aws_config
                 .bucket_region
                 .parse::<Region>()
                 .context("Failed to parse the s3 region from config")?,
         };
-        let request_dispatcher = HttpClient::new().context("Failed to create S3 http client")?;
-        let client = if aws_config.access_key_id.is_none() && aws_config.secret_access_key.is_none()
-        {
-            debug!("Using IAM-based AWS access");
-            S3Client::new_with(request_dispatcher, InstanceMetadataProvider::new(), region)
-        } else {
-            debug!("Using credentials-based AWS access");
-            S3Client::new_with(
-                request_dispatcher,
-                StaticProvider::new_minimal(
-                    aws_config.access_key_id.clone().unwrap_or_default(),
-                    aws_config.secret_access_key.clone().unwrap_or_default(),
-                ),
-                region,
-            )
-        };
+
+        let credentials = Credentials::new(
+            aws_config.access_key_id.as_deref(),
+            aws_config.secret_access_key.as_deref(),
+            None,
+            None,
+            None,
+        )
+        .context("Failed to create the s3 credentials")?;
 
         let prefix_in_bucket = aws_config.prefix_in_bucket.as_deref().map(|prefix| {
             let mut prefix = prefix;
@@ -119,17 +97,20 @@ impl S3Bucket {
         });
 
         Ok(Self {
-            client,
+            bucket: Bucket::new_with_path_style(
+                aws_config.bucket_name.as_str(),
+                region,
+                credentials,
+            )
+            .context("Failed to create the s3 bucket")?,
             pageserver_workdir,
-            bucket_name: aws_config.bucket_name.clone(),
             prefix_in_bucket,
-            concurrency_limiter: Semaphore::new(aws_config.concurrency_limit.get()),
         })
     }
 }
 
 #[async_trait::async_trait]
-impl RemoteStorage for S3Bucket {
+impl RemoteStorage for S3 {
     type StoragePath = S3ObjectKey;
 
     fn storage_path(&self, local_path: &Path) -> anyhow::Result<Self::StoragePath> {
@@ -148,93 +129,74 @@ impl RemoteStorage for S3Bucket {
     }
 
     async fn list(&self) -> anyhow::Result<Vec<Self::StoragePath>> {
-        let mut document_keys = Vec::new();
+        let list_response = self
+            .bucket
+            .list(self.prefix_in_bucket.clone().unwrap_or_default(), None)
+            .await
+            .context("Failed to list s3 objects")?;
 
-        let mut continuation_token = None;
-        loop {
-            let _guard = self
-                .concurrency_limiter
-                .acquire()
-                .await
-                .context("Concurrency limiter semaphore got closed during S3 list")?;
-            let fetch_response = self
-                .client
-                .list_objects_v2(ListObjectsV2Request {
-                    bucket: self.bucket_name.clone(),
-                    prefix: self.prefix_in_bucket.clone(),
-                    continuation_token,
-                    ..ListObjectsV2Request::default()
-                })
-                .await?;
-            document_keys.extend(
-                fetch_response
-                    .contents
-                    .unwrap_or_default()
-                    .into_iter()
-                    .filter_map(|o| Some(S3ObjectKey(o.key?))),
-            );
-
-            match fetch_response.continuation_token {
-                Some(new_token) => continuation_token = Some(new_token),
-                None => break,
-            }
-        }
-
-        Ok(document_keys)
+        Ok(list_response
+            .into_iter()
+            .flat_map(|response| response.contents)
+            .map(|s3_object| S3ObjectKey(s3_object.key))
+            .collect())
     }
 
     async fn upload(
         &self,
-        from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
-        from_size_bytes: usize,
+        mut from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
         to: &Self::StoragePath,
-        metadata: Option<StorageMetadata>,
     ) -> anyhow::Result<()> {
-        let _guard = self
-            .concurrency_limiter
-            .acquire()
+        let mut upload_contents = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
+        io::copy(&mut from, &mut upload_contents)
             .await
-            .context("Concurrency limiter semaphore got closed during S3 upload")?;
-        self.client
-            .put_object(PutObjectRequest {
-                body: Some(StreamingBody::new_with_size(
-                    ReaderStream::new(from),
-                    from_size_bytes,
-                )),
-                bucket: self.bucket_name.clone(),
-                key: to.key().to_owned(),
-                metadata: metadata.map(|m| m.0),
-                ..PutObjectRequest::default()
-            })
-            .await?;
-        Ok(())
+            .context("Failed to read the upload contents")?;
+        upload_contents
+            .flush()
+            .await
+            .context("Failed to read the upload contents")?;
+        let upload_contents = upload_contents.into_inner().into_inner();
+
+        let (_, code) = self
+            .bucket
+            .put_object(to.key(), &upload_contents)
+            .await
+            .with_context(|| format!("Failed to create s3 object with key {}", to.key()))?;
+        if code != 200 {
+            Err(anyhow::format_err!(
+                "Received non-200 exit code during creating object with key '{}', code: {}",
+                to.key(),
+                code
+            ))
+        } else {
+            Ok(())
+        }
     }
 
     async fn download(
         &self,
         from: &Self::StoragePath,
         to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>> {
-        let _guard = self
-            .concurrency_limiter
-            .acquire()
+    ) -> anyhow::Result<()> {
+        let (data, code) = self
+            .bucket
+            .get_object(from.key())
             .await
-            .context("Concurrency limiter semaphore got closed during S3 download")?;
-        let object_output = self
-            .client
-            .get_object(GetObjectRequest {
-                bucket: self.bucket_name.clone(),
-                key: from.key().to_owned(),
-                ..GetObjectRequest::default()
-            })
-            .await?;
-
-        if let Some(body) = object_output.body {
-            let mut from = io::BufReader::new(body.into_async_read());
-            io::copy(&mut from, to).await?;
+            .with_context(|| format!("Failed to download s3 object with key {}", from.key()))?;
+        if code != 200 {
+            Err(anyhow::format_err!(
+                "Received non-200 exit code during downloading object, code: {}",
+                code
+            ))
+        } else {
+            // we don't have to write vector into the destination this way, `to_write_all` would be enough.
+            // but we want to prepare for migration on `rusoto`, that has a streaming HTTP body instead here, with
+            // which it makes more sense to use `io::copy`.
+            io::copy(&mut data.as_slice(), to)
+                .await
+                .context("Failed to write downloaded data into the destination buffer")?;
+            Ok(())
         }
-
-        Ok(object_output.metadata.map(StorageMetadata))
     }
 
     async fn download_range(
@@ -243,51 +205,44 @@ impl RemoteStorage for S3Bucket {
         start_inclusive: u64,
         end_exclusive: Option<u64>,
         to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>> {
+    ) -> anyhow::Result<()> {
         // S3 accepts ranges as https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35
         // and needs both ends to be exclusive
         let end_inclusive = end_exclusive.map(|end| end.saturating_sub(1));
-        let range = Some(match end_inclusive {
-            Some(end_inclusive) => format!("bytes={}-{}", start_inclusive, end_inclusive),
-            None => format!("bytes={}-", start_inclusive),
-        });
-        let _guard = self
-            .concurrency_limiter
-            .acquire()
+        let (data, code) = self
+            .bucket
+            .get_object_range(from.key(), start_inclusive, end_inclusive)
             .await
-            .context("Concurrency limiter semaphore got closed during S3 range download")?;
-        let object_output = self
-            .client
-            .get_object(GetObjectRequest {
-                bucket: self.bucket_name.clone(),
-                key: from.key().to_owned(),
-                range,
-                ..GetObjectRequest::default()
-            })
-            .await?;
-
-        if let Some(body) = object_output.body {
-            let mut from = io::BufReader::new(body.into_async_read());
-            io::copy(&mut from, to).await?;
+            .with_context(|| format!("Failed to download s3 object with key {}", from.key()))?;
+        if code != 206 {
+            Err(anyhow::format_err!(
+                "Received non-206 exit code during downloading object range, code: {}",
+                code
+            ))
+        } else {
+            // see `download` function above for the comment on why `Vec<u8>` buffer is copied this way
+            io::copy(&mut data.as_slice(), to)
+                .await
+                .context("Failed to write downloaded range into the destination buffer")?;
+            Ok(())
         }
-
-        Ok(object_output.metadata.map(StorageMetadata))
     }
 
     async fn delete(&self, path: &Self::StoragePath) -> anyhow::Result<()> {
-        let _guard = self
-            .concurrency_limiter
-            .acquire()
+        let (_, code) = self
+            .bucket
+            .delete_object(path.key())
             .await
-            .context("Concurrency limiter semaphore got closed during S3 delete")?;
-        self.client
-            .delete_object(DeleteObjectRequest {
-                bucket: self.bucket_name.clone(),
-                key: path.key().to_owned(),
-                ..DeleteObjectRequest::default()
-            })
-            .await?;
-        Ok(())
+            .with_context(|| format!("Failed to delete s3 object with key {}", path.key()))?;
+        if code != 204 {
+            Err(anyhow::format_err!(
+                "Received non-204 exit code during deleting object with key '{}', code: {}",
+                path.key(),
+                code
+            ))
+        } else {
+            Ok(())
+        }
     }
 }
 
@@ -359,7 +314,7 @@ mod tests {
     #[test]
     fn storage_path_negatives() -> anyhow::Result<()> {
         #[track_caller]
-        fn storage_path_error(storage: &S3Bucket, mismatching_path: &Path) -> String {
+        fn storage_path_error(storage: &S3, mismatching_path: &Path) -> String {
             match storage.storage_path(mismatching_path) {
                 Ok(wrong_key) => panic!(
                     "Expected path '{}' to error, but got S3 key: {:?}",
@@ -457,13 +412,16 @@ mod tests {
         Ok(())
     }
 
-    fn dummy_storage(pageserver_workdir: &'static Path) -> S3Bucket {
-        S3Bucket {
+    fn dummy_storage(pageserver_workdir: &'static Path) -> S3 {
+        S3 {
             pageserver_workdir,
-            client: S3Client::new("us-east-1".parse().unwrap()),
-            bucket_name: "dummy-bucket".to_string(),
+            bucket: Bucket::new(
+                "dummy-bucket",
+                "us-east-1".parse().unwrap(),
+                Credentials::anonymous().unwrap(),
+            )
+            .unwrap(),
             prefix_in_bucket: Some("dummy_prefix/".to_string()),
-            concurrency_limiter: Semaphore::new(1),
         }
     }
 

pageserver/src/remote_storage/storage_sync/compression.rs

@@ -0,0 +1,613 @@
+//! A set of structs to represent a compressed part of the timeline, and methods to asynchronously compress and uncompress a stream of data,
+//! without holding the entire data in memory.
+//! For the latter, both compress and uncompress functions operate buffered streams (currently hardcoded size of [`ARCHIVE_STREAM_BUFFER_SIZE_BYTES`]),
+//! not attempting to hold the entire archive in memory.
+//!
+//! The compression is done with <a href="https://datatracker.ietf.org/doc/html/rfc8878">zstd</a> streaming algorithm via the `async-compression` crate.
+//! The crate does not contain any knobs to tweak the compression, but otherwise is one of the only ones that's both async and has an API to manage the part of an archive.
+//! Zstd was picked as the best algorithm among the ones available in the crate, after testing the initial timeline file compression.
+//!
+//! Archiving is almost agnostic to timeline file types, with an exception of the metadata file, that's currently distinguished in the [un]compression code.
+//! The metadata file is treated separately when [de]compression is involved, to reduce the risk of corrupting the metadata file.
+//! When compressed, the metadata file is always required and stored as the last file in the archive stream.
+//! When uncompressed, the metadata file gets naturally uncompressed last, to ensure that all other layer files are decompressed successfully first.
+//!
+//! Archive structure:
+//! +----------------------------------------+
+//! | header | file_1, ..., file_k, metadata |
+//! +----------------------------------------+
+//!
+//! The archive consists of two separate zstd archives:
+//! * header archive, that contains all files names and their sizes and relative paths in the timeline directory
+//! Header is a Rust structure, serialized into bytes and compressed with zstd.
+//! * files archive, that has metadata file as the last one, all compressed with zstd into a single binary blob
+//!
+//! Header offset is stored in the file name, along with the `disk_consistent_lsn` from the metadata file.
+//! See [`parse_archive_name`] and [`ARCHIVE_EXTENSION`] for the name details, example: `00000000016B9150-.zst_9732`.
+//! This way, the header could be retrieved without reading an entire archive file.
+
+use std::{
+    collections::BTreeSet,
+    future::Future,
+    io::Cursor,
+    path::{Path, PathBuf},
+    sync::Arc,
+};
+
+use anyhow::{bail, ensure, Context};
+use async_compression::tokio::bufread::{ZstdDecoder, ZstdEncoder};
+use serde::{Deserialize, Serialize};
+use tokio::{
+    fs,
+    io::{self, AsyncReadExt, AsyncWriteExt},
+};
+use tracing::*;
+use zenith_utils::{bin_ser::BeSer, lsn::Lsn};
+
+use crate::layered_repository::metadata::{TimelineMetadata, METADATA_FILE_NAME};
+
+use super::index::RelativePath;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArchiveHeader {
+    /// All regular timeline files, excluding the metadata file.
+    pub files: Vec<FileEntry>,
+    // Metadata file name is known to the system, as its location relative to the timeline dir,
+    // so no need to store anything but its size in bytes.
+    pub metadata_file_size: u64,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
+pub struct FileEntry {
+    /// Uncompressed file size, bytes.
+    pub size: u64,
+    /// A path, relative to the directory root, used when compressing the directory contents.
+    pub subpath: RelativePath,
+}
+
+const ARCHIVE_EXTENSION: &str = "-.zst_";
+const ARCHIVE_STREAM_BUFFER_SIZE_BYTES: usize = 4 * 1024 * 1024;
+
+/// Streams an archive of files given into a stream target, defined by the closure.
+///
+/// The closure approach is picked for cases like S3, where we would need a name of the file before we can get a stream to write the bytes into.
+/// Current idea is to place the header size in the name of the file, to enable the fast partial remote file index restoration without actually reading remote storage file contents.
+///
+/// Performs the compression in multiple steps:
+/// * prepares an archive header, stripping the `source_dir` prefix from the `files`
+/// * generates the name of the archive
+/// * prepares archive producer future, knowing the header and the file list
+/// An `impl AsyncRead` and `impl AsyncWrite` pair of connected streams is created to implement the partial contents streaming.
+/// The writer end gets into the archive producer future, to put the header and a stream of compressed files.
+/// * prepares archive consumer future, by executing the provided closure
+/// The closure gets the reader end stream and the name of the file to create a future that would stream the file contents elsewhere.
+/// * runs and waits for both futures to complete
+/// * on a successful completion of both futures, header, its size and the user-defined consumer future return data is returned
+/// Due to the design above, the archive name and related data is visible inside the consumer future only, so it's possible to return the data,
+/// needed for future processing.
+pub async fn archive_files_as_stream<Cons, ConsRet, Fut>(
+    source_dir: &Path,
+    files: impl Iterator<Item = &PathBuf>,
+    metadata: &TimelineMetadata,
+    create_archive_consumer: Cons,
+) -> anyhow::Result<(ArchiveHeader, u64, ConsRet)>
+where
+    Cons: FnOnce(Box<dyn io::AsyncRead + Unpin + Send + Sync + 'static>, String) -> Fut
+        + Send
+        + 'static,
+    Fut: Future<Output = anyhow::Result<ConsRet>> + Send + 'static,
+    ConsRet: Send + Sync + 'static,
+{
+    let metadata_bytes = metadata
+        .to_bytes()
+        .context("Failed to create metadata bytes")?;
+    let (archive_header, compressed_header_bytes) =
+        prepare_header(source_dir, files, &metadata_bytes)
+            .await
+            .context("Failed to prepare file for archivation")?;
+
+    let header_size = compressed_header_bytes.len() as u64;
+    let (write, read) = io::duplex(ARCHIVE_STREAM_BUFFER_SIZE_BYTES);
+    let archive_filler = write_archive_contents(
+        source_dir.to_path_buf(),
+        archive_header.clone(),
+        metadata_bytes,
+        write,
+    );
+    let archive_name = archive_name(metadata.disk_consistent_lsn(), header_size);
+    let archive_stream =
+        Cursor::new(compressed_header_bytes).chain(ZstdEncoder::new(io::BufReader::new(read)));
+
+    let (archive_creation_result, archive_upload_result) = tokio::join!(
+        tokio::spawn(archive_filler),
+        tokio::spawn(async move {
+            create_archive_consumer(Box::new(archive_stream), archive_name).await
+        })
+    );
+    archive_creation_result
+        .context("Failed to spawn archive creation future")?
+        .context("Failed to create an archive")?;
+    let upload_return_value = archive_upload_result
+        .context("Failed to spawn archive upload future")?
+        .context("Failed to upload the archive")?;
+
+    Ok((archive_header, header_size, upload_return_value))
+}
+
+/// Similar to [`archive_files_as_stream`], creates a pair of streams to uncompress the 2nd part of the archive,
+/// that contains files and is located after the header.
+/// S3 allows downloading partial file contents for a given file key (i.e. name), to accommodate this retrieval,
+/// a closure is used.
+/// Same concepts with two concurrent futures, user-defined closure, future and return value apply here, but the
+/// consumer and the receiver ends are swapped, since the uncompression happens.
+pub async fn uncompress_file_stream_with_index<Prod, ProdRet, Fut>(
+    destination_dir: PathBuf,
+    files_to_skip: Arc<BTreeSet<PathBuf>>,
+    disk_consistent_lsn: Lsn,
+    header: ArchiveHeader,
+    header_size: u64,
+    create_archive_file_part: Prod,
+) -> anyhow::Result<ProdRet>
+where
+    Prod: FnOnce(Box<dyn io::AsyncWrite + Unpin + Send + Sync + 'static>, String) -> Fut
+        + Send
+        + 'static,
+    Fut: Future<Output = anyhow::Result<ProdRet>> + Send + 'static,
+    ProdRet: Send + Sync + 'static,
+{
+    let (write, mut read) = io::duplex(ARCHIVE_STREAM_BUFFER_SIZE_BYTES);
+    let archive_name = archive_name(disk_consistent_lsn, header_size);
+
+    let (archive_download_result, archive_uncompress_result) = tokio::join!(
+        tokio::spawn(async move { create_archive_file_part(Box::new(write), archive_name).await }),
+        tokio::spawn(async move {
+            uncompress_with_header(&files_to_skip, &destination_dir, header, &mut read).await
+        })
+    );
+
+    let download_value = archive_download_result
+        .context("Failed to spawn archive download future")?
+        .context("Failed to download an archive")?;
+    archive_uncompress_result
+        .context("Failed to spawn archive uncompress future")?
+        .context("Failed to uncompress the archive")?;
+
+    Ok(download_value)
+}
+
+/// Reads archive header from the stream given:
+/// * parses the file name to get the header size
+/// * reads the exact amount of bytes
+/// * uncompresses and deserializes those
+pub async fn read_archive_header<A: io::AsyncRead + Send + Sync + Unpin>(
+    archive_name: &str,
+    from: &mut A,
+) -> anyhow::Result<ArchiveHeader> {
+    let (_, header_size) = parse_archive_name(Path::new(archive_name))?;
+
+    let mut compressed_header_bytes = vec![0; header_size as usize];
+    from.read_exact(&mut compressed_header_bytes)
+        .await
+        .with_context(|| {
+            format!(
+                "Failed to read header header from the archive {}",
+                archive_name
+            )
+        })?;
+
+    let mut header_bytes = Vec::new();
+    ZstdDecoder::new(io::BufReader::new(compressed_header_bytes.as_slice()))
+        .read_to_end(&mut header_bytes)
+        .await
+        .context("Failed to decompress a header from the archive")?;
+
+    Ok(ArchiveHeader::des(&header_bytes)
+        .context("Failed to deserialize a header from the archive")?)
+}
+
+/// Reads the archive metadata out of the archive name:
+/// * `disk_consistent_lsn` of the checkpoint that was archived
+/// * size of the archive header
+pub fn parse_archive_name(archive_path: &Path) -> anyhow::Result<(Lsn, u64)> {
+    let archive_name = archive_path
+        .file_name()
+        .with_context(|| format!("Archive '{}' has no file name", archive_path.display()))?
+        .to_string_lossy();
+    let (lsn_str, header_size_str) =
+        archive_name
+            .rsplit_once(ARCHIVE_EXTENSION)
+            .with_context(|| {
+                format!(
+                    "Archive '{}' has incorrect extension, expected to contain '{}'",
+                    archive_path.display(),
+                    ARCHIVE_EXTENSION
+                )
+            })?;
+    let disk_consistent_lsn = Lsn::from_hex(lsn_str).with_context(|| {
+        format!(
+            "Archive '{}' has an invalid disk consistent lsn in its extension",
+            archive_path.display(),
+        )
+    })?;
+    let header_size = header_size_str.parse::<u64>().with_context(|| {
+        format!(
+            "Archive '{}' has an invalid a header offset number in its extension",
+            archive_path.display(),
+        )
+    })?;
+    Ok((disk_consistent_lsn, header_size))
+}
+
+fn archive_name(disk_consistent_lsn: Lsn, header_size: u64) -> String {
+    let archive_name = format!(
+        "{:016X}{ARCHIVE_EXTENSION}{}",
+        u64::from(disk_consistent_lsn),
+        header_size,
+        ARCHIVE_EXTENSION = ARCHIVE_EXTENSION,
+    );
+    archive_name
+}
+
+pub async fn uncompress_with_header(
+    files_to_skip: &BTreeSet<PathBuf>,
+    destination_dir: &Path,
+    header: ArchiveHeader,
+    archive_after_header: impl io::AsyncRead + Send + Sync + Unpin,
+) -> anyhow::Result<()> {
+    debug!("Uncompressing archive into {}", destination_dir.display());
+    let mut archive = ZstdDecoder::new(io::BufReader::new(archive_after_header));
+
+    if !destination_dir.exists() {
+        fs::create_dir_all(&destination_dir)
+            .await
+            .with_context(|| {
+                format!(
+                    "Failed to create target directory at {}",
+                    destination_dir.display()
+                )
+            })?;
+    } else if !destination_dir.is_dir() {
+        bail!(
+            "Destination path '{}' is not a valid directory",
+            destination_dir.display()
+        );
+    }
+    debug!("Will extract {} files from the archive", header.files.len());
+    for entry in header.files {
+        uncompress_entry(
+            &mut archive,
+            &entry.subpath.as_path(destination_dir),
+            entry.size,
+            files_to_skip,
+        )
+        .await
+        .with_context(|| format!("Failed to uncompress archive entry {:?}", entry))?;
+    }
+    uncompress_entry(
+        &mut archive,
+        &destination_dir.join(METADATA_FILE_NAME),
+        header.metadata_file_size,
+        files_to_skip,
+    )
+    .await
+    .context("Failed to uncompress the metadata entry")?;
+    Ok(())
+}
+
+async fn uncompress_entry(
+    archive: &mut ZstdDecoder<io::BufReader<impl io::AsyncRead + Send + Sync + Unpin>>,
+    destination_path: &Path,
+    entry_size: u64,
+    files_to_skip: &BTreeSet<PathBuf>,
+) -> anyhow::Result<()> {
+    if let Some(parent) = destination_path.parent() {
+        fs::create_dir_all(parent).await.with_context(|| {
+            format!(
+                "Failed to create parent directory for {}",
+                destination_path.display()
+            )
+        })?;
+    };
+
+    if files_to_skip.contains(destination_path) {
+        debug!("Skipping {}", destination_path.display());
+        copy_n_bytes(entry_size, archive, &mut io::sink())
+            .await
+            .context("Failed to skip bytes in the archive")?;
+        return Ok(());
+    }
+
+    let mut destination =
+        io::BufWriter::new(fs::File::create(&destination_path).await.with_context(|| {
+            format!(
+                "Failed to open file {} for extraction",
+                destination_path.display()
+            )
+        })?);
+    copy_n_bytes(entry_size, archive, &mut destination)
+        .await
+        .with_context(|| {
+            format!(
+                "Failed to write extracted archive contents into file {}",
+                destination_path.display()
+            )
+        })?;
+    destination
+        .flush()
+        .await
+        .context("Failed to flush the streaming archive bytes")?;
+    Ok(())
+}
+
+async fn write_archive_contents(
+    source_dir: PathBuf,
+    header: ArchiveHeader,
+    metadata_bytes: Vec<u8>,
+    mut archive_input: io::DuplexStream,
+) -> anyhow::Result<()> {
+    debug!("Starting writing files into archive");
+    for file_entry in header.files {
+        let path = file_entry.subpath.as_path(&source_dir);
+        let mut source_file =
+            io::BufReader::new(fs::File::open(&path).await.with_context(|| {
+                format!(
+                    "Failed to open file for archiving to path {}",
+                    path.display()
+                )
+            })?);
+        let bytes_written = io::copy(&mut source_file, &mut archive_input)
+            .await
+            .with_context(|| {
+                format!(
+                    "Failed to open add a file into archive, file path {}",
+                    path.display()
+                )
+            })?;
+        ensure!(
+            file_entry.size == bytes_written,
+            "File {} was written to the archive incompletely",
+            path.display()
+        );
+        trace!(
+            "Added file '{}' ({} bytes) into the archive",
+            path.display(),
+            bytes_written
+        );
+    }
+    let metadata_bytes_written = io::copy(&mut metadata_bytes.as_slice(), &mut archive_input)
+        .await
+        .context("Failed to add metadata into the archive")?;
+    ensure!(
+        header.metadata_file_size == metadata_bytes_written,
+        "Metadata file was written to the archive incompletely",
+    );
+
+    archive_input
+        .shutdown()
+        .await
+        .context("Failed to finalize the archive")?;
+    debug!("Successfully streamed all files into the archive");
+    Ok(())
+}
+
+async fn prepare_header(
+    source_dir: &Path,
+    files: impl Iterator<Item = &PathBuf>,
+    metadata_bytes: &[u8],
+) -> anyhow::Result<(ArchiveHeader, Vec<u8>)> {
+    let mut archive_files = Vec::new();
+    for file_path in files {
+        let file_metadata = fs::metadata(file_path).await.with_context(|| {
+            format!(
+                "Failed to read metadata during archive indexing for {}",
+                file_path.display()
+            )
+        })?;
+        ensure!(
+            file_metadata.is_file(),
+            "Archive indexed path {} is not a file",
+            file_path.display()
+        );
+
+        if file_path.file_name().and_then(|name| name.to_str()) != Some(METADATA_FILE_NAME) {
+            let entry = FileEntry {
+                subpath: RelativePath::new(source_dir, file_path).with_context(|| {
+                    format!(
+                        "File '{}' does not belong to pageserver workspace",
+                        file_path.display()
+                    )
+                })?,
+                size: file_metadata.len(),
+            };
+            archive_files.push(entry);
+        }
+    }
+
+    let header = ArchiveHeader {
+        files: archive_files,
+        metadata_file_size: metadata_bytes.len() as u64,
+    };
+
+    debug!("Appending a header for {} files", header.files.len());
+    let header_bytes = header.ser().context("Failed to serialize a header")?;
+    debug!("Header bytes len {}", header_bytes.len());
+    let mut compressed_header_bytes = Vec::new();
+    ZstdEncoder::new(io::BufReader::new(header_bytes.as_slice()))
+        .read_to_end(&mut compressed_header_bytes)
+        .await
+        .context("Failed to compress header bytes")?;
+    debug!(
+        "Compressed header bytes len {}",
+        compressed_header_bytes.len()
+    );
+    Ok((header, compressed_header_bytes))
+}
+
+async fn copy_n_bytes(
+    n: u64,
+    from: &mut (impl io::AsyncRead + Send + Sync + Unpin),
+    into: &mut (impl io::AsyncWrite + Send + Sync + Unpin),
+) -> anyhow::Result<()> {
+    let bytes_written = io::copy(&mut from.take(n), into).await?;
+    ensure!(
+        bytes_written == n,
+        "Failed to read exactly {} bytes from the input, bytes written: {}",
+        n,
+        bytes_written,
+    );
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use tokio::{fs, io::AsyncSeekExt};
+
+    use crate::repository::repo_harness::{RepoHarness, TIMELINE_ID};
+
+    use super::*;
+
+    #[tokio::test]
+    async fn compress_and_uncompress() -> anyhow::Result<()> {
+        let repo_harness = RepoHarness::create("compress_and_uncompress")?;
+        let timeline_dir = repo_harness.timeline_path(&TIMELINE_ID);
+        init_directory(
+            &timeline_dir,
+            vec![
+                ("first", "first_contents"),
+                ("second", "second_contents"),
+                (METADATA_FILE_NAME, "wrong_metadata"),
+            ],
+        )
+        .await?;
+        let timeline_files = list_file_paths_with_contents(&timeline_dir).await?;
+        assert_eq!(
+            timeline_files,
+            vec![
+                (
+                    timeline_dir.join("first"),
+                    FileContents::Text("first_contents".to_string())
+                ),
+                (
+                    timeline_dir.join(METADATA_FILE_NAME),
+                    FileContents::Text("wrong_metadata".to_string())
+                ),
+                (
+                    timeline_dir.join("second"),
+                    FileContents::Text("second_contents".to_string())
+                ),
+            ],
+            "Initial timeline contents should contain two normal files and a wrong metadata file"
+        );
+
+        let metadata = TimelineMetadata::new(Lsn(0x30), None, None, Lsn(0), Lsn(0), Lsn(0));
+        let paths_to_archive = timeline_files
+            .into_iter()
+            .map(|(path, _)| path)
+            .collect::<Vec<_>>();
+
+        let tempdir = tempfile::tempdir()?;
+        let base_path = tempdir.path().to_path_buf();
+        let (header, header_size, archive_target) = archive_files_as_stream(
+            &timeline_dir,
+            paths_to_archive.iter(),
+            &metadata,
+            move |mut archive_streamer, archive_name| async move {
+                let archive_target = base_path.join(&archive_name);
+                let mut archive_file = fs::File::create(&archive_target).await?;
+                io::copy(&mut archive_streamer, &mut archive_file).await?;
+                Ok(archive_target)
+            },
+        )
+        .await?;
+
+        let mut file = fs::File::open(&archive_target).await?;
+        file.seek(io::SeekFrom::Start(header_size)).await?;
+        let target_dir = tempdir.path().join("extracted");
+        uncompress_with_header(&BTreeSet::new(), &target_dir, header, file).await?;
+
+        let extracted_files = list_file_paths_with_contents(&target_dir).await?;
+
+        assert_eq!(
+            extracted_files,
+            vec![
+                (
+                    target_dir.join("first"),
+                    FileContents::Text("first_contents".to_string())
+                ),
+                (
+                    target_dir.join(METADATA_FILE_NAME),
+                    FileContents::Binary(metadata.to_bytes()?)
+                ),
+                (
+                    target_dir.join("second"),
+                    FileContents::Text("second_contents".to_string())
+                ),
+            ],
+            "Extracted files should contain all local timeline files besides its metadata, which should be taken from the arguments"
+        );
+
+        Ok(())
+    }
+
+    async fn init_directory(
+        root: &Path,
+        files_with_contents: Vec<(&str, &str)>,
+    ) -> anyhow::Result<()> {
+        fs::create_dir_all(root).await?;
+        for (file_name, contents) in files_with_contents {
+            fs::File::create(root.join(file_name))
+                .await?
+                .write_all(contents.as_bytes())
+                .await?;
+        }
+        Ok(())
+    }
+
+    #[derive(PartialEq, Eq, PartialOrd, Ord)]
+    enum FileContents {
+        Text(String),
+        Binary(Vec<u8>),
+    }
+
+    impl std::fmt::Debug for FileContents {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            match self {
+                Self::Text(text) => f.debug_tuple("Text").field(text).finish(),
+                Self::Binary(bytes) => f
+                    .debug_tuple("Binary")
+                    .field(&format!("{} bytes", bytes.len()))
+                    .finish(),
+            }
+        }
+    }
+
+    async fn list_file_paths_with_contents(
+        root: &Path,
+    ) -> anyhow::Result<Vec<(PathBuf, FileContents)>> {
+        let mut file_paths = Vec::new();
+
+        let mut dir_listings = vec![fs::read_dir(root).await?];
+        while let Some(mut dir_listing) = dir_listings.pop() {
+            while let Some(entry) = dir_listing.next_entry().await? {
+                let entry_path = entry.path();
+                if entry_path.is_file() {
+                    let contents = match String::from_utf8(fs::read(&entry_path).await?) {
+                        Ok(text) => FileContents::Text(text),
+                        Err(e) => FileContents::Binary(e.into_bytes()),
+                    };
+                    file_paths.push((entry_path, contents));
+                } else if entry_path.is_dir() {
+                    dir_listings.push(fs::read_dir(entry_path).await?);
+                } else {
+                    info!(
+                        "Skipping path '{}' as it's not a file or a directory",
+                        entry_path.display()
+                    );
+                }
+            }
+        }
+
+        file_paths.sort();
+        Ok(file_paths)
+    }
+}

pageserver/src/remote_storage/storage_sync/download.rs

@@ -1,68 +1,30 @@
-//! Timeline synchrnonization logic to fetch the layer files from remote storage into pageserver's local directory.
+//! Timeline synchrnonization logic to put files from archives on remote storage into pageserver's local directory.
 
-use std::fmt::Debug;
+use std::{borrow::Cow, collections::BTreeSet, path::PathBuf, sync::Arc};
 
-use anyhow::Context;
-use futures::stream::{FuturesUnordered, StreamExt};
+use anyhow::{ensure, Context};
 use tokio::fs;
-use tracing::{debug, error, info, warn};
+use tracing::{debug, error, trace, warn};
+use zenith_utils::zid::ZTenantId;
 
 use crate::{
     config::PageServerConf,
-    layered_repository::metadata::metadata_path,
+    layered_repository::metadata::{metadata_path, TimelineMetadata},
     remote_storage::{
-        storage_sync::{sync_queue, SyncTask},
-        RemoteStorage,
+        storage_sync::{
+            compression, fetch_full_index, index::TimelineIndexEntryInner, sync_queue, SyncKind,
+            SyncTask,
+        },
+        RemoteStorage, ZTenantTimelineId,
     },
 };
-use utils::zid::ZTenantTimelineId;
 
 use super::{
-    index::{IndexPart, RemoteTimeline},
-    SyncData, TimelineDownload,
+    index::{ArchiveId, RemoteTimeline},
+    RemoteIndex, TimelineDownload,
 };
 
-/// Retrieves index data from the remote storage for a given timeline.
-pub async fn download_index_part<P, S>(
-    conf: &'static PageServerConf,
-    storage: &S,
-    sync_id: ZTenantTimelineId,
-) -> anyhow::Result<IndexPart>
-where
-    P: Debug + Send + Sync + 'static,
-    S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
-{
-    let index_part_path = metadata_path(conf, sync_id.timeline_id, sync_id.tenant_id)
-        .with_file_name(IndexPart::FILE_NAME)
-        .with_extension(IndexPart::FILE_EXTENSION);
-    let part_storage_path = storage.storage_path(&index_part_path).with_context(|| {
-        format!(
-            "Failed to get the index part storage path for local path '{}'",
-            index_part_path.display()
-        )
-    })?;
-    let mut index_part_bytes = Vec::new();
-    storage
-        .download(&part_storage_path, &mut index_part_bytes)
-        .await
-        .with_context(|| {
-            format!("Failed to download an index part from storage path '{part_storage_path:?}'")
-        })?;
-
-    let index_part: IndexPart = serde_json::from_slice(&index_part_bytes).with_context(|| {
-        format!("Failed to deserialize index part file from storage path '{part_storage_path:?}'")
-    })?;
-
-    let missing_files = index_part.missing_files();
-    if !missing_files.is_empty() {
-        warn!("Found missing layers in index part for timeline {sync_id}: {missing_files:?}");
-    }
-
-    Ok(index_part)
-}
-
 /// Timeline download result, with extra data, needed for downloading.
-#[derive(Debug)]
 pub(super) enum DownloadedTimeline {
     /// Remote timeline data is either absent or corrupt, no download possible.
     Abort,
@@ -71,127 +33,217 @@ pub(super) enum DownloadedTimeline {
     FailedAndRescheduled,
     /// Remote timeline data is found, its latest checkpoint's metadata contents (disk_consistent_lsn) is known.
     /// Initial download successful.
-    Successful(SyncData<TimelineDownload>),
+    Successful,
 }
 
-/// Attempts to download all given timeline's layers.
+/// Attempts to download and uncompress files from all remote archives for the timeline given.
 /// Timeline files that already exist locally are skipped during the download, but the local metadata file is
-/// updated in the end, if the remote one contains a newer disk_consistent_lsn.
+/// updated in the end of every checkpoint archive extraction.
 ///
-/// On an error, bumps the retries count and updates the files to skip with successful downloads, rescheduling the task.
-pub(super) async fn download_timeline_layers<'a, P, S>(
-    storage: &'a S,
-    remote_timeline: Option<&'a RemoteTimeline>,
-    sync_id: ZTenantTimelineId,
-    mut download_data: SyncData<TimelineDownload>,
-) -> DownloadedTimeline
-where
-    P: Debug + Send + Sync + 'static,
+/// On an error, bumps the retries count and reschedules the download, with updated archive skip list
+/// (for any new successful archive downloads and extractions).
+pub(super) async fn download_timeline<
+    P: std::fmt::Debug + Send + Sync + 'static,
     S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
-{
-    let remote_timeline = match remote_timeline {
-        Some(remote_timeline) => {
-            if !remote_timeline.awaits_download {
-                error!("Timeline with sync id {sync_id} is not awaiting download");
-                return DownloadedTimeline::Abort;
-            }
-            remote_timeline
-        }
+>(
+    conf: &'static PageServerConf,
+    remote_assets: Arc<(S, RemoteIndex)>,
+    sync_id: ZTenantTimelineId,
+    mut download: TimelineDownload,
+    retries: u32,
+) -> DownloadedTimeline {
+    debug!("Downloading layers for sync id {}", sync_id);
+
+    let ZTenantTimelineId {
+        tenant_id,
+        timeline_id,
+    } = sync_id;
+    let index = &remote_assets.1;
+
+    let index_read = index.read().await;
+    let remote_timeline = match index_read.timeline_entry(&sync_id) {
         None => {
-            error!("Timeline with sync id {sync_id} is not present in the remote index");
+            error!("Cannot download: no timeline is present in the index for given id");
             return DownloadedTimeline::Abort;
         }
+
+        Some(index_entry) => match index_entry.inner() {
+            TimelineIndexEntryInner::Full(remote_timeline) => Cow::Borrowed(remote_timeline),
+            TimelineIndexEntryInner::Description(_) => {
+                // we do not check here for awaits_download because it is ok
+                // to call this function while the download is in progress
+                // so it is not a concurrent download, it is the same one
+
+                let remote_disk_consistent_lsn = index_entry.disk_consistent_lsn();
+                drop(index_read);
+                debug!("Found timeline description for the given ids, downloading the full index");
+                match fetch_full_index(
+                    remote_assets.as_ref(),
+                    &conf.timeline_path(&timeline_id, &tenant_id),
+                    sync_id,
+                )
+                .await
+                {
+                    Ok(remote_timeline) => Cow::Owned(remote_timeline),
+                    Err(e) => {
+                        error!("Failed to download full timeline index: {:?}", e);
+
+                        return match remote_disk_consistent_lsn {
+                            Some(_) => {
+                                sync_queue::push(SyncTask::new(
+                                    sync_id,
+                                    retries,
+                                    SyncKind::Download(download),
+                                ));
+                                DownloadedTimeline::FailedAndRescheduled
+                            }
+                            None => {
+                                error!("Cannot download: no disk consistent Lsn is present for the index entry");
+                                DownloadedTimeline::Abort
+                            }
+                        };
+                    }
+                }
+            }
+        },
+    };
+    if remote_timeline.checkpoints().max().is_none() {
+        debug!("Cannot download: no disk consistent Lsn is present for the remote timeline");
+        return DownloadedTimeline::Abort;
     };
 
-    let download = &mut download_data.data;
-
-    let layers_to_download = remote_timeline
-        .stored_files()
-        .difference(&download.layers_to_skip)
-        .cloned()
+    debug!("Downloading timeline archives");
+    let archives_to_download = remote_timeline
+        .checkpoints()
+        .map(ArchiveId)
+        .filter(|remote_archive| !download.archives_to_skip.contains(remote_archive))
         .collect::<Vec<_>>();
 
-    debug!("Layers to download: {layers_to_download:?}");
-    info!("Downloading {} timeline layers", layers_to_download.len());
+    let archives_total = archives_to_download.len();
+    debug!("Downloading {} archives of a timeline", archives_total);
+    trace!("Archives to download: {:?}", archives_to_download);
 
-    let mut download_tasks = layers_to_download
-        .into_iter()
-        .map(|layer_desination_path| async move {
-            if layer_desination_path.exists() {
-                debug!(
-                    "Layer already exists locally, skipping download: {}",
-                    layer_desination_path.display()
-                );
-            } else {
-                let layer_storage_path = storage
-                    .storage_path(&layer_desination_path)
-                    .with_context(|| {
-                        format!(
-                            "Failed to get the layer storage path for local path '{}'",
-                            layer_desination_path.display()
-                        )
-                    })?;
-
-                let mut destination_file = fs::File::create(&layer_desination_path)
-                    .await
-                    .with_context(|| {
-                        format!(
-                            "Failed to create a destination file for layer '{}'",
-                            layer_desination_path.display()
-                        )
-                    })?;
-
-                storage
-                    .download(&layer_storage_path, &mut destination_file)
-                    .await
-                    .with_context(|| {
-                        format!(
-                            "Failed to download a layer from storage path '{layer_storage_path:?}'"
-                        )
-                    })?;
-            }
-            Ok::<_, anyhow::Error>(layer_desination_path)
-        })
-        .collect::<FuturesUnordered<_>>();
-
-    let mut errors_happened = false;
-    while let Some(download_result) = download_tasks.next().await {
-        match download_result {
-            Ok(downloaded_path) => {
-                download.layers_to_skip.insert(downloaded_path);
-            }
+    for (archives_downloaded, archive_id) in archives_to_download.into_iter().enumerate() {
+        match try_download_archive(
+            conf,
+            sync_id,
+            Arc::clone(&remote_assets),
+            &remote_timeline,
+            archive_id,
+            Arc::clone(&download.files_to_skip),
+        )
+        .await
+        {
             Err(e) => {
-                errors_happened = true;
-                error!("Failed to download a layer for timeline {sync_id}: {e:?}");
+                let archives_left = archives_total - archives_downloaded;
+                error!(
+                    "Failed to download archive {:?} (archives downloaded: {}; archives left: {}) for tenant {} timeline {}, requeueing the download: {:?}",
+                    archive_id, archives_downloaded, archives_left, tenant_id, timeline_id, e
+                );
+                sync_queue::push(SyncTask::new(
+                    sync_id,
+                    retries,
+                    SyncKind::Download(download),
+                ));
+                return DownloadedTimeline::FailedAndRescheduled;
+            }
+            Ok(()) => {
+                debug!("Successfully downloaded archive {:?}", archive_id);
+                download.archives_to_skip.insert(archive_id);
             }
         }
     }
 
-    if errors_happened {
-        debug!("Reenqueuing failed download task for timeline {sync_id}");
-        download_data.retries += 1;
-        sync_queue::push(sync_id, SyncTask::Download(download_data));
-        DownloadedTimeline::FailedAndRescheduled
-    } else {
-        info!("Successfully downloaded all layers");
-        DownloadedTimeline::Successful(download_data)
+    debug!("Finished downloading all timeline's archives");
+    DownloadedTimeline::Successful
+}
+
+async fn try_download_archive<
+    P: Send + Sync + 'static,
+    S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
+>(
+    conf: &'static PageServerConf,
+    ZTenantTimelineId {
+        tenant_id,
+        timeline_id,
+    }: ZTenantTimelineId,
+    remote_assets: Arc<(S, RemoteIndex)>,
+    remote_timeline: &RemoteTimeline,
+    archive_id: ArchiveId,
+    files_to_skip: Arc<BTreeSet<PathBuf>>,
+) -> anyhow::Result<()> {
+    debug!("Downloading archive {:?}", archive_id);
+    let archive_to_download = remote_timeline
+        .archive_data(archive_id)
+        .with_context(|| format!("Archive {:?} not found in remote storage", archive_id))?;
+    let (archive_header, header_size) = remote_timeline
+        .restore_header(archive_id)
+        .context("Failed to restore header when downloading an archive")?;
+
+    match read_local_metadata(conf, timeline_id, tenant_id).await {
+        Ok(local_metadata) => ensure!(
+            // need to allow `<=` instead of `<` due to cases when a failed archive can be redownloaded
+            local_metadata.disk_consistent_lsn() <= archive_to_download.disk_consistent_lsn(),
+            "Cannot download archive with Lsn {} since it's earlier than local Lsn {}",
+            archive_to_download.disk_consistent_lsn(),
+            local_metadata.disk_consistent_lsn()
+        ),
+        Err(e) => warn!("Failed to read local metadata file, assuming it's safe to override its with the download. Read: {:#}", e),
     }
+    compression::uncompress_file_stream_with_index(
+        conf.timeline_path(&timeline_id, &tenant_id),
+        files_to_skip,
+        archive_to_download.disk_consistent_lsn(),
+        archive_header,
+        header_size,
+        move |mut archive_target, archive_name| async move {
+            let archive_local_path = conf
+                .timeline_path(&timeline_id, &tenant_id)
+                .join(&archive_name);
+            let remote_storage = &remote_assets.0;
+            remote_storage
+                .download_range(
+                    &remote_storage.storage_path(&archive_local_path)?,
+                    header_size,
+                    None,
+                    &mut archive_target,
+                )
+                .await
+        },
+    )
+    .await?;
+
+    Ok(())
+}
+
+async fn read_local_metadata(
+    conf: &'static PageServerConf,
+    timeline_id: zenith_utils::zid::ZTimelineId,
+    tenant_id: ZTenantId,
+) -> anyhow::Result<TimelineMetadata> {
+    let local_metadata_path = metadata_path(conf, timeline_id, tenant_id);
+    let local_metadata_bytes = fs::read(&local_metadata_path)
+        .await
+        .context("Failed to read local metadata file bytes")?;
+    Ok(TimelineMetadata::from_bytes(&local_metadata_bytes)
+        .context("Failed to read local metadata files bytes")?)
 }
 
 #[cfg(test)]
 mod tests {
-    use std::collections::{BTreeSet, HashSet};
+    use std::collections::BTreeSet;
 
     use tempfile::tempdir;
-    use utils::lsn::Lsn;
+    use tokio::fs;
+    use zenith_utils::lsn::Lsn;
 
     use crate::{
         remote_storage::{
-            storage_sync::{
-                index::RelativePath,
-                test_utils::{create_local_timeline, dummy_metadata},
+            local_fs::LocalFs,
+            storage_sync::test_utils::{
+                assert_index_descriptions, assert_timeline_files_match, create_local_timeline,
+                dummy_metadata, ensure_correct_timeline_upload, expect_timeline,
             },
-            LocalFs,
         },
         repository::repo_harness::{RepoHarness, TIMELINE_ID},
     };
@@ -199,184 +251,80 @@ mod tests {
     use super::*;
 
     #[tokio::test]
-    async fn download_timeline() -> anyhow::Result<()> {
-        let harness = RepoHarness::create("download_timeline")?;
-        let sync_id = ZTenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-        let layer_files = ["a", "b", "layer_to_skip", "layer_to_keep_locally"];
-        let storage = LocalFs::new(tempdir()?.path().to_path_buf(), &harness.conf.workdir)?;
-        let current_retries = 3;
-        let metadata = dummy_metadata(Lsn(0x30));
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let timeline_upload =
-            create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?;
-
-        for local_path in timeline_upload.layers_to_upload {
-            let remote_path = storage.storage_path(&local_path)?;
-            let remote_parent_dir = remote_path.parent().unwrap();
-            if !remote_parent_dir.exists() {
-                fs::create_dir_all(&remote_parent_dir).await?;
-            }
-            fs::copy(&local_path, &remote_path).await?;
-        }
-        let mut read_dir = fs::read_dir(&local_timeline_path).await?;
-        while let Some(dir_entry) = read_dir.next_entry().await? {
-            if dir_entry.file_name().to_str() == Some("layer_to_keep_locally") {
-                continue;
-            } else {
-                fs::remove_file(dir_entry.path()).await?;
-            }
-        }
-
-        let mut remote_timeline = RemoteTimeline::new(metadata.clone());
-        remote_timeline.awaits_download = true;
-        remote_timeline.add_timeline_layers(
-            layer_files
-                .iter()
-                .map(|layer| local_timeline_path.join(layer)),
-        );
-
-        let download_data = match download_timeline_layers(
-            &storage,
-            Some(&remote_timeline),
-            sync_id,
-            SyncData::new(
-                current_retries,
-                TimelineDownload {
-                    layers_to_skip: HashSet::from([local_timeline_path.join("layer_to_skip")]),
-                },
-            ),
-        )
-        .await
-        {
-            DownloadedTimeline::Successful(data) => data,
-            wrong_result => {
-                panic!("Expected a successful download for timeline, but got: {wrong_result:?}")
-            }
-        };
-
-        assert_eq!(
-            current_retries, download_data.retries,
-            "On successful download, retries are not expected to change"
-        );
-        assert_eq!(
-            download_data
-                .data
-                .layers_to_skip
+    async fn test_download_timeline() -> anyhow::Result<()> {
+        let repo_harness = RepoHarness::create("test_download_timeline")?;
+        let sync_id = ZTenantTimelineId::new(repo_harness.tenant_id, TIMELINE_ID);
+        let storage = LocalFs::new(tempdir()?.path().to_owned(), &repo_harness.conf.workdir)?;
+        let index = RemoteIndex::try_parse_descriptions_from_paths(
+            repo_harness.conf,
+            storage
+                .list()
+                .await?
                 .into_iter()
-                .collect::<BTreeSet<_>>(),
-            layer_files
-                .iter()
-                .map(|layer| local_timeline_path.join(layer))
-                .collect(),
-            "On successful download, layers to skip should contain all downloaded files and present layers that were skipped"
+                .map(|storage_path| storage.local_path(&storage_path).unwrap()),
         );
+        let remote_assets = Arc::new((storage, index));
+        let storage = &remote_assets.0;
+        let index = &remote_assets.1;
 
-        let mut downloaded_files = BTreeSet::new();
-        let mut read_dir = fs::read_dir(&local_timeline_path).await?;
-        while let Some(dir_entry) = read_dir.next_entry().await? {
-            downloaded_files.insert(dir_entry.path());
-        }
+        let regular_timeline_path = repo_harness.timeline_path(&TIMELINE_ID);
+        let regular_timeline = create_local_timeline(
+            &repo_harness,
+            TIMELINE_ID,
+            &["a", "b"],
+            dummy_metadata(Lsn(0x30)),
+        )?;
+        ensure_correct_timeline_upload(
+            &repo_harness,
+            Arc::clone(&remote_assets),
+            TIMELINE_ID,
+            regular_timeline,
+        )
+        .await;
+        // upload multiple checkpoints for the same timeline
+        let regular_timeline = create_local_timeline(
+            &repo_harness,
+            TIMELINE_ID,
+            &["c", "d"],
+            dummy_metadata(Lsn(0x40)),
+        )?;
+        ensure_correct_timeline_upload(
+            &repo_harness,
+            Arc::clone(&remote_assets),
+            TIMELINE_ID,
+            regular_timeline,
+        )
+        .await;
 
-        assert_eq!(
-            downloaded_files,
-            layer_files
-                .iter()
-                .filter(|layer| layer != &&"layer_to_skip")
-                .map(|layer| local_timeline_path.join(layer))
-                .collect(),
-            "On successful download, all layers that were not skipped, should be downloaded"
-        );
+        fs::remove_dir_all(&regular_timeline_path).await?;
+        let remote_regular_timeline = expect_timeline(index, sync_id).await;
 
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn download_timeline_negatives() -> anyhow::Result<()> {
-        let harness = RepoHarness::create("download_timeline_negatives")?;
-        let sync_id = ZTenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-        let storage = LocalFs::new(tempdir()?.path().to_owned(), &harness.conf.workdir)?;
-
-        let empty_remote_timeline_download = download_timeline_layers(
-            &storage,
-            None,
+        download_timeline(
+            repo_harness.conf,
+            Arc::clone(&remote_assets),
             sync_id,
-            SyncData::new(
-                0,
-                TimelineDownload {
-                    layers_to_skip: HashSet::new(),
-                },
+            TimelineDownload {
+                files_to_skip: Arc::new(BTreeSet::new()),
+                archives_to_skip: BTreeSet::new(),
+            },
+            0,
+        )
+        .await;
+        assert_index_descriptions(
+            index,
+            &RemoteIndex::try_parse_descriptions_from_paths(
+                repo_harness.conf,
+                remote_assets
+                    .0
+                    .list()
+                    .await
+                    .unwrap()
+                    .into_iter()
+                    .map(|storage_path| storage.local_path(&storage_path).unwrap()),
             ),
         )
         .await;
-        assert!(
-            matches!(empty_remote_timeline_download, DownloadedTimeline::Abort),
-            "Should not allow downloading for empty remote timeline"
-        );
-
-        let not_expecting_download_remote_timeline = RemoteTimeline::new(dummy_metadata(Lsn(5)));
-        assert!(
-            !not_expecting_download_remote_timeline.awaits_download,
-            "Should not expect download for the timeline"
-        );
-        let already_downloading_remote_timeline_download = download_timeline_layers(
-            &storage,
-            Some(&not_expecting_download_remote_timeline),
-            sync_id,
-            SyncData::new(
-                0,
-                TimelineDownload {
-                    layers_to_skip: HashSet::new(),
-                },
-            ),
-        )
-        .await;
-        assert!(
-            matches!(
-                dbg!(already_downloading_remote_timeline_download),
-                DownloadedTimeline::Abort,
-            ),
-            "Should not allow downloading for remote timeline that does not expect it"
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_download_index_part() -> anyhow::Result<()> {
-        let harness = RepoHarness::create("test_download_index_part")?;
-        let sync_id = ZTenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-
-        let storage = LocalFs::new(tempdir()?.path().to_path_buf(), &harness.conf.workdir)?;
-        let metadata = dummy_metadata(Lsn(0x30));
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-
-        let index_part = IndexPart::new(
-            HashSet::from([
-                RelativePath::new(&local_timeline_path, local_timeline_path.join("one"))?,
-                RelativePath::new(&local_timeline_path, local_timeline_path.join("two"))?,
-            ]),
-            HashSet::from([RelativePath::new(
-                &local_timeline_path,
-                local_timeline_path.join("three"),
-            )?]),
-            metadata.disk_consistent_lsn(),
-            metadata.to_bytes()?,
-        );
-
-        let local_index_part_path =
-            metadata_path(harness.conf, sync_id.timeline_id, sync_id.tenant_id)
-                .with_file_name(IndexPart::FILE_NAME)
-                .with_extension(IndexPart::FILE_EXTENSION);
-        let storage_path = storage.storage_path(&local_index_part_path)?;
-        fs::create_dir_all(storage_path.parent().unwrap()).await?;
-        fs::write(&storage_path, serde_json::to_vec(&index_part)?).await?;
-
-        let downloaded_index_part = download_index_part(harness.conf, &storage, sync_id).await?;
-
-        assert_eq!(
-            downloaded_index_part, index_part,
-            "Downloaded index part should be the same as the one in storage"
-        );
+        assert_timeline_files_match(&repo_harness, TIMELINE_ID, remote_regular_timeline);
 
         Ok(())
     }

pageserver/src/remote_storage/storage_sync/index.rs

@@ -1,53 +1,63 @@
-//! In-memory index to track the tenant files on the remote storage.
-//! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about
-//! remote timeline layers and its metadata.
+//! In-memory index to track the tenant files on the remote strorage, mitigating the storage format differences between the local and remote files.
+//! Able to restore itself from the storage archive data and reconstruct archive indices on demand.
+//!
+//! The index is intended to be portable, so deliberately does not store any local paths inside.
+//! This way in the future, the index could be restored fast from its serialized stored form.
 
 use std::{
-    collections::{HashMap, HashSet},
+    collections::{BTreeMap, BTreeSet, HashMap},
     path::{Path, PathBuf},
     sync::Arc,
 };
 
-use anyhow::{Context, Ok};
+use anyhow::{bail, ensure, Context};
 use serde::{Deserialize, Serialize};
-use serde_with::{serde_as, DisplayFromStr};
 use tokio::sync::RwLock;
+use tracing::*;
+use zenith_utils::{
+    lsn::Lsn,
+    zid::{ZTenantId, ZTimelineId},
+};
 
-use crate::{config::PageServerConf, layered_repository::metadata::TimelineMetadata};
-use utils::{lsn::Lsn, zid::ZTenantTimelineId};
+use crate::{
+    config::PageServerConf,
+    layered_repository::TIMELINES_SEGMENT_NAME,
+    remote_storage::{
+        storage_sync::compression::{parse_archive_name, FileEntry},
+        ZTenantTimelineId,
+    },
+};
+
+use super::compression::ArchiveHeader;
 
 /// A part of the filesystem path, that needs a root to become a path again.
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
-#[serde(transparent)]
 pub struct RelativePath(String);
 
 impl RelativePath {
     /// Attempts to strip off the base from path, producing a relative path or an error.
     pub fn new<P: AsRef<Path>>(base: &Path, path: P) -> anyhow::Result<Self> {
-        let path = path.as_ref();
-        let relative = path.strip_prefix(base).with_context(|| {
-            format!(
-                "path '{}' is not relative to base '{}'",
-                path.display(),
-                base.display()
-            )
-        })?;
+        let relative = path
+            .as_ref()
+            .strip_prefix(base)
+            .context("path is not relative to base")?;
         Ok(RelativePath(relative.to_string_lossy().to_string()))
     }
 
     /// Joins the relative path with the base path.
-    fn as_path(&self, base: &Path) -> PathBuf {
+    pub fn as_path(&self, base: &Path) -> PathBuf {
         base.join(&self.0)
     }
 }
 
 /// An index to track tenant files that exist on the remote storage.
+/// Currently, timeline archive files are tracked only.
 #[derive(Debug, Clone)]
 pub struct RemoteTimelineIndex {
-    timeline_entries: HashMap<ZTenantTimelineId, RemoteTimeline>,
+    timeline_entries: HashMap<ZTenantTimelineId, TimelineIndexEntry>,
 }
 
-/// A wrapper to synchronize the access to the index, should be created and used before dealing with any [`RemoteTimelineIndex`].
+/// A wrapper to synchrnize access to the index, should be created and used before dealing with any [`RemoteTimelineIndex`].
 pub struct RemoteIndex(Arc<RwLock<RemoteTimelineIndex>>);
 
 impl RemoteIndex {
@@ -57,22 +67,27 @@ impl RemoteIndex {
         })))
     }
 
-    pub fn from_parts(
+    /// Attempts to parse file paths (not checking the file contents) and find files
+    /// that can be tracked wiht the index.
+    /// On parse falures, logs the error and continues, so empty index can be created from not suitable paths.
+    pub fn try_parse_descriptions_from_paths<P: AsRef<Path>>(
         conf: &'static PageServerConf,
-        index_parts: HashMap<ZTenantTimelineId, IndexPart>,
-    ) -> anyhow::Result<Self> {
-        let mut timeline_entries = HashMap::new();
-
-        for (sync_id, index_part) in index_parts {
-            let timeline_path = conf.timeline_path(&sync_id.timeline_id, &sync_id.tenant_id);
-            let remote_timeline = RemoteTimeline::from_index_part(&timeline_path, index_part)
-                .context("Failed to restore remote timeline data from index part")?;
-            timeline_entries.insert(sync_id, remote_timeline);
+        paths: impl Iterator<Item = P>,
+    ) -> Self {
+        let mut index = RemoteTimelineIndex {
+            timeline_entries: HashMap::new(),
+        };
+        for path in paths {
+            if let Err(e) = try_parse_index_entry(&mut index, conf, path.as_ref()) {
+                debug!(
+                    "Failed to parse path '{}' as index entry: {:#}",
+                    path.as_ref().display(),
+                    e
+                );
+            }
         }
 
-        Ok(Self(Arc::new(RwLock::new(RemoteTimelineIndex {
-            timeline_entries,
-        }))))
+        Self(Arc::new(RwLock::new(index)))
     }
 
     pub async fn read(&self) -> tokio::sync::RwLockReadGuard<'_, RemoteTimelineIndex> {
@@ -91,18 +106,39 @@ impl Clone for RemoteIndex {
 }
 
 impl RemoteTimelineIndex {
-    pub fn timeline_entry(&self, id: &ZTenantTimelineId) -> Option<&RemoteTimeline> {
+    pub fn timeline_entry(&self, id: &ZTenantTimelineId) -> Option<&TimelineIndexEntry> {
         self.timeline_entries.get(id)
     }
 
-    pub fn timeline_entry_mut(&mut self, id: &ZTenantTimelineId) -> Option<&mut RemoteTimeline> {
+    pub fn timeline_entry_mut(
+        &mut self,
+        id: &ZTenantTimelineId,
+    ) -> Option<&mut TimelineIndexEntry> {
         self.timeline_entries.get_mut(id)
     }
 
-    pub fn add_timeline_entry(&mut self, id: ZTenantTimelineId, entry: RemoteTimeline) {
+    pub fn add_timeline_entry(&mut self, id: ZTenantTimelineId, entry: TimelineIndexEntry) {
         self.timeline_entries.insert(id, entry);
     }
 
+    pub fn upgrade_timeline_entry(
+        &mut self,
+        id: &ZTenantTimelineId,
+        remote_timeline: RemoteTimeline,
+    ) -> anyhow::Result<()> {
+        let mut entry = self.timeline_entries.get_mut(id).ok_or(anyhow::anyhow!(
+            "timeline is unexpectedly missing from remote index"
+        ))?;
+
+        if !matches!(entry.inner, TimelineIndexEntryInner::Description(_)) {
+            anyhow::bail!("timeline entry is not a description entry")
+        };
+
+        entry.inner = TimelineIndexEntryInner::Full(remote_timeline);
+
+        Ok(())
+    }
+
     pub fn all_sync_ids(&self) -> impl Iterator<Item = ZTenantTimelineId> + '_ {
         self.timeline_entries.keys().copied()
     }
@@ -114,295 +150,351 @@ impl RemoteTimelineIndex {
     ) -> anyhow::Result<()> {
         self.timeline_entry_mut(id)
             .ok_or_else(|| anyhow::anyhow!("unknown timeline sync {}", id))?
-            .awaits_download = awaits_download;
+            .set_awaits_download(awaits_download);
         Ok(())
     }
 }
 
-/// Restored index part data about the timeline, stored in the remote index.
-#[derive(Debug, PartialEq, Eq, Clone)]
-pub struct RemoteTimeline {
-    timeline_layers: HashSet<PathBuf>,
-    missing_layers: HashSet<PathBuf>,
-
-    pub metadata: TimelineMetadata,
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub struct DescriptionTimelineIndexEntry {
+    pub description: BTreeMap<ArchiveId, ArchiveDescription>,
     pub awaits_download: bool,
 }
 
-impl RemoteTimeline {
-    pub fn new(metadata: TimelineMetadata) -> Self {
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FullTimelineIndexEntry {
+    pub remote_timeline: RemoteTimeline,
+    pub awaits_download: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum TimelineIndexEntryInner {
+    Description(BTreeMap<ArchiveId, ArchiveDescription>),
+    Full(RemoteTimeline),
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TimelineIndexEntry {
+    inner: TimelineIndexEntryInner,
+    awaits_download: bool,
+}
+
+impl TimelineIndexEntry {
+    pub fn new(inner: TimelineIndexEntryInner, awaits_download: bool) -> Self {
         Self {
-            timeline_layers: HashSet::new(),
-            missing_layers: HashSet::new(),
-            metadata,
-            awaits_download: false,
+            inner,
+            awaits_download,
         }
     }
 
-    pub fn add_timeline_layers(&mut self, new_layers: impl IntoIterator<Item = PathBuf>) {
-        self.timeline_layers.extend(new_layers.into_iter());
+    pub fn inner(&self) -> &TimelineIndexEntryInner {
+        &self.inner
     }
 
-    pub fn add_upload_failures(&mut self, upload_failures: impl IntoIterator<Item = PathBuf>) {
-        self.missing_layers.extend(upload_failures.into_iter());
+    pub fn inner_mut(&mut self) -> &mut TimelineIndexEntryInner {
+        &mut self.inner
+    }
+
+    pub fn uploaded_checkpoints(&self) -> BTreeSet<Lsn> {
+        match &self.inner {
+            TimelineIndexEntryInner::Description(description) => {
+                description.keys().map(|archive_id| archive_id.0).collect()
+            }
+            TimelineIndexEntryInner::Full(remote_timeline) => remote_timeline
+                .checkpoint_archives
+                .keys()
+                .map(|archive_id| archive_id.0)
+                .collect(),
+        }
+    }
+
+    /// Gets latest uploaded checkpoint's disk consisten Lsn for the corresponding timeline.
+    pub fn disk_consistent_lsn(&self) -> Option<Lsn> {
+        match &self.inner {
+            TimelineIndexEntryInner::Description(description) => {
+                description.keys().map(|archive_id| archive_id.0).max()
+            }
+            TimelineIndexEntryInner::Full(remote_timeline) => remote_timeline
+                .checkpoint_archives
+                .keys()
+                .map(|archive_id| archive_id.0)
+                .max(),
+        }
+    }
+
+    pub fn get_awaits_download(&self) -> bool {
+        self.awaits_download
+    }
+
+    pub fn set_awaits_download(&mut self, awaits_download: bool) {
+        self.awaits_download = awaits_download;
+    }
+}
+
+/// Checkpoint archive's id, corresponding to the `disk_consistent_lsn` from the timeline's metadata file during checkpointing.
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
+pub struct ArchiveId(pub(super) Lsn);
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
+struct FileId(ArchiveId, ArchiveEntryNumber);
+
+type ArchiveEntryNumber = usize;
+
+/// All archives and files in them, representing a certain timeline.
+/// Uses file and archive IDs to reference those without ownership issues.
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct RemoteTimeline {
+    timeline_files: BTreeMap<FileId, FileEntry>,
+    checkpoint_archives: BTreeMap<ArchiveId, CheckpointArchive>,
+}
+
+/// Archive metadata, enough to restore a header with the timeline data.
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct CheckpointArchive {
+    disk_consistent_lsn: Lsn,
+    metadata_file_size: u64,
+    files: BTreeSet<FileId>,
+    archive_header_size: u64,
+}
+
+impl CheckpointArchive {
+    pub fn disk_consistent_lsn(&self) -> Lsn {
+        self.disk_consistent_lsn
+    }
+}
+
+impl RemoteTimeline {
+    pub fn empty() -> Self {
+        Self {
+            timeline_files: BTreeMap::new(),
+            checkpoint_archives: BTreeMap::new(),
+        }
+    }
+
+    pub fn checkpoints(&self) -> impl Iterator<Item = Lsn> + '_ {
+        self.checkpoint_archives
+            .values()
+            .map(CheckpointArchive::disk_consistent_lsn)
     }
 
     /// Lists all layer files in the given remote timeline. Omits the metadata file.
-    pub fn stored_files(&self) -> &HashSet<PathBuf> {
-        &self.timeline_layers
+    pub fn stored_files(&self, timeline_dir: &Path) -> BTreeSet<PathBuf> {
+        self.timeline_files
+            .values()
+            .map(|file_entry| file_entry.subpath.as_path(timeline_dir))
+            .collect()
     }
 
-    pub fn from_index_part(timeline_path: &Path, index_part: IndexPart) -> anyhow::Result<Self> {
-        let metadata = TimelineMetadata::from_bytes(&index_part.metadata_bytes)?;
-        Ok(Self {
-            timeline_layers: to_local_paths(timeline_path, index_part.timeline_layers),
-            missing_layers: to_local_paths(timeline_path, index_part.missing_layers),
-            metadata,
-            awaits_download: false,
-        })
+    pub fn contains_checkpoint_at(&self, disk_consistent_lsn: Lsn) -> bool {
+        self.checkpoint_archives
+            .contains_key(&ArchiveId(disk_consistent_lsn))
     }
-}
 
-/// Part of the remote index, corresponding to a certain timeline.
-/// Contains the data about all files in the timeline, present remotely and its metadata.
-#[serde_as]
-#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
-pub struct IndexPart {
-    timeline_layers: HashSet<RelativePath>,
-    /// Currently is not really used in pageserver,
-    /// present to manually keep track of the layer files that pageserver might never retrieve.
-    ///
-    /// Such "holes" might appear if any upload task was evicted on an error threshold:
-    /// the this layer will only be rescheduled for upload on pageserver restart.
-    missing_layers: HashSet<RelativePath>,
-    #[serde_as(as = "DisplayFromStr")]
-    disk_consistent_lsn: Lsn,
-    metadata_bytes: Vec<u8>,
-}
+    pub fn archive_data(&self, archive_id: ArchiveId) -> Option<&CheckpointArchive> {
+        self.checkpoint_archives.get(&archive_id)
+    }
 
-impl IndexPart {
-    pub const FILE_NAME: &'static str = "index_part";
-    pub const FILE_EXTENSION: &'static str = "json";
+    /// Restores a header of a certain remote archive from the memory data.
+    /// Returns the header and its compressed size in the archive, both can be used to uncompress that archive.
+    pub fn restore_header(&self, archive_id: ArchiveId) -> anyhow::Result<(ArchiveHeader, u64)> {
+        let archive = self
+            .checkpoint_archives
+            .get(&archive_id)
+            .with_context(|| format!("Archive {:?} not found", archive_id))?;
 
-    #[cfg(test)]
-    pub fn new(
-        timeline_layers: HashSet<RelativePath>,
-        missing_layers: HashSet<RelativePath>,
+        let mut header_files = Vec::with_capacity(archive.files.len());
+        for (expected_archive_position, archive_file) in archive.files.iter().enumerate() {
+            let &FileId(archive_id, archive_position) = archive_file;
+            ensure!(
+                expected_archive_position == archive_position,
+                "Archive header is corrupt, file # {} from archive {:?} header is missing",
+                expected_archive_position,
+                archive_id,
+            );
+
+            let timeline_file = self.timeline_files.get(archive_file).with_context(|| {
+                format!(
+                    "File with id {:?} not found for archive {:?}",
+                    archive_file, archive_id
+                )
+            })?;
+            header_files.push(timeline_file.clone());
+        }
+
+        Ok((
+            ArchiveHeader {
+                files: header_files,
+                metadata_file_size: archive.metadata_file_size,
+            },
+            archive.archive_header_size,
+        ))
+    }
+
+    /// Updates (creates, if necessary) the data about certain archive contents.
+    pub fn update_archive_contents(
+        &mut self,
         disk_consistent_lsn: Lsn,
-        metadata_bytes: Vec<u8>,
-    ) -> Self {
-        Self {
-            timeline_layers,
-            missing_layers,
-            disk_consistent_lsn,
-            metadata_bytes,
+        header: ArchiveHeader,
+        header_size: u64,
+    ) {
+        let archive_id = ArchiveId(disk_consistent_lsn);
+        let mut common_archive_files = BTreeSet::new();
+        for (file_index, file_entry) in header.files.into_iter().enumerate() {
+            let file_id = FileId(archive_id, file_index);
+            self.timeline_files.insert(file_id, file_entry);
+            common_archive_files.insert(file_id);
+        }
+
+        let metadata_file_size = header.metadata_file_size;
+        self.checkpoint_archives
+            .entry(archive_id)
+            .or_insert_with(|| CheckpointArchive {
+                metadata_file_size,
+                files: BTreeSet::new(),
+                archive_header_size: header_size,
+                disk_consistent_lsn,
+            })
+            .files
+            .extend(common_archive_files.into_iter());
+    }
+}
+
+/// Metadata abput timeline checkpoint archive, parsed from its remote storage path.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ArchiveDescription {
+    pub header_size: u64,
+    pub disk_consistent_lsn: Lsn,
+    pub archive_name: String,
+}
+
+fn try_parse_index_entry(
+    index: &mut RemoteTimelineIndex,
+    conf: &'static PageServerConf,
+    path: &Path,
+) -> anyhow::Result<()> {
+    let tenants_dir = conf.tenants_path();
+    let tenant_id = path
+        .strip_prefix(&tenants_dir)
+        .with_context(|| {
+            format!(
+                "Path '{}' does not belong to tenants directory '{}'",
+                path.display(),
+                tenants_dir.display(),
+            )
+        })?
+        .iter()
+        .next()
+        .with_context(|| format!("Found no tenant id in path '{}'", path.display()))?
+        .to_string_lossy()
+        .parse::<ZTenantId>()
+        .with_context(|| format!("Failed to parse tenant id from path '{}'", path.display()))?;
+
+    let timelines_path = conf.timelines_path(&tenant_id);
+    match path.strip_prefix(&timelines_path) {
+        Ok(timelines_subpath) => {
+            let mut segments = timelines_subpath.iter();
+            let timeline_id = segments
+                .next()
+                .with_context(|| {
+                    format!(
+                        "{} directory of tenant {} (path '{}') is not an index entry",
+                        TIMELINES_SEGMENT_NAME,
+                        tenant_id,
+                        path.display()
+                    )
+                })?
+                .to_string_lossy()
+                .parse::<ZTimelineId>()
+                .with_context(|| {
+                    format!("Failed to parse timeline id from path '{}'", path.display())
+                })?;
+
+            let (disk_consistent_lsn, header_size) =
+                parse_archive_name(path).with_context(|| {
+                    format!(
+                        "Failed to parse archive name out in path '{}'",
+                        path.display()
+                    )
+                })?;
+
+            let archive_name = path
+                .file_name()
+                .with_context(|| format!("Archive '{}' has no file name", path.display()))?
+                .to_string_lossy()
+                .to_string();
+
+            let sync_id = ZTenantTimelineId {
+                tenant_id,
+                timeline_id,
+            };
+            let timeline_index_entry = index.timeline_entries.entry(sync_id).or_insert_with(|| {
+                TimelineIndexEntry::new(
+                    TimelineIndexEntryInner::Description(BTreeMap::default()),
+                    false,
+                )
+            });
+            match timeline_index_entry.inner_mut() {
+                TimelineIndexEntryInner::Description(description) => {
+                    description.insert(
+                        ArchiveId(disk_consistent_lsn),
+                        ArchiveDescription {
+                            header_size,
+                            disk_consistent_lsn,
+                            archive_name,
+                        },
+                    );
+                }
+                TimelineIndexEntryInner::Full(_) => {
+                    bail!("Cannot add parsed archive description to its full context in index with sync id {}", sync_id)
+                }
+            }
+        }
+        Err(timelines_strip_error) => {
+            bail!(
+                "Path '{}' is not an archive entry '{}'",
+                path.display(),
+                timelines_strip_error,
+            )
         }
     }
-
-    pub fn missing_files(&self) -> &HashSet<RelativePath> {
-        &self.missing_layers
-    }
-
-    pub fn from_remote_timeline(
-        timeline_path: &Path,
-        remote_timeline: RemoteTimeline,
-    ) -> anyhow::Result<Self> {
-        let metadata_bytes = remote_timeline.metadata.to_bytes()?;
-        Ok(Self {
-            timeline_layers: to_relative_paths(timeline_path, remote_timeline.timeline_layers)
-                .context("Failed to convert timeline layers' paths to relative ones")?,
-            missing_layers: to_relative_paths(timeline_path, remote_timeline.missing_layers)
-                .context("Failed to convert missing layers' paths to relative ones")?,
-            disk_consistent_lsn: remote_timeline.metadata.disk_consistent_lsn(),
-            metadata_bytes,
-        })
-    }
-}
-
-fn to_local_paths(
-    timeline_path: &Path,
-    paths: impl IntoIterator<Item = RelativePath>,
-) -> HashSet<PathBuf> {
-    paths
-        .into_iter()
-        .map(|path| path.as_path(timeline_path))
-        .collect()
-}
-
-fn to_relative_paths(
-    timeline_path: &Path,
-    paths: impl IntoIterator<Item = PathBuf>,
-) -> anyhow::Result<HashSet<RelativePath>> {
-    paths
-        .into_iter()
-        .map(|path| RelativePath::new(timeline_path, path))
-        .collect()
+    Ok(())
 }
 
 #[cfg(test)]
 mod tests {
-    use std::collections::BTreeSet;
-
     use super::*;
-    use crate::repository::repo_harness::{RepoHarness, TIMELINE_ID};
 
     #[test]
-    fn index_part_conversion() {
-        let harness = RepoHarness::create("index_part_conversion").unwrap();
-        let timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let metadata =
-            TimelineMetadata::new(Lsn(5).align(), Some(Lsn(4)), None, Lsn(3), Lsn(2), Lsn(1));
-        let remote_timeline = RemoteTimeline {
-            timeline_layers: HashSet::from([
-                timeline_path.join("layer_1"),
-                timeline_path.join("layer_2"),
-            ]),
-            missing_layers: HashSet::from([
-                timeline_path.join("missing_1"),
-                timeline_path.join("missing_2"),
-            ]),
-            metadata: metadata.clone(),
-            awaits_download: false,
+    fn header_restoration_preserves_file_order() {
+        let header = ArchiveHeader {
+            files: vec![
+                FileEntry {
+                    size: 5,
+                    subpath: RelativePath("one".to_string()),
+                },
+                FileEntry {
+                    size: 1,
+                    subpath: RelativePath("two".to_string()),
+                },
+                FileEntry {
+                    size: 222,
+                    subpath: RelativePath("zero".to_string()),
+                },
+            ],
+            metadata_file_size: 5,
         };
 
-        let index_part = IndexPart::from_remote_timeline(&timeline_path, remote_timeline.clone())
-            .expect("Correct remote timeline should be convertable to index part");
+        let lsn = Lsn(1);
+        let mut remote_timeline = RemoteTimeline::empty();
+        remote_timeline.update_archive_contents(lsn, header.clone(), 15);
+
+        let (restored_header, _) = remote_timeline
+            .restore_header(ArchiveId(lsn))
+            .expect("Should be able to restore header from a valid remote timeline");
 
         assert_eq!(
-            index_part.timeline_layers.iter().collect::<BTreeSet<_>>(),
-            BTreeSet::from([
-                &RelativePath("layer_1".to_string()),
-                &RelativePath("layer_2".to_string())
-            ]),
-            "Index part should have all remote timeline layers after the conversion"
+            header, restored_header,
+            "Header restoration should preserve file order"
         );
-        assert_eq!(
-            index_part.missing_layers.iter().collect::<BTreeSet<_>>(),
-            BTreeSet::from([
-                &RelativePath("missing_1".to_string()),
-                &RelativePath("missing_2".to_string())
-            ]),
-            "Index part should have all missing remote timeline layers after the conversion"
-        );
-        assert_eq!(
-            index_part.disk_consistent_lsn,
-            metadata.disk_consistent_lsn(),
-            "Index part should have disk consistent lsn from the timeline"
-        );
-        assert_eq!(
-            index_part.metadata_bytes,
-            metadata
-                .to_bytes()
-                .expect("Failed to serialize correct metadata into bytes"),
-            "Index part should have all missing remote timeline layers after the conversion"
-        );
-
-        let restored_timeline = RemoteTimeline::from_index_part(&timeline_path, index_part)
-            .expect("Correct index part should be convertable to remote timeline");
-
-        let original_metadata = &remote_timeline.metadata;
-        let restored_metadata = &restored_timeline.metadata;
-        // we have to compare the metadata this way, since its header is different after creation and restoration,
-        // but that is now consireded ok.
-        assert_eq!(
-            original_metadata.disk_consistent_lsn(),
-            restored_metadata.disk_consistent_lsn(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-        assert_eq!(
-            original_metadata.prev_record_lsn(),
-            restored_metadata.prev_record_lsn(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-        assert_eq!(
-            original_metadata.ancestor_timeline(),
-            restored_metadata.ancestor_timeline(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-        assert_eq!(
-            original_metadata.ancestor_lsn(),
-            restored_metadata.ancestor_lsn(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-        assert_eq!(
-            original_metadata.latest_gc_cutoff_lsn(),
-            restored_metadata.latest_gc_cutoff_lsn(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-        assert_eq!(
-            original_metadata.initdb_lsn(),
-            restored_metadata.initdb_lsn(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-
-        assert_eq!(
-            remote_timeline.awaits_download, restored_timeline.awaits_download,
-            "remote timeline -> index part -> remote timeline conversion should not loose download flag"
-        );
-
-        assert_eq!(
-            remote_timeline
-                .timeline_layers
-                .into_iter()
-                .collect::<BTreeSet<_>>(),
-            restored_timeline
-                .timeline_layers
-                .into_iter()
-                .collect::<BTreeSet<_>>(),
-            "remote timeline -> index part -> remote timeline conversion should not loose layer data"
-        );
-        assert_eq!(
-            remote_timeline
-                .missing_layers
-                .into_iter()
-                .collect::<BTreeSet<_>>(),
-            restored_timeline
-                .missing_layers
-                .into_iter()
-                .collect::<BTreeSet<_>>(),
-            "remote timeline -> index part -> remote timeline conversion should not loose missing file data"
-        );
-    }
-
-    #[test]
-    fn index_part_conversion_negatives() {
-        let harness = RepoHarness::create("index_part_conversion_negatives").unwrap();
-        let timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let metadata =
-            TimelineMetadata::new(Lsn(5).align(), Some(Lsn(4)), None, Lsn(3), Lsn(2), Lsn(1));
-
-        let conversion_result = IndexPart::from_remote_timeline(
-            &timeline_path,
-            RemoteTimeline {
-                timeline_layers: HashSet::from([
-                    PathBuf::from("bad_path"),
-                    timeline_path.join("layer_2"),
-                ]),
-                missing_layers: HashSet::from([
-                    timeline_path.join("missing_1"),
-                    timeline_path.join("missing_2"),
-                ]),
-                metadata: metadata.clone(),
-                awaits_download: false,
-            },
-        );
-        assert!(conversion_result.is_err(), "Should not be able to convert metadata with layer paths that are not in the timeline directory");
-
-        let conversion_result = IndexPart::from_remote_timeline(
-            &timeline_path,
-            RemoteTimeline {
-                timeline_layers: HashSet::from([
-                    timeline_path.join("layer_1"),
-                    timeline_path.join("layer_2"),
-                ]),
-                missing_layers: HashSet::from([
-                    PathBuf::from("bad_path"),
-                    timeline_path.join("missing_2"),
-                ]),
-                metadata,
-                awaits_download: false,
-            },
-        );
-        assert!(conversion_result.is_err(), "Should not be able to convert metadata with missing layer paths that are not in the timeline directory");
     }
 }

pageserver/src/remote_storage/storage_sync/upload.rs

@@ -1,452 +1,512 @@
 //! Timeline synchronization logic to compress and upload to the remote storage all new timeline files from the checkpoints.
 
-use std::{fmt::Debug, path::PathBuf};
+use std::{borrow::Cow, collections::BTreeSet, path::PathBuf, sync::Arc};
 
-use anyhow::Context;
-use futures::stream::{FuturesUnordered, StreamExt};
-use tokio::fs;
-use tracing::{debug, error, info, warn};
+use tracing::{debug, error, warn};
 
 use crate::{
     config::PageServerConf,
-    layered_repository::metadata::metadata_path,
     remote_storage::{
-        storage_sync::{index::RemoteTimeline, sync_queue, SyncTask},
-        RemoteStorage,
+        storage_sync::{
+            compression, fetch_full_index,
+            index::{RemoteTimeline, TimelineIndexEntry, TimelineIndexEntryInner},
+            sync_queue, SyncKind, SyncTask,
+        },
+        RemoteStorage, ZTenantTimelineId,
     },
 };
-use utils::zid::ZTenantTimelineId;
 
-use super::{index::IndexPart, SyncData, TimelineUpload};
+use super::{compression::ArchiveHeader, NewCheckpoint, RemoteIndex};
 
-/// Serializes and uploads the given index part data to the remote storage.
-pub(super) async fn upload_index_part<P, S>(
-    conf: &'static PageServerConf,
-    storage: &S,
-    sync_id: ZTenantTimelineId,
-    index_part: IndexPart,
-) -> anyhow::Result<()>
-where
-    P: Debug + Send + Sync + 'static,
-    S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
-{
-    let index_part_bytes = serde_json::to_vec(&index_part)
-        .context("Failed to serialize index part file into bytes")?;
-    let index_part_size = index_part_bytes.len();
-    let index_part_bytes = tokio::io::BufReader::new(std::io::Cursor::new(index_part_bytes));
-
-    let index_part_path = metadata_path(conf, sync_id.timeline_id, sync_id.tenant_id)
-        .with_file_name(IndexPart::FILE_NAME)
-        .with_extension(IndexPart::FILE_EXTENSION);
-    let index_part_storage_path = storage.storage_path(&index_part_path).with_context(|| {
-        format!(
-            "Failed to get the index part storage path for local path '{}'",
-            index_part_path.display()
-        )
-    })?;
-
-    storage
-        .upload(
-            index_part_bytes,
-            index_part_size,
-            &index_part_storage_path,
-            None,
-        )
-        .await
-        .with_context(|| {
-            format!("Failed to upload index part to the storage path '{index_part_storage_path:?}'")
-        })
-}
-
-/// Timeline upload result, with extra data, needed for uploading.
-#[derive(Debug)]
-pub(super) enum UploadedTimeline {
-    /// Upload failed due to some error, the upload task is rescheduled for another retry.
-    FailedAndRescheduled,
-    /// No issues happened during the upload, all task files were put into the remote storage.
-    Successful(SyncData<TimelineUpload>),
-    /// No failures happened during the upload, but some files were removed locally before the upload task completed
-    /// (could happen due to retries, for instance, if GC happens in the interim).
-    /// Such files are considered "not needed" and ignored, but the task's metadata should be discarded and the new one loaded from the local file.
-    SuccessfulAfterLocalFsUpdate(SyncData<TimelineUpload>),
-}
-
-/// Attempts to upload given layer files.
-/// No extra checks for overlapping files is made and any files that are already present remotely will be overwritten, if submitted during the upload.
+/// Attempts to compress and upload given checkpoint files.
+/// No extra checks for overlapping files is made: download takes care of that, ensuring no non-metadata local timeline files are overwritten.
 ///
 /// On an error, bumps the retries count and reschedules the entire task.
-pub(super) async fn upload_timeline_layers<'a, P, S>(
-    storage: &'a S,
-    remote_timeline: Option<&'a RemoteTimeline>,
-    sync_id: ZTenantTimelineId,
-    mut upload_data: SyncData<TimelineUpload>,
-) -> UploadedTimeline
-where
-    P: Debug + Send + Sync + 'static,
+/// On success, populates index data with new downloads.
+pub(super) async fn upload_timeline_checkpoint<
+    P: std::fmt::Debug + Send + Sync + 'static,
     S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
-{
-    let upload = &mut upload_data.data;
-    let new_upload_lsn = upload.metadata.disk_consistent_lsn();
+>(
+    config: &'static PageServerConf,
+    remote_assets: Arc<(S, RemoteIndex)>,
+    sync_id: ZTenantTimelineId,
+    new_checkpoint: NewCheckpoint,
+    retries: u32,
+) -> Option<bool> {
+    debug!("Uploading checkpoint for sync id {}", sync_id);
+    let new_upload_lsn = new_checkpoint.metadata.disk_consistent_lsn();
 
-    let already_uploaded_layers = remote_timeline
-        .map(|timeline| timeline.stored_files())
-        .cloned()
-        .unwrap_or_default();
+    let index = &remote_assets.1;
 
-    let layers_to_upload = upload
-        .layers_to_upload
-        .difference(&already_uploaded_layers)
-        .cloned()
-        .collect::<Vec<_>>();
+    let ZTenantTimelineId {
+        tenant_id,
+        timeline_id,
+    } = sync_id;
+    let timeline_dir = config.timeline_path(&timeline_id, &tenant_id);
 
-    debug!("Layers to upload: {layers_to_upload:?}");
-    info!(
-        "Uploading {} timeline layers, new lsn: {new_upload_lsn}",
-        layers_to_upload.len(),
-    );
-
-    let mut upload_tasks = layers_to_upload
-        .into_iter()
-        .map(|source_path| async move {
-            let storage_path = storage
-                .storage_path(&source_path)
-                .with_context(|| {
-                    format!(
-                        "Failed to get the layer storage path for local path '{}'",
-                        source_path.display()
-                    )
-                })
-                .map_err(UploadError::Other)?;
-
-            let source_file = match fs::File::open(&source_path).await.with_context(|| {
-                format!(
-                    "Failed to upen a source file for layer '{}'",
-                    source_path.display()
-                )
-            }) {
-                Ok(file) => file,
-                Err(e) => return Err(UploadError::MissingLocalFile(source_path, e)),
-            };
-
-            let source_size = source_file
-                .metadata()
-                .await
-                .with_context(|| {
-                    format!(
-                        "Failed to get the source file metadata for layer '{}'",
-                        source_path.display()
-                    )
-                })
-                .map_err(UploadError::Other)?
-                .len() as usize;
-
-            match storage
-                .upload(source_file, source_size, &storage_path, None)
-                .await
-                .with_context(|| {
-                    format!(
-                        "Failed to upload a layer from local path '{}'",
-                        source_path.display()
-                    )
-                }) {
-                Ok(()) => Ok(source_path),
-                Err(e) => Err(UploadError::MissingLocalFile(source_path, e)),
-            }
-        })
-        .collect::<FuturesUnordered<_>>();
-
-    let mut errors_happened = false;
-    let mut local_fs_updated = false;
-    while let Some(upload_result) = upload_tasks.next().await {
-        match upload_result {
-            Ok(uploaded_path) => {
-                upload.layers_to_upload.remove(&uploaded_path);
-                upload.uploaded_layers.insert(uploaded_path);
-            }
-            Err(e) => match e {
-                UploadError::Other(e) => {
-                    errors_happened = true;
-                    error!("Failed to upload a layer for timeline {sync_id}: {e:?}");
-                }
-                UploadError::MissingLocalFile(source_path, e) => {
-                    if source_path.exists() {
-                        errors_happened = true;
-                        error!("Failed to upload a layer for timeline {sync_id}: {e:?}");
-                    } else {
-                        local_fs_updated = true;
-                        upload.layers_to_upload.remove(&source_path);
-                        warn!(
-                            "Missing locally a layer file {} scheduled for upload, skipping",
-                            source_path.display()
-                        );
+    let index_read = index.read().await;
+    let remote_timeline = match index_read.timeline_entry(&sync_id) {
+        None => None,
+        Some(entry) => match entry.inner() {
+            TimelineIndexEntryInner::Full(remote_timeline) => Some(Cow::Borrowed(remote_timeline)),
+            TimelineIndexEntryInner::Description(_) => {
+                debug!("Found timeline description for the given ids, downloading the full index");
+                match fetch_full_index(remote_assets.as_ref(), &timeline_dir, sync_id).await {
+                    Ok(remote_timeline) => Some(Cow::Owned(remote_timeline)),
+                    Err(e) => {
+                        error!("Failed to download full timeline index: {:?}", e);
+                        sync_queue::push(SyncTask::new(
+                            sync_id,
+                            retries,
+                            SyncKind::Upload(new_checkpoint),
+                        ));
+                        return Some(false);
                     }
                 }
-            },
-        }
+            }
+        },
+    };
+
+    let already_contains_upload_lsn = remote_timeline
+        .as_ref()
+        .map(|remote_timeline| remote_timeline.contains_checkpoint_at(new_upload_lsn))
+        .unwrap_or(false);
+    if already_contains_upload_lsn {
+        warn!(
+            "Received a checkpoint with Lsn {} that's already been uploaded to remote storage, skipping the upload.",
+            new_upload_lsn
+        );
+        return None;
     }
 
-    if errors_happened {
-        debug!("Reenqueuing failed upload task for timeline {sync_id}");
-        upload_data.retries += 1;
-        sync_queue::push(sync_id, SyncTask::Upload(upload_data));
-        UploadedTimeline::FailedAndRescheduled
-    } else if local_fs_updated {
-        info!("Successfully uploaded all layers, some local layers were removed during the upload");
-        UploadedTimeline::SuccessfulAfterLocalFsUpdate(upload_data)
-    } else {
-        info!("Successfully uploaded all layers");
-        UploadedTimeline::Successful(upload_data)
+    let already_uploaded_files = remote_timeline
+        .map(|timeline| timeline.stored_files(&timeline_dir))
+        .unwrap_or_default();
+    drop(index_read);
+
+    match try_upload_checkpoint(
+        config,
+        Arc::clone(&remote_assets),
+        sync_id,
+        &new_checkpoint,
+        already_uploaded_files,
+    )
+    .await
+    {
+        Some(Ok((archive_header, header_size))) => {
+            let mut index_write = index.write().await;
+            match index_write
+                .timeline_entry_mut(&sync_id)
+                .map(|e| e.inner_mut())
+            {
+                None => {
+                    let mut new_timeline = RemoteTimeline::empty();
+                    new_timeline.update_archive_contents(
+                        new_checkpoint.metadata.disk_consistent_lsn(),
+                        archive_header,
+                        header_size,
+                    );
+                    index_write.add_timeline_entry(
+                        sync_id,
+                        TimelineIndexEntry::new(TimelineIndexEntryInner::Full(new_timeline), false),
+                    )
+                }
+                Some(TimelineIndexEntryInner::Full(remote_timeline)) => {
+                    remote_timeline.update_archive_contents(
+                        new_checkpoint.metadata.disk_consistent_lsn(),
+                        archive_header,
+                        header_size,
+                    );
+                }
+                Some(TimelineIndexEntryInner::Description(_)) => {
+                    let mut new_timeline = RemoteTimeline::empty();
+                    new_timeline.update_archive_contents(
+                        new_checkpoint.metadata.disk_consistent_lsn(),
+                        archive_header,
+                        header_size,
+                    );
+                    index_write.add_timeline_entry(
+                        sync_id,
+                        TimelineIndexEntry::new(TimelineIndexEntryInner::Full(new_timeline), false),
+                    )
+                }
+            }
+            debug!("Checkpoint uploaded successfully");
+            Some(true)
+        }
+        Some(Err(e)) => {
+            error!(
+                "Failed to upload checkpoint: {:?}, requeueing the upload",
+                e
+            );
+            sync_queue::push(SyncTask::new(
+                sync_id,
+                retries,
+                SyncKind::Upload(new_checkpoint),
+            ));
+            Some(false)
+        }
+        None => Some(true),
     }
 }
 
-enum UploadError {
-    MissingLocalFile(PathBuf, anyhow::Error),
-    Other(anyhow::Error),
+async fn try_upload_checkpoint<
+    P: Send + Sync + 'static,
+    S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
+>(
+    config: &'static PageServerConf,
+    remote_assets: Arc<(S, RemoteIndex)>,
+    sync_id: ZTenantTimelineId,
+    new_checkpoint: &NewCheckpoint,
+    files_to_skip: BTreeSet<PathBuf>,
+) -> Option<anyhow::Result<(ArchiveHeader, u64)>> {
+    let ZTenantTimelineId {
+        tenant_id,
+        timeline_id,
+    } = sync_id;
+    let timeline_dir = config.timeline_path(&timeline_id, &tenant_id);
+
+    let files_to_upload = new_checkpoint
+        .layers
+        .iter()
+        .filter(|&path_to_upload| {
+            if files_to_skip.contains(path_to_upload) {
+                warn!(
+                    "Skipping file upload '{}', since it was already uploaded",
+                    path_to_upload.display()
+                );
+                false
+            } else {
+                true
+            }
+        })
+        .collect::<Vec<_>>();
+
+    if files_to_upload.is_empty() {
+        warn!(
+            "No files to upload. Upload request was: {:?}, already uploaded files: {:?}",
+            new_checkpoint.layers, files_to_skip
+        );
+        return None;
+    }
+
+    let upload_result = compression::archive_files_as_stream(
+        &timeline_dir,
+        files_to_upload.into_iter(),
+        &new_checkpoint.metadata,
+        move |archive_streamer, archive_name| async move {
+            let timeline_dir = config.timeline_path(&timeline_id, &tenant_id);
+            let remote_storage = &remote_assets.0;
+            remote_storage
+                .upload(
+                    archive_streamer,
+                    &remote_storage.storage_path(&timeline_dir.join(&archive_name))?,
+                )
+                .await
+        },
+    )
+    .await
+    .map(|(header, header_size, _)| (header, header_size));
+
+    Some(upload_result)
 }
 
 #[cfg(test)]
 mod tests {
-    use std::collections::{BTreeSet, HashSet};
-
     use tempfile::tempdir;
-    use utils::lsn::Lsn;
+    use zenith_utils::lsn::Lsn;
 
     use crate::{
         remote_storage::{
+            local_fs::LocalFs,
             storage_sync::{
-                index::RelativePath,
-                test_utils::{create_local_timeline, dummy_metadata},
+                index::ArchiveId,
+                test_utils::{
+                    assert_index_descriptions, create_local_timeline, dummy_metadata,
+                    ensure_correct_timeline_upload, expect_timeline,
+                },
             },
-            LocalFs,
         },
         repository::repo_harness::{RepoHarness, TIMELINE_ID},
     };
 
-    use super::{upload_index_part, *};
+    use super::*;
 
     #[tokio::test]
-    async fn regular_layer_upload() -> anyhow::Result<()> {
-        let harness = RepoHarness::create("regular_layer_upload")?;
-        let sync_id = ZTenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-
-        let layer_files = ["a", "b"];
-        let storage = LocalFs::new(tempdir()?.path().to_path_buf(), &harness.conf.workdir)?;
-        let current_retries = 3;
-        let metadata = dummy_metadata(Lsn(0x30));
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let timeline_upload =
-            create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?;
-        assert!(
-            storage.list().await?.is_empty(),
-            "Storage should be empty before any uploads are made"
+    async fn reupload_timeline() -> anyhow::Result<()> {
+        let repo_harness = RepoHarness::create("reupload_timeline")?;
+        let sync_id = ZTenantTimelineId::new(repo_harness.tenant_id, TIMELINE_ID);
+        let storage = LocalFs::new(tempdir()?.path().to_owned(), &repo_harness.conf.workdir)?;
+        let index = RemoteIndex::try_parse_descriptions_from_paths(
+            repo_harness.conf,
+            storage
+                .list()
+                .await?
+                .into_iter()
+                .map(|storage_path| storage.local_path(&storage_path).unwrap()),
         );
+        let remote_assets = Arc::new((storage, index));
+        let index = &remote_assets.1;
 
-        let upload_result = upload_timeline_layers(
-            &storage,
-            None,
-            sync_id,
-            SyncData::new(current_retries, timeline_upload.clone()),
+        let first_upload_metadata = dummy_metadata(Lsn(0x10));
+        let first_checkpoint = create_local_timeline(
+            &repo_harness,
+            TIMELINE_ID,
+            &["a", "b"],
+            first_upload_metadata.clone(),
+        )?;
+        let local_timeline_path = repo_harness.timeline_path(&TIMELINE_ID);
+        ensure_correct_timeline_upload(
+            &repo_harness,
+            Arc::clone(&remote_assets),
+            TIMELINE_ID,
+            first_checkpoint,
         )
         .await;
 
-        let upload_data = match upload_result {
-            UploadedTimeline::Successful(upload_data) => upload_data,
-            wrong_result => {
-                panic!("Expected a successful upload for timeline, but got: {wrong_result:?}")
-            }
-        };
-
+        let uploaded_timeline = expect_timeline(index, sync_id).await;
+        let uploaded_archives = uploaded_timeline
+            .checkpoints()
+            .map(ArchiveId)
+            .collect::<Vec<_>>();
         assert_eq!(
-            current_retries, upload_data.retries,
-            "On successful upload, retries are not expected to change"
-        );
-        let upload = &upload_data.data;
-        assert!(
-            upload.layers_to_upload.is_empty(),
-            "Successful upload should have no layers left to upload"
-        );
-        assert_eq!(
-            upload
-                .uploaded_layers
-                .iter()
-                .cloned()
-                .collect::<BTreeSet<_>>(),
-            layer_files
-                .iter()
-                .map(|layer_file| local_timeline_path.join(layer_file))
-                .collect(),
-            "Successful upload should have all layers uploaded"
-        );
-        assert_eq!(
-            upload.metadata, metadata,
-            "Successful upload should not chage its metadata"
-        );
-
-        let storage_files = storage.list().await?;
-        assert_eq!(
-            storage_files.len(),
-            layer_files.len(),
-            "All layers should be uploaded"
-        );
-        assert_eq!(
-            storage_files
-                .into_iter()
-                .map(|storage_path| storage.local_path(&storage_path))
-                .collect::<anyhow::Result<BTreeSet<_>>>()?,
-            layer_files
-                .into_iter()
-                .map(|file| local_timeline_path.join(file))
-                .collect(),
-            "Uploaded files should match with the local ones"
-        );
-
-        Ok(())
-    }
-
-    // Currently, GC can run between upload retries, removing local layers scheduled for upload. Test this scenario.
-    #[tokio::test]
-    async fn layer_upload_after_local_fs_update() -> anyhow::Result<()> {
-        let harness = RepoHarness::create("layer_upload_after_local_fs_update")?;
-        let sync_id = ZTenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-
-        let layer_files = ["a1", "b1"];
-        let storage = LocalFs::new(tempdir()?.path().to_owned(), &harness.conf.workdir)?;
-        let current_retries = 5;
-        let metadata = dummy_metadata(Lsn(0x40));
-
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let layers_to_upload = {
-            let mut layers = layer_files.to_vec();
-            layers.push("layer_to_remove");
-            layers
-        };
-        let timeline_upload =
-            create_local_timeline(&harness, TIMELINE_ID, &layers_to_upload, metadata.clone())
-                .await?;
-        assert!(
-            storage.list().await?.is_empty(),
-            "Storage should be empty before any uploads are made"
-        );
-
-        fs::remove_file(local_timeline_path.join("layer_to_remove")).await?;
-
-        let upload_result = upload_timeline_layers(
-            &storage,
-            None,
-            sync_id,
-            SyncData::new(current_retries, timeline_upload.clone()),
-        )
-        .await;
-
-        let upload_data = match upload_result {
-            UploadedTimeline::SuccessfulAfterLocalFsUpdate(upload_data) => upload_data,
-            wrong_result => panic!(
-                "Expected a successful after local fs upload for timeline, but got: {wrong_result:?}"
-            ),
-        };
-
-        assert_eq!(
-            current_retries, upload_data.retries,
-            "On successful upload, retries are not expected to change"
-        );
-        let upload = &upload_data.data;
-        assert!(
-                upload.layers_to_upload.is_empty(),
-                "Successful upload should have no layers left to upload, even those that were removed from the local fs"
-            );
-        assert_eq!(
-            upload
-                .uploaded_layers
-                .iter()
-                .cloned()
-                .collect::<BTreeSet<_>>(),
-            layer_files
-                .iter()
-                .map(|layer_file| local_timeline_path.join(layer_file))
-                .collect(),
-            "Successful upload should have all layers uploaded"
-        );
-        assert_eq!(
-            upload.metadata, metadata,
-            "Successful upload should not chage its metadata"
-        );
-
-        let storage_files = storage.list().await?;
-        assert_eq!(
-            storage_files.len(),
-            layer_files.len(),
-            "All layers should be uploaded"
-        );
-        assert_eq!(
-            storage_files
-                .into_iter()
-                .map(|storage_path| storage.local_path(&storage_path))
-                .collect::<anyhow::Result<BTreeSet<_>>>()?,
-            layer_files
-                .into_iter()
-                .map(|file| local_timeline_path.join(file))
-                .collect(),
-            "Uploaded files should match with the local ones"
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_upload_index_part() -> anyhow::Result<()> {
-        let harness = RepoHarness::create("test_upload_index_part")?;
-        let sync_id = ZTenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-
-        let storage = LocalFs::new(tempdir()?.path().to_owned(), &harness.conf.workdir)?;
-        let metadata = dummy_metadata(Lsn(0x40));
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-
-        let index_part = IndexPart::new(
-            HashSet::from([
-                RelativePath::new(&local_timeline_path, local_timeline_path.join("one"))?,
-                RelativePath::new(&local_timeline_path, local_timeline_path.join("two"))?,
-            ]),
-            HashSet::from([RelativePath::new(
-                &local_timeline_path,
-                local_timeline_path.join("three"),
-            )?]),
-            metadata.disk_consistent_lsn(),
-            metadata.to_bytes()?,
-        );
-
-        assert!(
-            storage.list().await?.is_empty(),
-            "Storage should be empty before any uploads are made"
-        );
-        upload_index_part(harness.conf, &storage, sync_id, index_part.clone()).await?;
-
-        let storage_files = storage.list().await?;
-        assert_eq!(
-            storage_files.len(),
+            uploaded_archives.len(),
             1,
-            "Should have only the index part file uploaded"
+            "Only one archive is expected after a first upload"
+        );
+        let first_uploaded_archive = uploaded_archives.first().copied().unwrap();
+        assert_eq!(
+            uploaded_timeline.checkpoints().last(),
+            Some(first_upload_metadata.disk_consistent_lsn()),
+            "Metadata that was uploaded, should have its Lsn stored"
+        );
+        assert_eq!(
+            uploaded_timeline
+                .archive_data(uploaded_archives.first().copied().unwrap())
+                .unwrap()
+                .disk_consistent_lsn(),
+            first_upload_metadata.disk_consistent_lsn(),
+            "Uploaded archive should have corresponding Lsn"
+        );
+        assert_eq!(
+            uploaded_timeline.stored_files(&local_timeline_path),
+            vec![local_timeline_path.join("a"), local_timeline_path.join("b")]
+                .into_iter()
+                .collect(),
+            "Should have all files from the first checkpoint"
         );
 
-        let index_part_path = storage_files.first().unwrap();
+        let second_upload_metadata = dummy_metadata(Lsn(0x40));
+        let second_checkpoint = create_local_timeline(
+            &repo_harness,
+            TIMELINE_ID,
+            &["b", "c"],
+            second_upload_metadata.clone(),
+        )?;
+        assert!(
+            first_upload_metadata.disk_consistent_lsn()
+                < second_upload_metadata.disk_consistent_lsn()
+        );
+        ensure_correct_timeline_upload(
+            &repo_harness,
+            Arc::clone(&remote_assets),
+            TIMELINE_ID,
+            second_checkpoint,
+        )
+        .await;
+
+        let updated_timeline = expect_timeline(index, sync_id).await;
+        let mut updated_archives = updated_timeline
+            .checkpoints()
+            .map(ArchiveId)
+            .collect::<Vec<_>>();
         assert_eq!(
-            index_part_path.file_stem().and_then(|name| name.to_str()),
-            Some(IndexPart::FILE_NAME),
-            "Remote index part should have the correct name"
+            updated_archives.len(),
+            2,
+            "Two archives are expected after a successful update of the upload"
+        );
+        updated_archives.retain(|archive_id| archive_id != &first_uploaded_archive);
+        assert_eq!(
+            updated_archives.len(),
+            1,
+            "Only one new archive is expected among the uploaded"
+        );
+        let second_uploaded_archive = updated_archives.last().copied().unwrap();
+        assert_eq!(
+            updated_timeline.checkpoints().max(),
+            Some(second_upload_metadata.disk_consistent_lsn()),
+            "Metadata that was uploaded, should have its Lsn stored"
         );
         assert_eq!(
-            index_part_path
-                .extension()
-                .and_then(|extension| extension.to_str()),
-            Some(IndexPart::FILE_EXTENSION),
-            "Remote index part should have the correct extension"
+            updated_timeline
+                .archive_data(second_uploaded_archive)
+                .unwrap()
+                .disk_consistent_lsn(),
+            second_upload_metadata.disk_consistent_lsn(),
+            "Uploaded archive should have corresponding Lsn"
+        );
+        assert_eq!(
+            updated_timeline.stored_files(&local_timeline_path),
+            vec![
+                local_timeline_path.join("a"),
+                local_timeline_path.join("b"),
+                local_timeline_path.join("c"),
+            ]
+            .into_iter()
+            .collect(),
+            "Should have all files from both checkpoints without duplicates"
         );
 
-        let remote_index_part: IndexPart =
-            serde_json::from_slice(&fs::read(&index_part_path).await?)?;
-        assert_eq!(
-            index_part, remote_index_part,
-            "Remote index part should match the local one"
+        let third_upload_metadata = dummy_metadata(Lsn(0x20));
+        let third_checkpoint = create_local_timeline(
+            &repo_harness,
+            TIMELINE_ID,
+            &["d"],
+            third_upload_metadata.clone(),
+        )?;
+        assert_ne!(
+            third_upload_metadata.disk_consistent_lsn(),
+            first_upload_metadata.disk_consistent_lsn()
         );
+        assert!(
+            third_upload_metadata.disk_consistent_lsn()
+                < second_upload_metadata.disk_consistent_lsn()
+        );
+        ensure_correct_timeline_upload(
+            &repo_harness,
+            Arc::clone(&remote_assets),
+            TIMELINE_ID,
+            third_checkpoint,
+        )
+        .await;
+
+        let updated_timeline = expect_timeline(index, sync_id).await;
+        let mut updated_archives = updated_timeline
+            .checkpoints()
+            .map(ArchiveId)
+            .collect::<Vec<_>>();
+        assert_eq!(
+            updated_archives.len(),
+            3,
+            "Three archives are expected after two successful updates of the upload"
+        );
+        updated_archives.retain(|archive_id| {
+            archive_id != &first_uploaded_archive && archive_id != &second_uploaded_archive
+        });
+        assert_eq!(
+            updated_archives.len(),
+            1,
+            "Only one new archive is expected among the uploaded"
+        );
+        let third_uploaded_archive = updated_archives.last().copied().unwrap();
+        assert!(
+            updated_timeline.checkpoints().max().unwrap()
+                > third_upload_metadata.disk_consistent_lsn(),
+            "Should not influence the last lsn by uploading an older checkpoint"
+        );
+        assert_eq!(
+            updated_timeline
+                .archive_data(third_uploaded_archive)
+                .unwrap()
+                .disk_consistent_lsn(),
+            third_upload_metadata.disk_consistent_lsn(),
+            "Uploaded archive should have corresponding Lsn"
+        );
+        assert_eq!(
+            updated_timeline.stored_files(&local_timeline_path),
+            vec![
+                local_timeline_path.join("a"),
+                local_timeline_path.join("b"),
+                local_timeline_path.join("c"),
+                local_timeline_path.join("d"),
+            ]
+            .into_iter()
+            .collect(),
+            "Should have all files from three checkpoints without duplicates"
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn reupload_timeline_rejected() -> anyhow::Result<()> {
+        let repo_harness = RepoHarness::create("reupload_timeline_rejected")?;
+        let sync_id = ZTenantTimelineId::new(repo_harness.tenant_id, TIMELINE_ID);
+        let storage = LocalFs::new(tempdir()?.path().to_owned(), &repo_harness.conf.workdir)?;
+        let index = RemoteIndex::try_parse_descriptions_from_paths(
+            repo_harness.conf,
+            storage
+                .list()
+                .await?
+                .into_iter()
+                .map(|storage_path| storage.local_path(&storage_path).unwrap()),
+        );
+        let remote_assets = Arc::new((storage, index));
+        let storage = &remote_assets.0;
+        let index = &remote_assets.1;
+
+        let first_upload_metadata = dummy_metadata(Lsn(0x10));
+        let first_checkpoint = create_local_timeline(
+            &repo_harness,
+            TIMELINE_ID,
+            &["a", "b"],
+            first_upload_metadata.clone(),
+        )?;
+        ensure_correct_timeline_upload(
+            &repo_harness,
+            Arc::clone(&remote_assets),
+            TIMELINE_ID,
+            first_checkpoint,
+        )
+        .await;
+        let after_first_uploads = RemoteIndex::try_parse_descriptions_from_paths(
+            repo_harness.conf,
+            remote_assets
+                .0
+                .list()
+                .await
+                .unwrap()
+                .into_iter()
+                .map(|storage_path| storage.local_path(&storage_path).unwrap()),
+        );
+
+        let normal_upload_metadata = dummy_metadata(Lsn(0x20));
+        assert_ne!(
+            normal_upload_metadata.disk_consistent_lsn(),
+            first_upload_metadata.disk_consistent_lsn()
+        );
+
+        let checkpoint_with_no_files = create_local_timeline(
+            &repo_harness,
+            TIMELINE_ID,
+            &[],
+            normal_upload_metadata.clone(),
+        )?;
+        upload_timeline_checkpoint(
+            repo_harness.conf,
+            Arc::clone(&remote_assets),
+            sync_id,
+            checkpoint_with_no_files,
+            0,
+        )
+        .await;
+        assert_index_descriptions(index, &after_first_uploads).await;
+
+        let checkpoint_with_uploaded_lsn = create_local_timeline(
+            &repo_harness,
+            TIMELINE_ID,
+            &["something", "new"],
+            first_upload_metadata.clone(),
+        )?;
+        upload_timeline_checkpoint(
+            repo_harness.conf,
+            Arc::clone(&remote_assets),
+            sync_id,
+            checkpoint_with_uploaded_lsn,
+            0,
+        )
+        .await;
+        assert_index_descriptions(index, &after_first_uploads).await;
 
         Ok(())
     }

pageserver/src/repository.rs

@@ -1,5 +1,6 @@
 use crate::layered_repository::metadata::TimelineMetadata;
 use crate::remote_storage::RemoteIndex;
+use crate::tenant_config::TenantConf;
 use crate::walrecord::ZenithWalRecord;
 use crate::CheckpointConfig;
 use anyhow::{bail, Result};
@@ -11,10 +12,8 @@ use std::fmt::Display;
 use std::ops::{AddAssign, Range};
 use std::sync::{Arc, RwLockReadGuard};
 use std::time::Duration;
-use utils::{
-    lsn::{Lsn, RecordLsn},
-    zid::ZTimelineId,
-};
+use zenith_utils::lsn::{Lsn, RecordLsn};
+use zenith_utils::zid::ZTimelineId;
 
 #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)]
 /// Key used in the Repository kv-store.
@@ -184,12 +183,14 @@ impl Value {
 
 #[derive(Clone, Copy, Debug)]
 pub enum TimelineSyncStatusUpdate {
+    Uploaded,
     Downloaded,
 }
 
 impl Display for TimelineSyncStatusUpdate {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let s = match self {
+            TimelineSyncStatusUpdate::Uploaded => "Uploaded",
             TimelineSyncStatusUpdate::Downloaded => "Downloaded",
         };
         f.write_str(s)
@@ -253,10 +254,8 @@ pub trait Repository: Send + Sync {
         checkpoint_before_gc: bool,
     ) -> Result<GcResult>;
 
-    /// Perform one compaction iteration.
-    /// This function is periodically called by compactor thread.
-    /// Also it can be explicitly requested per timeline through page server
-    /// api's 'compact' command.
+    /// perform one compaction iteration.
+    /// this function is periodically called by compactor thread.
     fn compaction_iteration(&self) -> Result<()>;
 
     /// detaches locally available timeline by stopping all threads and removing all the data.
@@ -264,6 +263,8 @@ pub trait Repository: Send + Sync {
 
     // Allows to retrieve remote timeline index from the repo. Used in walreceiver to grab remote consistent lsn.
     fn get_remote_index(&self) -> &RemoteIndex;
+
+    fn get_tenant_conf(&self) -> TenantConf;
 }
 
 /// A timeline, that belongs to the current repository.
@@ -435,9 +436,8 @@ pub mod repo_harness {
     };
 
     use super::*;
-    use crate::tenant_config::{TenantConf, TenantConfOpt};
     use hex_literal::hex;
-    use utils::zid::ZTenantId;
+    use zenith_utils::zid::ZTenantId;
 
     pub const TIMELINE_ID: ZTimelineId =
         ZTimelineId::from_array(hex!("11223344556677881122334455667788"));
@@ -458,20 +458,6 @@ pub mod repo_harness {
         static ref LOCK: RwLock<()> = RwLock::new(());
     }
 
-    impl From<TenantConf> for TenantConfOpt {
-        fn from(tenant_conf: TenantConf) -> Self {
-            Self {
-                checkpoint_distance: Some(tenant_conf.checkpoint_distance),
-                compaction_target_size: Some(tenant_conf.compaction_target_size),
-                compaction_period: Some(tenant_conf.compaction_period),
-                compaction_threshold: Some(tenant_conf.compaction_threshold),
-                gc_horizon: Some(tenant_conf.gc_horizon),
-                gc_period: Some(tenant_conf.gc_period),
-                pitr_interval: Some(tenant_conf.pitr_interval),
-            }
-        }
-    }
-
     pub struct RepoHarness<'a> {
         pub conf: &'static PageServerConf,
         pub tenant_conf: TenantConf,
@@ -529,7 +515,7 @@ pub mod repo_harness {
 
             let repo = LayeredRepository::new(
                 self.conf,
-                TenantConfOpt::from(self.tenant_conf),
+                self.tenant_conf,
                 walredo_mgr,
                 self.tenant_id,
                 RemoteIndex::empty(),

pageserver/src/tenant_config.rs

@@ -9,10 +9,13 @@
 //! may lead to a data loss.
 //!
 use crate::config::PageServerConf;
+use crate::STORAGE_FORMAT_VERSION;
+use anyhow::ensure;
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 use std::time::Duration;
-use utils::zid::ZTenantId;
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::zid::ZTenantId;
 
 pub const TENANT_CONFIG_NAME: &str = "config";
 
@@ -28,99 +31,127 @@ pub mod defaults {
     pub const DEFAULT_COMPACTION_TARGET_SIZE: u64 = 128 * 1024 * 1024;
 
     pub const DEFAULT_COMPACTION_PERIOD: &str = "1 s";
-    pub const DEFAULT_COMPACTION_THRESHOLD: usize = 10;
 
     pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
     pub const DEFAULT_GC_PERIOD: &str = "100 s";
     pub const DEFAULT_PITR_INTERVAL: &str = "30 days";
 }
 
-/// Per-tenant configuration options
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
 pub struct TenantConf {
-    // Flush out an inmemory layer, if it's holding WAL older than this
-    // This puts a backstop on how much WAL needs to be re-digested if the
-    // page server crashes.
-    // This parameter actually determines L0 layer file size.
     pub checkpoint_distance: u64,
-    // Target file size, when creating image and delta layers.
-    // This parameter determines L1 layer file size.
     pub compaction_target_size: u64,
-    // How often to check if there's compaction work to be done.
     pub compaction_period: Duration,
-    // Level0 delta layer threshold for compaction.
-    pub compaction_threshold: usize,
-    // Determines how much history is retained, to allow
-    // branching and read replicas at an older point in time.
-    // The unit is #of bytes of WAL.
-    // Page versions older than this are garbage collected away.
     pub gc_horizon: u64,
-    // Interval at which garbage collection is triggered.
     pub gc_period: Duration,
-    // Determines how much history is retained, to allow
-    // branching and read replicas at an older point in time.
-    // The unit is time.
-    // Page versions older than this are garbage collected away.
     pub pitr_interval: Duration,
 }
 
-/// Same as TenantConf, but this struct preserves the information about
-/// which parameters are set and which are not.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
-pub struct TenantConfOpt {
-    pub checkpoint_distance: Option<u64>,
-    pub compaction_target_size: Option<u64>,
-    pub compaction_period: Option<Duration>,
-    pub compaction_threshold: Option<usize>,
-    pub gc_horizon: Option<u64>,
-    pub gc_period: Option<Duration>,
-    pub pitr_interval: Option<Duration>,
+/// We assume that a write of up to TENANTCONF_MAX_SIZE bytes is atomic.
+///
+/// This is the same assumption that PostgreSQL makes with the control file,
+/// see PG_CONTROL_MAX_SAFE_SIZE
+const TENANTCONF_MAX_SIZE: usize = 512;
+
+/// TenantConfFile is stored on disk in tenant's directory
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TenantConfFile {
+    hdr: TenantConfHeader,
+    pub body: TenantConf,
 }
 
-impl TenantConfOpt {
-    pub fn merge(&self, global_conf: TenantConf) -> TenantConf {
-        TenantConf {
-            checkpoint_distance: self
-                .checkpoint_distance
-                .unwrap_or(global_conf.checkpoint_distance),
-            compaction_target_size: self
-                .compaction_target_size
-                .unwrap_or(global_conf.compaction_target_size),
-            compaction_period: self
-                .compaction_period
-                .unwrap_or(global_conf.compaction_period),
-            compaction_threshold: self
-                .compaction_threshold
-                .unwrap_or(global_conf.compaction_threshold),
-            gc_horizon: self.gc_horizon.unwrap_or(global_conf.gc_horizon),
-            gc_period: self.gc_period.unwrap_or(global_conf.gc_period),
-            pitr_interval: self.pitr_interval.unwrap_or(global_conf.pitr_interval),
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+struct TenantConfHeader {
+    checksum: u32,       // CRC of serialized tenantconf body
+    size: u16,           // size of serialized tenantconf
+    format_version: u16, // storage format version (used for compatibility checks)
+}
+const TENANTCONF_HDR_SIZE: usize = std::mem::size_of::<TenantConfHeader>();
+
+impl TenantConfFile {
+    pub fn new(
+        checkpoint_distance: u64,
+        compaction_target_size: u64,
+        compaction_period: Duration,
+        gc_horizon: u64,
+        gc_period: Duration,
+        pitr_interval: Duration,
+    ) -> Self {
+        Self {
+            hdr: TenantConfHeader {
+                checksum: 0,
+                size: 0,
+                format_version: STORAGE_FORMAT_VERSION,
+            },
+            body: TenantConf {
+                gc_period,
+                gc_horizon,
+                pitr_interval,
+                checkpoint_distance,
+                compaction_period,
+                compaction_target_size,
+            },
         }
     }
 
-    pub fn update(&mut self, other: &TenantConfOpt) {
-        if let Some(checkpoint_distance) = other.checkpoint_distance {
-            self.checkpoint_distance = Some(checkpoint_distance);
-        }
-        if let Some(compaction_target_size) = other.compaction_target_size {
-            self.compaction_target_size = Some(compaction_target_size);
-        }
-        if let Some(compaction_period) = other.compaction_period {
-            self.compaction_period = Some(compaction_period);
-        }
-        if let Some(compaction_threshold) = other.compaction_threshold {
-            self.compaction_threshold = Some(compaction_threshold);
-        }
-        if let Some(gc_horizon) = other.gc_horizon {
-            self.gc_horizon = Some(gc_horizon);
-        }
-        if let Some(gc_period) = other.gc_period {
-            self.gc_period = Some(gc_period);
-        }
-        if let Some(pitr_interval) = other.pitr_interval {
-            self.pitr_interval = Some(pitr_interval);
+    pub fn from(tconf: TenantConf) -> Self {
+        Self {
+            hdr: TenantConfHeader {
+                checksum: 0,
+                size: 0,
+                format_version: STORAGE_FORMAT_VERSION,
+            },
+            body: TenantConf {
+                gc_period: tconf.gc_period,
+                gc_horizon: tconf.gc_horizon,
+                pitr_interval: tconf.pitr_interval,
+                checkpoint_distance: tconf.checkpoint_distance,
+                compaction_period: tconf.compaction_period,
+                compaction_target_size: tconf.compaction_target_size,
+            },
         }
     }
+
+    pub fn from_bytes(tenantconf_bytes: &[u8]) -> anyhow::Result<Self> {
+        ensure!(
+            tenantconf_bytes.len() == TENANTCONF_MAX_SIZE,
+            "tenantconf bytes size is wrong"
+        );
+        let hdr = TenantConfHeader::des(&tenantconf_bytes[0..TENANTCONF_HDR_SIZE])?;
+        ensure!(
+            hdr.format_version == STORAGE_FORMAT_VERSION,
+            "format version mismatch"
+        );
+        let tenantconf_size = hdr.size as usize;
+        ensure!(
+            tenantconf_size <= TENANTCONF_MAX_SIZE,
+            "corrupted tenantconf file"
+        );
+        let calculated_checksum =
+            crc32c::crc32c(&tenantconf_bytes[TENANTCONF_HDR_SIZE..tenantconf_size]);
+        ensure!(
+            hdr.checksum == calculated_checksum,
+            "tenantconf checksum mismatch"
+        );
+        let body = TenantConf::des(&tenantconf_bytes[TENANTCONF_HDR_SIZE..tenantconf_size])?;
+
+        Ok(TenantConfFile { hdr, body })
+    }
+
+    pub fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
+        let body_bytes = self.body.ser()?;
+        let tenantconf_size = TENANTCONF_HDR_SIZE + body_bytes.len();
+        let hdr = TenantConfHeader {
+            size: tenantconf_size as u16,
+            format_version: STORAGE_FORMAT_VERSION,
+            checksum: crc32c::crc32c(&body_bytes),
+        };
+        let hdr_bytes = hdr.ser()?;
+        let mut tenantconf_bytes = vec![0u8; TENANTCONF_MAX_SIZE];
+        tenantconf_bytes[0..TENANTCONF_HDR_SIZE].copy_from_slice(&hdr_bytes);
+        tenantconf_bytes[TENANTCONF_HDR_SIZE..tenantconf_size].copy_from_slice(&body_bytes);
+        Ok(tenantconf_bytes)
+    }
 }
 
 impl TenantConf {
@@ -132,7 +163,6 @@ impl TenantConf {
             compaction_target_size: DEFAULT_COMPACTION_TARGET_SIZE,
             compaction_period: humantime::parse_duration(DEFAULT_COMPACTION_PERIOD)
                 .expect("cannot parse default compaction period"),
-            compaction_threshold: DEFAULT_COMPACTION_THRESHOLD,
             gc_horizon: DEFAULT_GC_HORIZON,
             gc_period: humantime::parse_duration(DEFAULT_GC_PERIOD)
                 .expect("cannot parse default gc period"),
@@ -143,7 +173,7 @@ impl TenantConf {
 
     /// Points to a place in pageserver's local directory,
     /// where certain tenant's tenantconf file should be located.
-    pub fn path(conf: &'static PageServerConf, tenantid: ZTenantId) -> PathBuf {
+    pub fn tenantconf_path(conf: &'static PageServerConf, tenantid: ZTenantId) -> PathBuf {
         conf.tenant_path(&tenantid).join(TENANT_CONFIG_NAME)
     }
 
@@ -153,10 +183,38 @@ impl TenantConf {
             checkpoint_distance: defaults::DEFAULT_CHECKPOINT_DISTANCE,
             compaction_target_size: 4 * 1024 * 1024,
             compaction_period: Duration::from_secs(10),
-            compaction_threshold: defaults::DEFAULT_COMPACTION_THRESHOLD,
             gc_horizon: defaults::DEFAULT_GC_HORIZON,
             gc_period: Duration::from_secs(10),
             pitr_interval: Duration::from_secs(60 * 60),
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn tenantconf_serializes_correctly() {
+        let original_tenantconf = TenantConfFile::new(
+            111,
+            111,
+            Duration::from_secs(111),
+            222,
+            Duration::from_secs(111),
+            Duration::from_secs(60 * 60),
+        );
+
+        let tenantconf_bytes = original_tenantconf
+            .to_bytes()
+            .expect("Should serialize correct tenantconf to bytes");
+
+        let deserialized_tenantconf = TenantConfFile::from_bytes(&tenantconf_bytes)
+            .expect("Should deserialize its own bytes");
+
+        assert_eq!(
+            deserialized_tenantconf.body, original_tenantconf.body,
+            "Tenantconf that was serialized to bytes and deserialized back should not change"
+        );
+    }
+}

pageserver/src/tenant_mgr.rs

@@ -5,7 +5,7 @@ use crate::config::PageServerConf;
 use crate::layered_repository::LayeredRepository;
 use crate::remote_storage::RemoteIndex;
 use crate::repository::{Repository, TimelineSyncStatusUpdate};
-use crate::tenant_config::TenantConfOpt;
+use crate::tenant_config::TenantConf;
 use crate::thread_mgr;
 use crate::thread_mgr::ThreadKind;
 use crate::timelines;
@@ -14,14 +14,14 @@ use crate::walredo::PostgresRedoManager;
 use crate::{DatadirTimelineImpl, RepositoryImpl};
 use anyhow::{Context, Result};
 use lazy_static::lazy_static;
+use log::*;
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use std::collections::hash_map::Entry;
 use std::collections::HashMap;
 use std::fmt;
 use std::sync::{Arc, Mutex, MutexGuard};
-use tracing::*;
-use utils::zid::{ZTenantId, ZTimelineId};
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
 lazy_static! {
     static ref TENANTS: Mutex<HashMap<ZTenantId, Tenant>> = Mutex::new(HashMap::new());
@@ -70,16 +70,21 @@ pub fn load_local_repo(
     conf: &'static PageServerConf,
     tenant_id: ZTenantId,
     remote_index: &RemoteIndex,
-) -> Result<Arc<RepositoryImpl>> {
+) -> Arc<RepositoryImpl> {
     let mut m = access_tenants();
     let tenant = m.entry(tenant_id).or_insert_with(|| {
         // Set up a WAL redo manager, for applying WAL records.
         let walredo_mgr = PostgresRedoManager::new(conf, tenant_id);
 
+        // Try to load config file
+        let tenant_conf = LayeredRepository::load_tenantconf(conf, tenant_id)
+            .with_context(|| format!("Failed to load tenant state for id {}", tenant_id))
+            .unwrap();
+
         // Set up an object repository, for actual data storage.
         let repo: Arc<LayeredRepository> = Arc::new(LayeredRepository::new(
             conf,
-            Default::default(),
+            tenant_conf,
             Arc::new(walredo_mgr),
             tenant_id,
             remote_index.clone(),
@@ -91,18 +96,13 @@ pub fn load_local_repo(
             timelines: HashMap::new(),
         }
     });
-
-    // Restore tenant config
-    let tenant_conf = LayeredRepository::load_tenant_config(conf, tenant_id)?;
-    tenant.repo.update_tenant_config(tenant_conf)?;
-
-    Ok(Arc::clone(&tenant.repo))
+    Arc::clone(&tenant.repo)
 }
 
 /// Updates tenants' repositories, changing their timelines state in memory.
 pub fn apply_timeline_sync_status_updates(
     conf: &'static PageServerConf,
-    remote_index: &RemoteIndex,
+    remote_index: RemoteIndex,
     sync_status_updates: HashMap<ZTenantId, HashMap<ZTimelineId, TimelineSyncStatusUpdate>>,
 ) {
     if sync_status_updates.is_empty() {
@@ -116,16 +116,7 @@ pub fn apply_timeline_sync_status_updates(
     trace!("Sync status updates: {:?}", sync_status_updates);
 
     for (tenant_id, tenant_timelines_sync_status_updates) in sync_status_updates {
-        let repo = match load_local_repo(conf, tenant_id, remote_index) {
-            Ok(repo) => repo,
-            Err(e) => {
-                error!(
-                    "Failed to load repo for tenant {} Error: {:#}",
-                    tenant_id, e
-                );
-                continue;
-            }
-        };
+        let repo = load_local_repo(conf, tenant_id, &remote_index);
 
         for (timeline_id, timeline_sync_status_update) in tenant_timelines_sync_status_updates {
             match repo.apply_timeline_remote_sync_status_update(timeline_id, timeline_sync_status_update)
@@ -190,7 +181,7 @@ pub fn shutdown_all_tenants() {
 
 pub fn create_tenant_repository(
     conf: &'static PageServerConf,
-    tenant_conf: TenantConfOpt,
+    tenant_conf: TenantConf,
     tenantid: ZTenantId,
     remote_index: RemoteIndex,
 ) -> Result<Option<ZTenantId>> {
@@ -220,14 +211,6 @@ pub fn create_tenant_repository(
     }
 }
 
-pub fn update_tenant_config(tenant_conf: TenantConfOpt, tenantid: ZTenantId) -> Result<()> {
-    info!("configuring tenant {}", tenantid);
-    let repo = get_repository_for_tenant(tenantid)?;
-
-    repo.update_tenant_config(tenant_conf)?;
-    Ok(())
-}
-
 pub fn get_tenant_state(tenantid: ZTenantId) -> Option<TenantState> {
     Some(access_tenants().get(&tenantid)?.state)
 }
@@ -315,7 +298,7 @@ pub fn get_timeline_for_tenant_load(
         .get_timeline_load(timelineid)
         .with_context(|| format!("Timeline {} not found for tenant {}", timelineid, tenantid))?;
 
-    let repartition_distance = tenant.repo.get_checkpoint_distance() / 10;
+    let repartition_distance = tenant.repo.get_tenant_conf().checkpoint_distance / 10;
 
     let page_tline = Arc::new(DatadirTimelineImpl::new(tline, repartition_distance));
     page_tline.init_logical_size()?;

pageserver/src/tenant_threads.rs

@@ -6,7 +6,7 @@ use crate::tenant_mgr::TenantState;
 use anyhow::Result;
 use std::time::Duration;
 use tracing::*;
-use utils::zid::ZTenantId;
+use zenith_utils::zid::ZTenantId;
 
 ///
 /// Compaction thread's main loop
@@ -26,9 +26,9 @@ fn compact_loop_ext(tenantid: ZTenantId) -> Result<()> {
             break;
         }
         let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
-        let compaction_period = repo.get_compaction_period();
+        let tenant_conf = repo.get_tenant_conf();
 
-        std::thread::sleep(compaction_period);
+        std::thread::sleep(tenant_conf.compaction_period);
         trace!("compaction thread for tenant {} waking up", tenantid);
 
         // Compact timelines
@@ -55,15 +55,21 @@ pub fn gc_loop(tenantid: ZTenantId) -> Result<()> {
 
         trace!("gc thread for tenant {} waking up", tenantid);
         let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
-        let gc_horizon = repo.get_gc_horizon();
+        let tenant_conf = repo.get_tenant_conf();
+
         // Garbage collect old files that are not needed for PITR anymore
-        if gc_horizon > 0 {
-            repo.gc_iteration(None, gc_horizon, repo.get_pitr_interval(), false)?;
+        if tenant_conf.gc_horizon > 0 {
+            repo.gc_iteration(
+                None,
+                tenant_conf.gc_horizon,
+                tenant_conf.pitr_interval,
+                false,
+            )?;
         }
 
         // TODO Write it in more adequate way using
         // condvar.wait_timeout() or something
-        let mut sleep_time = repo.get_gc_period().as_secs();
+        let mut sleep_time = tenant_conf.gc_period.as_secs();
         while sleep_time > 0 && tenant_mgr::get_tenant_state(tenantid) == Some(TenantState::Active)
         {
             sleep_time -= 1;

pageserver/src/thread_mgr.rs

@@ -47,7 +47,7 @@ use tracing::{debug, error, info, warn};
 
 use lazy_static::lazy_static;
 
-use utils::zid::{ZTenantId, ZTimelineId};
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
 use crate::shutdown_pageserver;
 

pageserver/src/timelines.rs

@@ -14,18 +14,16 @@ use std::{
 };
 use tracing::*;
 
-use utils::{
-    crashsafe_dir, logging,
-    lsn::Lsn,
-    zid::{ZTenantId, ZTimelineId},
-};
+use zenith_utils::lsn::Lsn;
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
+use zenith_utils::{crashsafe_dir, logging};
 
 use crate::{
     config::PageServerConf,
     layered_repository::metadata::TimelineMetadata,
     remote_storage::RemoteIndex,
     repository::{LocalTimelineState, Repository},
-    tenant_config::TenantConfOpt,
+    tenant_config::TenantConf,
     DatadirTimeline, RepositoryImpl,
 };
 use crate::{import_datadir, LOG_FILE_NAME};
@@ -117,8 +115,8 @@ impl LocalTimelineInfo {
 #[serde_as]
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct RemoteTimelineInfo {
-    #[serde_as(as = "DisplayFromStr")]
-    pub remote_consistent_lsn: Lsn,
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub remote_consistent_lsn: Option<Lsn>,
     pub awaits_download: bool,
 }
 
@@ -152,7 +150,7 @@ pub fn init_pageserver(
 
     if let Some(tenant_id) = create_tenant {
         println!("initializing tenantid {}", tenant_id);
-        let repo = create_repo(conf, Default::default(), tenant_id, CreateRepo::Dummy)
+        let repo = create_repo(conf, TenantConf::default(), tenant_id, CreateRepo::Dummy)
             .context("failed to create repo")?;
         let new_timeline_id = initial_timeline_id.unwrap_or_else(ZTimelineId::generate);
         bootstrap_timeline(conf, tenant_id, new_timeline_id, repo.as_ref())
@@ -176,7 +174,7 @@ pub enum CreateRepo {
 
 pub fn create_repo(
     conf: &'static PageServerConf,
-    tenant_conf: TenantConfOpt,
+    tenant_conf: TenantConf,
     tenant_id: ZTenantId,
     create_repo: CreateRepo,
 ) -> Result<Arc<RepositoryImpl>> {
@@ -214,7 +212,7 @@ pub fn create_repo(
     info!("created directory structure in {}", repo_dir.display());
 
     // Save tenant's config
-    LayeredRepository::persist_tenant_config(conf, tenant_id, tenant_conf)?;
+    LayeredRepository::save_tenantconf(conf, tenant_id, tenant_conf, true)?;
 
     Ok(Arc::new(LayeredRepository::new(
         conf,

pageserver/src/virtual_file.rs

@@ -11,15 +11,15 @@
 //! src/backend/storage/file/fd.c
 //!
 use lazy_static::lazy_static;
-use once_cell::sync::OnceCell;
 use std::fs::{File, OpenOptions};
 use std::io::{Error, ErrorKind, Read, Seek, SeekFrom, Write};
 use std::os::unix::fs::FileExt;
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use std::sync::{RwLock, RwLockWriteGuard};
+use zenith_metrics::{register_histogram_vec, register_int_gauge_vec, HistogramVec, IntGaugeVec};
 
-use metrics::{register_histogram_vec, register_int_gauge_vec, HistogramVec, IntGaugeVec};
+use once_cell::sync::OnceCell;
 
 // Metrics collected on disk IO operations
 const STORAGE_IO_TIME_BUCKETS: &[f64] = &[

pageserver/src/wal_metadata.rs

@@ -1,101 +0,0 @@
-//!
-//! Utils for logging wal metadata. Useful for tests only, and too expensive to run in prod.
-//!
-//! Ideally we'd get this wal metadata using pg_waldump from the compute pg_wal directory,
-//! but pg_waldump doesn't provide all the metadata we need. We could write a rust program
-//! to analyze pg wal, but we'd need to port some c code for decoding wal files. This module
-//! is a temporary hack that allows us to print the metadata that the pageserver decodes
-//! using postgres_ffi::waldecoder.
-//!
-//! Logging wal metadata could add significant write overhead to the pageserver. Tests that
-//! rely on this should either spin up a dedicated pageserver for wal metadata logging, or
-//! only measure read performance.
-//!
-use anyhow::Result;
-use bytes::{BufMut, BytesMut};
-use once_cell::sync::OnceCell;
-use serde::{Deserialize, Serialize};
-use std::fs::File;
-use tokio::io::AsyncReadExt;
-use utils::lsn::Lsn;
-
-use crate::{config::PageServerConf, walrecord::DecodedBkpBlock};
-
-// TODO make a directory instead, and write one file per (tenant, timeline).
-pub static WAL_METADATA_FILE: OnceCell<File> = OnceCell::new();
-
-#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
-pub struct Page {
-    spcnode: u32,
-    dbnode: u32,
-    relnode: u32,
-    forknum: u8,
-    blkno: u32,
-}
-
-impl Page {
-    pub async fn read<Reader>(buf: &mut Reader) -> Result<Page>
-    where
-        Reader: tokio::io::AsyncRead + Unpin,
-    {
-        let spcnode = buf.read_u32().await?;
-        let dbnode = buf.read_u32().await?;
-        let relnode = buf.read_u32().await?;
-        let forknum = buf.read_u8().await?;
-        let blkno = buf.read_u32().await?;
-        Ok(Page {
-            spcnode,
-            dbnode,
-            relnode,
-            forknum,
-            blkno,
-        })
-    }
-
-    pub async fn write(&self, buf: &mut BytesMut) -> Result<()> {
-        buf.put_u32(self.spcnode);
-        buf.put_u32(self.dbnode);
-        buf.put_u32(self.relnode);
-        buf.put_u8(self.forknum);
-        buf.put_u32(self.blkno);
-        Ok(())
-    }
-}
-
-impl From<&DecodedBkpBlock> for Page {
-    fn from(blk: &DecodedBkpBlock) -> Self {
-        Page {
-            spcnode: blk.rnode_spcnode,
-            dbnode: blk.rnode_dbnode,
-            relnode: blk.rnode_relnode,
-            forknum: blk.forknum,
-            blkno: blk.blkno,
-        }
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct WalEntryMetadata {
-    pub lsn: Lsn,
-    pub size: usize,
-    pub affected_pages: Vec<Page>,
-}
-
-pub fn init(conf: &'static PageServerConf) -> Result<()> {
-    if conf.emit_wal_metadata {
-        let wal_metadata_file_dir = conf.workdir.join("wal_metadata.log");
-        WAL_METADATA_FILE
-            .set(File::create(wal_metadata_file_dir)?)
-            .expect("wal_metadata file is already created");
-    }
-    Ok(())
-}
-
-pub fn write(wal_meta: WalEntryMetadata) -> Result<()> {
-    if let Some(mut file) = WAL_METADATA_FILE.get() {
-        let mut line = serde_json::to_string(&wal_meta)?;
-        line.push('\n');
-        std::io::prelude::Write::write_all(&mut file, line.as_bytes())?;
-    }
-    Ok(())
-}

pageserver/src/walingest.rs

@@ -38,7 +38,7 @@ use postgres_ffi::nonrelfile_utils::mx_offset_to_member_segment;
 use postgres_ffi::xlog_utils::*;
 use postgres_ffi::TransactionId;
 use postgres_ffi::{pg_constants, CheckPoint};
-use utils::lsn::Lsn;
+use zenith_utils::lsn::Lsn;
 
 static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
 
@@ -82,7 +82,6 @@ impl<'a, R: Repository> WalIngest<'a, R> {
     ) -> Result<()> {
         let mut modification = timeline.begin_modification(lsn);
 
-        let recdata_len = recdata.len();
         let mut decoded = decode_wal_record(recdata);
         let mut buf = decoded.record.clone();
         buf.advance(decoded.main_data_offset);
@@ -250,13 +249,6 @@ impl<'a, R: Repository> WalIngest<'a, R> {
             self.ingest_decoded_block(&mut modification, lsn, &decoded, blk)?;
         }
 
-        // Emit wal entry metadata, if configured to do so
-        crate::wal_metadata::write(crate::wal_metadata::WalEntryMetadata {
-            lsn,
-            size: recdata_len,
-            affected_pages: decoded.blocks.iter().map(|blk| blk.into()).collect(),
-        })?;
-
         // If checkpoint data was updated, store the new version in the repository
         if self.checkpoint_modified {
             let new_checkpoint_bytes = self.checkpoint.encode();

pageserver/src/walreceiver.rs

@@ -29,11 +29,11 @@ use tokio_postgres::replication::ReplicationStream;
 use tokio_postgres::{Client, NoTls, SimpleQueryMessage, SimpleQueryRow};
 use tokio_stream::StreamExt;
 use tracing::*;
-use utils::{
-    lsn::Lsn,
-    pq_proto::ZenithFeedback,
-    zid::{ZTenantId, ZTenantTimelineId, ZTimelineId},
-};
+use zenith_utils::lsn::Lsn;
+use zenith_utils::pq_proto::ZenithFeedback;
+use zenith_utils::zid::ZTenantId;
+use zenith_utils::zid::ZTenantTimelineId;
+use zenith_utils::zid::ZTimelineId;
 
 //
 // We keep one WAL Receiver active per timeline.
@@ -305,7 +305,7 @@ fn walreceiver_main(
                         tenant_id,
                         timeline_id,
                     })
-                    .map(|remote_timeline| remote_timeline.metadata.disk_consistent_lsn())
+                    .and_then(|e| e.disk_consistent_lsn())
                     .unwrap_or(Lsn(0)) // no checkpoint was uploaded
             });
 

pageserver/src/walredo.rs

@@ -21,6 +21,7 @@
 use byteorder::{ByteOrder, LittleEndian};
 use bytes::{BufMut, Bytes, BytesMut};
 use lazy_static::lazy_static;
+use log::*;
 use nix::poll::*;
 use serde::Serialize;
 use std::fs;
@@ -34,15 +35,17 @@ use std::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command};
 use std::sync::Mutex;
 use std::time::Duration;
 use std::time::Instant;
-use tracing::*;
-use utils::{bin_ser::BeSer, lsn::Lsn, nonblock::set_nonblock, zid::ZTenantId};
+use zenith_metrics::{register_histogram, register_int_counter, Histogram, IntCounter};
+use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::Lsn;
+use zenith_utils::nonblock::set_nonblock;
+use zenith_utils::zid::ZTenantId;
 
 use crate::config::PageServerConf;
 use crate::pgdatadir_mapping::{key_to_rel_block, key_to_slru_block};
 use crate::reltag::{RelTag, SlruKind};
 use crate::repository::Key;
 use crate::walrecord::ZenithWalRecord;
-use metrics::{register_histogram, register_int_counter, Histogram, IntCounter};
 use postgres_ffi::nonrelfile_utils::mx_offset_to_flags_bitshift;
 use postgres_ffi::nonrelfile_utils::mx_offset_to_flags_offset;
 use postgres_ffi::nonrelfile_utils::mx_offset_to_member_offset;
@@ -700,12 +703,7 @@ impl PostgresRedoProcess {
             // If we have more data to write, wake up if 'stdin' becomes writeable or
             // we have data to read. Otherwise only wake up if there's data to read.
             let nfds = if nwrite < writebuf.len() { 3 } else { 2 };
-            let n = loop {
-                match nix::poll::poll(&mut pollfds[0..nfds], wal_redo_timeout.as_millis() as i32) {
-                    Err(e) if e == nix::errno::Errno::EINTR => continue,
-                    res => break res,
-                }
-            }?;
+            let n = nix::poll::poll(&mut pollfds[0..nfds], wal_redo_timeout.as_millis() as i32)?;
 
             if n == 0 {
                 return Err(Error::new(ErrorKind::Other, "WAL redo timed out"));

postgres_ffi/Cargo.toml

@@ -17,8 +17,8 @@ log = "0.4.14"
 memoffset = "0.6.2"
 thiserror = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-utils = { path = "../utils" }
-workspace_hack = { version = "0.1", path = "../../workspace_hack" }
+zenith_utils = { path = "../zenith_utils" }
+workspace_hack = { version = "0.1", path = "../workspace_hack" }
 
 [build-dependencies]
 bindgen = "0.59.1"

postgres_ffi/build.rs

@@ -88,8 +88,8 @@ fn main() {
         // 'pg_config --includedir-server' would perhaps be the more proper way to find it,
         // but this will do for now.
         //
-        .clang_arg("-I../../tmp_install/include/server")
-        .clang_arg("-I../../tmp_install/include/postgresql/server")
+        .clang_arg("-I../tmp_install/include/server")
+        .clang_arg("-I../tmp_install/include/postgresql/server")
         //
         // Finish the builder and generate the bindings.
         //

postgres_ffi/src/controlfile_utils.rs

@@ -43,7 +43,7 @@ impl ControlFileData {
     /// Interpret a slice of bytes as a Postgres control file.
     ///
     pub fn decode(buf: &[u8]) -> Result<ControlFileData> {
-        use utils::bin_ser::LeSer;
+        use zenith_utils::bin_ser::LeSer;
 
         // Check that the slice has the expected size. The control file is
         // padded with zeros up to a 512 byte sector size, so accept a
@@ -77,7 +77,7 @@ impl ControlFileData {
     ///
     /// The CRC is recomputed to match the contents of the fields.
     pub fn encode(&self) -> Bytes {
-        use utils::bin_ser::LeSer;
+        use zenith_utils::bin_ser::LeSer;
 
         // Serialize into a new buffer.
         let b = self.ser().unwrap();

postgres_ffi/src/waldecoder.rs

@@ -4,7 +4,7 @@
 //! This understands the WAL page and record format, enough to figure out where the WAL record
 //! boundaries are, and to reassemble WAL records that cross page boundaries.
 //!
-//! This functionality is needed by both the pageserver and the safekeepers. The pageserver needs
+//! This functionality is needed by both the pageserver and the walkeepers. The pageserver needs
 //! to look deeper into the WAL records to also understand which blocks they modify, the code
 //! for that is in pageserver/src/walrecord.rs
 //!
@@ -18,7 +18,7 @@ use crc32c::*;
 use log::*;
 use std::cmp::min;
 use thiserror::Error;
-use utils::lsn::Lsn;
+use zenith_utils::lsn::Lsn;
 
 pub struct WalStreamDecoder {
     lsn: Lsn,

postgres_ffi/src/xlog_utils.rs

@@ -28,7 +28,7 @@ use std::io::prelude::*;
 use std::io::SeekFrom;
 use std::path::{Path, PathBuf};
 use std::time::SystemTime;
-use utils::lsn::Lsn;
+use zenith_utils::lsn::Lsn;
 
 pub const XLOG_FNAME_LEN: usize = 24;
 pub const XLOG_BLCKSZ: usize = 8192;
@@ -351,17 +351,17 @@ pub fn main() {
 
 impl XLogRecord {
     pub fn from_slice(buf: &[u8]) -> XLogRecord {
-        use utils::bin_ser::LeSer;
+        use zenith_utils::bin_ser::LeSer;
         XLogRecord::des(buf).unwrap()
     }
 
     pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogRecord {
-        use utils::bin_ser::LeSer;
+        use zenith_utils::bin_ser::LeSer;
         XLogRecord::des_from(&mut buf.reader()).unwrap()
     }
 
     pub fn encode(&self) -> Bytes {
-        use utils::bin_ser::LeSer;
+        use zenith_utils::bin_ser::LeSer;
         self.ser().unwrap().into()
     }
 
@@ -373,19 +373,19 @@ impl XLogRecord {
 
 impl XLogPageHeaderData {
     pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogPageHeaderData {
-        use utils::bin_ser::LeSer;
+        use zenith_utils::bin_ser::LeSer;
         XLogPageHeaderData::des_from(&mut buf.reader()).unwrap()
     }
 }
 
 impl XLogLongPageHeaderData {
     pub fn from_bytes<B: Buf>(buf: &mut B) -> XLogLongPageHeaderData {
-        use utils::bin_ser::LeSer;
+        use zenith_utils::bin_ser::LeSer;
         XLogLongPageHeaderData::des_from(&mut buf.reader()).unwrap()
     }
 
     pub fn encode(&self) -> Bytes {
-        use utils::bin_ser::LeSer;
+        use zenith_utils::bin_ser::LeSer;
         self.ser().unwrap().into()
     }
 }
@@ -394,12 +394,12 @@ pub const SIZEOF_CHECKPOINT: usize = std::mem::size_of::<CheckPoint>();
 
 impl CheckPoint {
     pub fn encode(&self) -> Bytes {
-        use utils::bin_ser::LeSer;
+        use zenith_utils::bin_ser::LeSer;
         self.ser().unwrap().into()
     }
 
     pub fn decode(buf: &[u8]) -> Result<CheckPoint, anyhow::Error> {
-        use utils::bin_ser::LeSer;
+        use zenith_utils::bin_ser::LeSer;
         Ok(CheckPoint::des(buf)?)
     }
 
@@ -477,9 +477,7 @@ mod tests {
     #[test]
     pub fn test_find_end_of_wal() {
         // 1. Run initdb to generate some WAL
-        let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-            .join("..")
-            .join("..");
+        let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
         let data_dir = top_path.join("test_output/test_find_end_of_wal");
         let initdb_path = top_path.join("tmp_install/bin/initdb");
         let lib_path = top_path.join("tmp_install/lib");

pre-commit.py

@@ -29,7 +29,7 @@ def colorify(
 
 
 def rustfmt(fix_inplace: bool = False, no_color: bool = False) -> str:
-    cmd = "rustfmt --edition=2021"
+    cmd = "rustfmt --edition=2018"
     if not fix_inplace:
         cmd += " --check"
     if no_color: