remove unused import, to silence flake8

It's annoying that many of the tests required a special build with the
"testing" feature. I think it's better to have a runtime check. It adds
a few CPU instructions to where failpoints are defined, even if they are
disabled, but that's a small price to pay for the convenience.

Fixes issue 2531, although differently from what was discussed on that
issue.

.cargo/config.toml

@@ -11,6 +11,3 @@ opt-level = 3
 [profile.dev]
 # Turn on a small amount of optimization in Development mode.
 opt-level = 1
-
-[alias]
-build_testing = ["build", "--features", "testing"]

.github/actions/allure-report/action.yml

@@ -32,8 +32,8 @@ runs:
           exit 2
         fi
 
-    - name: Calculate key
-      id: calculate-key
+    - name: Calculate variables
+      id: calculate-vars
       shell: bash -euxo pipefail {0}
       run: |
         # TODO: for manually triggered workflows (via workflow_dispatch) we need to have a separate key
@@ -41,14 +41,22 @@ runs:
         pr_number=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH" || true)
         if [ "${pr_number}" != "null" ]; then
           key=pr-${pr_number}
-        elif [ "${GITHUB_REF}" = "refs/heads/main" ]; then
+        elif [ "${GITHUB_REF_NAME}" = "main" ]; then
           # Shortcut for a special branch
           key=main
+        elif [ "${GITHUB_REF_NAME}" = "release" ]; then
+          # Shortcut for a special branch
+          key=release
         else
-          key=branch-$(echo ${GITHUB_REF#refs/heads/} | tr -c "[:alnum:]._-" "-")
+          key=branch-$(printf "${GITHUB_REF_NAME}" | tr -c "[:alnum:]._-" "-")
         fi
         echo "KEY=${key}" >> $GITHUB_OUTPUT
 
+        # Sanitize test selection to remove `/` and any other special characters
+        # Use printf instead of echo to avoid having `\n` at the end of the string
+        test_selection=$(printf "${{ inputs.test_selection }}" | tr -c "[:alnum:]._-" "-" )
+        echo "TEST_SELECTION=${test_selection}" >> $GITHUB_OUTPUT
+
     - uses: actions/setup-java@v3
       if: ${{ inputs.action == 'generate' }}
       with:
@@ -74,10 +82,11 @@ runs:
     - name: Upload Allure results
       if: ${{ inputs.action == 'store' }}
       env:
-        REPORT_PREFIX: reports/${{ steps.calculate-key.outputs.KEY }}/${{ inputs.build_type }}
-        RAW_PREFIX: reports-raw/${{ steps.calculate-key.outputs.KEY }}/${{ inputs.build_type }}
+        REPORT_PREFIX: reports/${{ steps.calculate-vars.outputs.KEY }}/${{ inputs.build_type }}
+        RAW_PREFIX: reports-raw/${{ steps.calculate-vars.outputs.KEY }}/${{ inputs.build_type }}
         TEST_OUTPUT: /tmp/test_output
         BUCKET: neon-github-public-dev
+        TEST_SELECTION: ${{ steps.calculate-vars.outputs.TEST_SELECTION }}
       shell: bash -euxo pipefail {0}
       run: |
         # Add metadata
@@ -98,7 +107,7 @@ runs:
           BUILD_TYPE=${{ inputs.build_type }}
         EOF
 
-        ARCHIVE="${GITHUB_RUN_ID}-${{ inputs.test_selection }}-${GITHUB_RUN_ATTEMPT}-$(date +%s).tar.zst"
+        ARCHIVE="${GITHUB_RUN_ID}-${TEST_SELECTION}-${GITHUB_RUN_ATTEMPT}-$(date +%s).tar.zst"
         ZSTD_NBTHREADS=0
 
         tar -C ${TEST_OUTPUT}/allure/results -cf ${ARCHIVE} --zstd .
@@ -109,8 +118,9 @@ runs:
       if: ${{ inputs.action == 'generate' }}
       shell: bash -euxo pipefail {0}
       env:
-        LOCK_FILE: reports/${{ steps.calculate-key.outputs.KEY }}/lock.txt
+        LOCK_FILE: reports/${{ steps.calculate-vars.outputs.KEY }}/lock.txt
         BUCKET: neon-github-public-dev
+        TEST_SELECTION: ${{ steps.calculate-vars.outputs.TEST_SELECTION }}
       run: |
         LOCK_TIMEOUT=300 # seconds
 
@@ -123,12 +133,12 @@ runs:
             fi
             sleep 1
           done
-          echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ inputs.test_selection }}" > lock.txt
+          echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${TEST_SELECTION}" > lock.txt
           aws s3 mv --only-show-errors lock.txt "s3://${BUCKET}/${LOCK_FILE}"
 
           # A double-check that exactly WE have acquired the lock
           aws s3 cp --only-show-errors "s3://${BUCKET}/${LOCK_FILE}" ./lock.txt
-          if [ "$(cat lock.txt)" = "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ inputs.test_selection }}" ]; then
+          if [ "$(cat lock.txt)" = "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${TEST_SELECTION}" ]; then
             break
           fi
         done
@@ -137,8 +147,8 @@ runs:
       if: ${{ inputs.action == 'generate' }}
       id: generate-report
       env:
-        REPORT_PREFIX: reports/${{ steps.calculate-key.outputs.KEY }}/${{ inputs.build_type }}
-        RAW_PREFIX: reports-raw/${{ steps.calculate-key.outputs.KEY }}/${{ inputs.build_type }}
+        REPORT_PREFIX: reports/${{ steps.calculate-vars.outputs.KEY }}/${{ inputs.build_type }}
+        RAW_PREFIX: reports-raw/${{ steps.calculate-vars.outputs.KEY }}/${{ inputs.build_type }}
         TEST_OUTPUT: /tmp/test_output
         BUCKET: neon-github-public-dev
       shell: bash -euxo pipefail {0}
@@ -192,12 +202,13 @@ runs:
       if: ${{ inputs.action == 'generate' && always() }}
       shell: bash -euxo pipefail {0}
       env:
-        LOCK_FILE: reports/${{ steps.calculate-key.outputs.KEY }}/lock.txt
+        LOCK_FILE: reports/${{ steps.calculate-vars.outputs.KEY }}/lock.txt
         BUCKET: neon-github-public-dev
+        TEST_SELECTION: ${{ steps.calculate-vars.outputs.TEST_SELECTION }}
       run: |
         aws s3 cp --only-show-errors "s3://${BUCKET}/${LOCK_FILE}" ./lock.txt || exit 0
 
-        if [ "$(cat lock.txt)" = "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ inputs.test_selection }}" ]; then
+        if [ "$(cat lock.txt)" = "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${TEST_SELECTION}" ]; then
           aws s3 rm "s3://${BUCKET}/${LOCK_FILE}"
         fi
 

.github/actions/neon-branch-create/action.yml

@@ -0,0 +1,154 @@
+name: 'Create Branch'
+description: 'Create Branch using API'
+
+inputs:
+  api_key:
+    desctiption: 'Neon API key'
+    required: true
+  environment:
+    desctiption: 'dev (aka captest) or staging'
+    required: true
+  project_id:
+    desctiption: 'ID of the Project to create Branch in'
+    required: true
+outputs:
+  dsn:
+    description: 'Created Branch DSN (for main database)'
+    value: ${{ steps.change-password.outputs.dsn }}
+  branch_id:
+    description: 'Created Branch ID'
+    value: ${{ steps.create-branch.outputs.branch_id }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Parse Input
+      id: parse-input
+      shell: bash -euxo pipefail {0}
+      run: |
+        case "${ENVIRONMENT}" in
+          dev)
+            API_HOST=console.dev.neon.tech
+            ;;
+          staging)
+            API_HOST=console.stage.neon.tech
+            ;;
+          *)
+            echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only"
+            exit 1
+            ;;
+        esac
+
+        echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT
+      env:
+        ENVIRONMENT: ${{ inputs.environment }}
+
+    - name: Create New Branch
+      id: create-branch
+      shell: bash -euxo pipefail {0}
+      run: |
+        for i in $(seq 1 10); do
+         branch=$(curl \
+            "https://${API_HOST}/api/v2/projects/${PROJECT_ID}/branches" \
+            --header "Accept: application/json" \
+            --header "Content-Type: application/json" \
+            --header "Authorization: Bearer ${API_KEY}" \
+            --data "{
+              \"branch\": {
+                \"name\": \"Created by actions/neon-branch-create; GITHUB_RUN_ID=${GITHUB_RUN_ID} at $(date +%s)\"
+              }
+            }")
+
+          if [ -z "${branch}" ]; then
+            sleep 1
+            continue
+          fi
+
+          branch_id=$(echo $branch | jq --raw-output '.branch.id')
+          if [ "${branch_id}" == "null" ]; then
+            sleep 1
+            continue
+          fi
+
+          break
+        done
+
+        if [ -z "${branch_id}" ] || [ "${branch_id}" == "null" ]; then
+          echo 2>&1 "Failed to create branch after 10 attempts, the latest response was: ${branch}"
+          exit 1
+        fi
+
+        branch_id=$(echo $branch | jq --raw-output '.branch.id')
+        echo "branch_id=${branch_id}" >> $GITHUB_OUTPUT
+
+        host=$(echo $branch | jq --raw-output '.endpoints[0].host')
+        echo "host=${host}" >> $GITHUB_OUTPUT
+      env:
+        API_KEY: ${{ inputs.api_key }}
+        API_HOST: ${{ steps.parse-input.outputs.api_host }}
+        PROJECT_ID: ${{ inputs.project_id }}
+
+    - name: Get Role name
+      id: role-name
+      shell: bash -euxo pipefail {0}
+      run: |
+        roles=$(curl \
+          "https://${API_HOST}/api/v2/projects/${PROJECT_ID}/branches/${BRANCH_ID}/roles" \
+          --fail \
+          --header "Accept: application/json" \
+          --header "Content-Type: application/json" \
+          --header "Authorization: Bearer ${API_KEY}"
+          )
+
+        role_name=$(echo $roles | jq --raw-output '.roles[] | select(.protected == false) | .name')
+        echo "role_name=${role_name}" >> $GITHUB_OUTPUT
+      env:
+        API_KEY: ${{ inputs.api_key }}
+        API_HOST: ${{ steps.parse-input.outputs.api_host }}
+        PROJECT_ID: ${{ inputs.project_id }}
+        BRANCH_ID: ${{ steps.create-branch.outputs.branch_id }}
+
+    - name: Change Password
+      id: change-password
+      # A shell without `set -x` to not to expose password/dsn in logs
+      shell: bash -euo pipefail {0}
+      run: |
+        for i in $(seq 1 10); do
+          reset_password=$(curl \
+            "https://${API_HOST}/api/v2/projects/${PROJECT_ID}/branches/${BRANCH_ID}/roles/${ROLE_NAME}/reset_password" \
+            --request POST \
+            --header "Accept: application/json" \
+            --header "Content-Type: application/json" \
+            --header "Authorization: Bearer ${API_KEY}"
+            )
+
+          if [ -z "${reset_password}" ]; then
+            sleep 1
+            continue
+          fi
+
+          password=$(echo $reset_password | jq --raw-output '.role.password')
+          if [ "${password}" == "null" ]; then
+            sleep 1
+            continue
+          fi
+
+          echo "::add-mask::${password}"
+          break
+        done
+
+        if [ -z "${password}" ] || [ "${password}" == "null" ]; then
+          echo 2>&1 "Failed to reset password after 10 attempts, the latest response was: ${reset_password}"
+          exit 1
+        fi
+
+        dsn="postgres://${ROLE_NAME}:${password}@${HOST}/neondb"
+        echo "::add-mask::${dsn}"
+        echo "dsn=${dsn}" >> $GITHUB_OUTPUT
+      env:
+        API_KEY: ${{ inputs.api_key }}
+        API_HOST: ${{ steps.parse-input.outputs.api_host }}
+        PROJECT_ID: ${{ inputs.project_id }}
+        BRANCH_ID: ${{ steps.create-branch.outputs.branch_id }}
+        ROLE_NAME: ${{ steps.role-name.outputs.role_name }}
+        HOST: ${{ steps.create-branch.outputs.host }}

.github/actions/neon-branch-delete/action.yml

@@ -0,0 +1,79 @@
+name: 'Delete Branch'
+description: 'Delete Branch using API'
+
+inputs:
+  api_key:
+    desctiption: 'Neon API key'
+    required: true
+  environment:
+    desctiption: 'dev (aka captest) or staging'
+    required: true
+  project_id:
+    desctiption: 'ID of the Project which should be deleted'
+    required: true
+  branch_id:
+    desctiption: 'ID of the branch to delete'
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Parse Input
+      id: parse-input
+      shell: bash -euxo pipefail {0}
+      run: |
+        case "${ENVIRONMENT}" in
+          dev)
+            API_HOST=console.dev.neon.tech
+            ;;
+          staging)
+            API_HOST=console.stage.neon.tech
+            ;;
+          *)
+            echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only"
+            exit 1
+            ;;
+        esac
+
+        echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT
+      env:
+        ENVIRONMENT: ${{ inputs.environment }}
+
+    - name: Delete Branch
+      # Do not try to delete a branch if .github/actions/neon-project-create
+      # or .github/actions/neon-branch-create failed before
+      if: ${{ inputs.project_id != '' && inputs.branch_id != '' }}
+      shell: bash -euxo pipefail {0}
+      run: |
+        for i in $(seq 1 10); do
+          deleted_branch=$(curl \
+            "https://${API_HOST}/api/v2/projects/${PROJECT_ID}/branches/${BRANCH_ID}" \
+            --request DELETE \
+            --header "Accept: application/json" \
+            --header "Content-Type: application/json" \
+            --header "Authorization: Bearer ${API_KEY}"
+            )
+
+          if [ -z "${deleted_branch}" ]; then
+            sleep 1
+            continue
+          fi
+
+          branch_id=$(echo $deleted_branch | jq --raw-output '.branch.id')
+          if [ "${branch_id}" == "null" ]; then
+            sleep 1
+            continue
+          fi
+
+          break
+        done
+
+        if [ -z "${branch_id}" ] || [ "${branch_id}" == "null" ]; then
+          echo 2>&1 "Failed to delete branch after 10 attempts, the latest response was: ${deleted_branch}"
+          exit 1
+        fi
+      env:
+        API_KEY: ${{ inputs.api_key }}
+        PROJECT_ID: ${{ inputs.project_id }}
+        BRANCH_ID: ${{ inputs.branch_id }}
+        API_HOST: ${{ steps.parse-input.outputs.api_host }}

.github/actions/neon-project-create/action.yml

@@ -6,7 +6,7 @@ inputs:
     desctiption: 'Neon API key'
     required: true
   environment:
-    desctiption: 'dev (aka captest) or stage'
+    desctiption: 'dev (aka captest) or staging'
     required: true
   region_id:
     desctiption: 'Region ID, if not set the project will be created in the default region'
@@ -29,11 +29,11 @@ runs:
         case "${ENVIRONMENT}" in
           dev)
             API_HOST=console.dev.neon.tech
-            REGION_ID=${REGION_ID:-eu-west-1}
+            REGION_ID=${REGION_ID:-aws-eu-west-1}
             ;;
           staging)
             API_HOST=console.stage.neon.tech
-            REGION_ID=${REGION_ID:-us-east-1}
+            REGION_ID=${REGION_ID:-aws-us-east-2}
             ;;
           *)
             echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only"
@@ -53,7 +53,7 @@ runs:
       shell: bash -euo pipefail {0}
       run: |
         project=$(curl \
-          "https://${API_HOST}/api/v1/projects" \
+          "https://${API_HOST}/api/v2/projects" \
           --fail \
           --header "Accept: application/json" \
           --header "Content-Type: application/json" \
@@ -61,7 +61,6 @@ runs:
           --data "{
             \"project\": {
               \"name\": \"Created by actions/neon-project-create; GITHUB_RUN_ID=${GITHUB_RUN_ID}\",
-              \"platform_id\": \"aws\",
               \"region_id\": \"${REGION_ID}\",
               \"settings\": { }
             }
@@ -70,11 +69,11 @@ runs:
         # Mask password
         echo "::add-mask::$(echo $project | jq --raw-output '.roles[] | select(.name != "web_access") | .password')"
 
-        dsn=$(echo $project | jq --raw-output '.roles[] | select(.name != "web_access") | .dsn')/main
+        dsn=$(echo $project | jq --raw-output '.connection_uris[0].connection_uri')
         echo "::add-mask::${dsn}"
         echo "dsn=${dsn}" >> $GITHUB_OUTPUT
 
-        project_id=$(echo $project | jq --raw-output '.id')
+        project_id=$(echo $project | jq --raw-output '.project.id')
         echo "project_id=${project_id}" >> $GITHUB_OUTPUT
       env:
         API_KEY: ${{ inputs.api_key }}

.github/actions/neon-project-delete/action.yml

@@ -6,7 +6,7 @@ inputs:
     desctiption: 'Neon API key'
     required: true
   environment:
-    desctiption: 'dev (aka captest) or stage'
+    desctiption: 'dev (aka captest) or staging'
     required: true
   project_id:
     desctiption: 'ID of the Project to delete'
@@ -37,17 +37,17 @@ runs:
         ENVIRONMENT: ${{ inputs.environment }}
 
     - name: Delete Neon Project
+      # Do not try to delete a project if .github/actions/neon-project-create failed before
+      if: ${{ inputs.project_id != '' }}
       shell: bash -euxo pipefail {0}
       run: |
-        # Allow PROJECT_ID to be empty/null for cases when .github/actions/neon-project-create failed
-        if [ -n "${PROJECT_ID}" ]; then
-          curl -X "POST" \
-            "https://${API_HOST}/api/v1/projects/${PROJECT_ID}/delete" \
-            --fail \
-            --header "Accept: application/json" \
-            --header "Content-Type: application/json" \
-            --header "Authorization: Bearer ${API_KEY}"
-        fi
+        curl \
+          "https://${API_HOST}/api/v2/projects/${PROJECT_ID}" \
+          --fail \
+          --request DELETE \
+          --header "Accept: application/json" \
+          --header "Content-Type: application/json" \
+          --header "Authorization: Bearer ${API_KEY}"
       env:
         API_KEY: ${{ inputs.api_key }}
         PROJECT_ID: ${{ inputs.project_id }}

.github/ansible/scripts/init_pageserver.sh

@@ -1,7 +1,8 @@
 #!/bin/sh
 
-# get instance id from meta-data service
+# fetch params from meta-data service
 INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
+AZ_ID=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)
 
 # store fqdn hostname in var
 HOST=$(hostname -f)
@@ -16,7 +17,8 @@ cat <<EOF | tee /tmp/payload
   "instance_id": "${INSTANCE_ID}",
   "http_host": "${HOST}",
   "http_port": 9898,
-  "active": false
+  "active": false,
+  "availability_zone_id": "${AZ_ID}"
 }
 EOF
 

.github/ansible/systemd/pageserver.service

@@ -5,7 +5,7 @@ After=network.target auditd.service
 [Service]
 Type=simple
 User=pageserver
-Environment=RUST_BACKTRACE=1 NEON_REPO_DIR=/storage/pageserver LD_LIBRARY_PATH=/usr/local/v14/lib
+Environment=RUST_BACKTRACE=1 NEON_REPO_DIR=/storage/pageserver LD_LIBRARY_PATH=/usr/local/v14/lib SENTRY_DSN={{ SENTRY_URL_PAGESERVER }}
 ExecStart=/usr/local/bin/pageserver -c "pg_distrib_dir='/usr/local'" -c "listen_pg_addr='0.0.0.0:6400'" -c "listen_http_addr='0.0.0.0:9898'" -c "broker_endpoints=['{{ etcd_endpoints }}']" -D /storage/pageserver/data
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=mixed

.github/ansible/systemd/safekeeper.service

@@ -5,7 +5,7 @@ After=network.target auditd.service
 [Service]
 Type=simple
 User=safekeeper
-Environment=RUST_BACKTRACE=1 NEON_REPO_DIR=/storage/safekeeper/data LD_LIBRARY_PATH=/usr/local/v14/lib
+Environment=RUST_BACKTRACE=1 NEON_REPO_DIR=/storage/safekeeper/data LD_LIBRARY_PATH=/usr/local/v14/lib SENTRY_DSN={{ SENTRY_URL_SAFEKEEPER }}
 ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}{{ hostname_suffix }}:6500 --listen-http {{ inventory_hostname }}{{ hostname_suffix }}:7676 -D /storage/safekeeper/data --broker-endpoints={{ etcd_endpoints }} --remote-storage='{bucket_name="{{bucket_name}}", bucket_region="{{bucket_region}}", prefix_in_bucket="{{ safekeeper_s3_prefix }}"}'
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=mixed

.github/helm-values/dev-eu-west-1-zeta.neon-storage-broker.yaml

@@ -0,0 +1,53 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: staging
+  neon_service: storage-broker
+
+ingress:
+  enabled: true
+  annotations:
+    kubernetes.io/ingress.class: nginx-internal
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+    cert-manager.io/cluster-issuer: "cert-manager-clusterissuer"
+
+  hosts:
+    - host: storage-broker-zeta.eu-west-1.aws.neon.build
+      paths:
+        - path: /
+          pathType: Prefix
+  tls:
+    - hosts:
+        - storage-broker-zeta.eu-west-1.aws.neon.build
+      secretName: storage-broker-tls
+
+
+metrics:
+  enabled: false
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+

.github/helm-values/dev-us-east-2-beta.neon-proxy-scram-legacy.yaml

@@ -0,0 +1,31 @@
+# Helm chart values for neon-proxy-scram.
+# This is a YAML-formatted file.
+
+image:
+  repository: neondatabase/neon
+
+settings:
+  authBackend: "console"
+  authEndpoint: "http://console-staging.local/management/api/v2"
+  domain: "*.cloud.stage.neon.tech"
+
+# -- Additional labels for neon-proxy pods
+podLabels:
+  zenith_service: proxy-scram-legacy
+  zenith_env: dev
+  zenith_region: us-east-2
+  zenith_region_slug: us-east-2
+
+exposedService:
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
+    external-dns.alpha.kubernetes.io/hostname: neon-proxy-scram-legacy.beta.us-east-2.aws.neon.build
+
+#metrics:
+#  enabled: true
+#  serviceMonitor:
+#    enabled: true
+#    selector:
+#      release: kube-prometheus-stack

.github/helm-values/dev-us-east-2-beta.neon-storage-broker.yaml

@@ -0,0 +1,53 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: staging
+  neon_service: storage-broker
+
+ingress:
+  enabled: true
+  annotations:
+    kubernetes.io/ingress.class: nginx-internal
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+    cert-manager.io/cluster-issuer: "cert-manager-clusterissuer"
+
+  hosts:
+    - host: storage-broker-beta.us-east-2.aws.neon.build
+      paths:
+        - path: /
+          pathType: Prefix
+  tls:
+    - hosts:
+        - storage-broker-beta.us-east-2.aws.neon.build
+      secretName: storage-broker-tls
+
+
+metrics:
+  enabled: false
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+

.github/helm-values/neon-stress.neon-storage-broker.yaml

@@ -0,0 +1,54 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: neon-stress
+  neon_service: storage-broker
+
+ingress:
+  enabled: true
+  annotations:
+    kubernetes.io/ingress.class: alb
+    alb.ingress.kubernetes.io/healthcheck-path: /status
+    alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]'
+    alb.ingress.kubernetes.io/scheme: "internal"
+    alb.ingress.kubernetes.io/target-type: "ip"
+    alb.ingress.kubernetes.io/ssl-redirect: "443"
+    alb.ingress.kubernetes.io/backend-protocol-version: "GRPC"
+
+  hosts:
+    - host: storage-broker-stress.stage.neon.tech
+      paths:
+        - path: /
+          pathType: Prefix
+
+metrics:
+  enabled: true
+  serviceMonitor:
+    enabled: true
+    selector:
+      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+

.github/helm-values/prod-ap-southeast-1-epsilon.neon-storage-broker.yaml

@@ -0,0 +1,53 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: production
+  neon_service: storage-broker
+
+ingress:
+  enabled: true
+  annotations:
+    kubernetes.io/ingress.class: nginx-internal
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+    cert-manager.io/cluster-issuer: "cert-manager-clusterissuer"
+
+  hosts:
+    - host: storage-broker-epsilon.ap-southeast-1.aws.neon.tech
+      paths:
+        - path: /
+          pathType: Prefix
+  tls:
+    - hosts:
+        - storage-broker-epsilon.ap-southeast-1.aws.neon.tech
+      secretName: storage-broker-tls
+
+
+metrics:
+  enabled: false
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+

.github/helm-values/prod-eu-central-1-gamma.neon-storage-broker.yaml

@@ -0,0 +1,53 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: production
+  neon_service: storage-broker
+
+ingress:
+  enabled: true
+  annotations:
+    kubernetes.io/ingress.class: nginx-internal
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+    cert-manager.io/cluster-issuer: "cert-manager-clusterissuer"
+
+  hosts:
+    - host: storage-broker-gamma.eu-central-1.aws.neon.tech
+      paths:
+        - path: /
+          pathType: Prefix
+  tls:
+    - hosts:
+        - storage-broker-gamma.eu-central-1.aws.neon.tech
+      secretName: storage-broker-tls
+
+
+metrics:
+  enabled: false
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+

.github/helm-values/prod-us-east-2-delta.neon-storage-broker.yaml

@@ -0,0 +1,53 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: production
+  neon_service: storage-broker
+
+ingress:
+  enabled: true
+  annotations:
+    kubernetes.io/ingress.class: nginx-internal
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+    cert-manager.io/cluster-issuer: "cert-manager-clusterissuer"
+
+  hosts:
+    - host: storage-broker-delta.us-east-2.aws.neon.tech
+      paths:
+        - path: /
+          pathType: Prefix
+  tls:
+    - hosts:
+        - storage-broker-delta.us-east-2.aws.neon.tech
+      secretName: storage-broker-tls
+
+
+metrics:
+  enabled: false
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+

.github/helm-values/prod-us-west-2-eta.neon-proxy-scram.yaml

@@ -0,0 +1,31 @@
+# Helm chart values for neon-proxy-scram.
+# This is a YAML-formatted file.
+
+image:
+  repository: neondatabase/neon
+
+settings:
+  authBackend: "console"
+  authEndpoint: "http://console-release.local/management/api/v2"
+  domain: "*.us-west-2.aws.neon.tech"
+
+# -- Additional labels for neon-proxy pods
+podLabels:
+  zenith_service: proxy-scram
+  zenith_env: prod
+  zenith_region: us-west-2
+  zenith_region_slug: us-west-2
+
+exposedService:
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
+    external-dns.alpha.kubernetes.io/hostname: us-west-2.aws.neon.tech
+
+#metrics:
+#  enabled: true
+#  serviceMonitor:
+#    enabled: true
+#    selector:
+#      release: kube-prometheus-stack

.github/helm-values/prod-us-west-2-eta.neon-storage-broker.yaml

@@ -0,0 +1,53 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: production
+  neon_service: storage-broker
+
+ingress:
+  enabled: true
+  annotations:
+    kubernetes.io/ingress.class: nginx-internal
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+    cert-manager.io/cluster-issuer: "cert-manager-clusterissuer"
+
+  hosts:
+    - host: storage-broker-eta.us-west-2.aws.neon.tech
+      paths:
+        - path: /
+          pathType: Prefix
+  tls:
+    - hosts:
+        - storage-broker-eta.us-west-2.aws.neon.tech
+      secretName: storage-broker-tls
+
+
+metrics:
+  enabled: false
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+

.github/helm-values/production.neon-storage-broker.yaml

@@ -0,0 +1,54 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: production
+  neon_service: storage-broker
+
+ingress:
+  enabled: true
+  annotations:
+    kubernetes.io/ingress.class: alb
+    alb.ingress.kubernetes.io/healthcheck-path: /status
+    alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]'
+    alb.ingress.kubernetes.io/scheme: "internal"
+    alb.ingress.kubernetes.io/target-type: "ip"
+    alb.ingress.kubernetes.io/ssl-redirect: "443"
+    alb.ingress.kubernetes.io/backend-protocol-version: "GRPC"
+
+  hosts:
+    - host: storage-broker.neon.tech
+      paths:
+        - path: /
+          pathType: Prefix
+
+metrics:
+  enabled: true
+  serviceMonitor:
+    enabled: true
+    selector:
+      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+

.github/helm-values/staging.neon-storage-broker.yaml

@@ -0,0 +1,54 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: staging
+  neon_service: storage-broker
+
+ingress:
+  enabled: true
+  annotations:
+    kubernetes.io/ingress.class: alb
+    alb.ingress.kubernetes.io/healthcheck-path: /status
+    alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]'
+    alb.ingress.kubernetes.io/scheme: "internal"
+    alb.ingress.kubernetes.io/target-type: "ip"
+    alb.ingress.kubernetes.io/ssl-redirect: "443"
+    alb.ingress.kubernetes.io/backend-protocol-version: "GRPC"
+
+  hosts:
+    - host: storage-broker.stage.neon.tech
+      paths:
+        - path: /
+          pathType: Prefix
+
+metrics:
+  enabled: true
+  serviceMonitor:
+    enabled: true
+    selector:
+      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+

.github/workflows/benchmarking.yml

@@ -110,8 +110,14 @@ jobs:
         rm -rf perf-report-staging
         mkdir -p perf-report-staging
         # Set --sparse-ordering option of pytest-order plugin to ensure tests are running in order of appears in the file,
-        # it's important for test_perf_pgbench.py::test_pgbench_remote_* tests
-        ./scripts/pytest test_runner/performance/ -v -m "remote_cluster" --sparse-ordering --out-dir perf-report-staging --timeout 5400
+        # it's important for test_perf_pgbench.py::test_pgbench_remote_* tests.
+        # Do not run tests from test_runner/performance/test_perf_olap.py because they require a prepared DB. We run them separately in `clickbench-compare` job.
+        ./scripts/pytest test_runner/performance/ -v \
+          -m "remote_cluster" \
+          --sparse-ordering \
+          --out-dir perf-report-staging \
+          --timeout 5400 \
+          --ignore test_runner/performance/test_perf_olap.py
 
     - name: Submit result
       env:
@@ -207,7 +213,7 @@ jobs:
             CONNSTR=${{ steps.create-neon-project.outputs.dsn }}
             ;;
           rds-aurora)
-            CONNSTR=${{ secrets.BENCHMARK_RDS_CONNSTR }}
+            CONNSTR=${{ secrets.BENCHMARK_RDS_AURORA_CONNSTR }}
             ;;
           rds-postgres)
             CONNSTR=${{ secrets.BENCHMARK_RDS_POSTGRES_CONNSTR }}
@@ -225,8 +231,11 @@ jobs:
     - name: Set database options
       if: matrix.platform == 'neon-captest-prefetch'
       run: |
-        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE main SET enable_seqscan_prefetch=on"
-        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE main SET seqscan_prefetch_buffers=10"
+        DB_NAME=$(psql ${BENCHMARK_CONNSTR} --no-align --quiet -t -c "SELECT current_database()")
+
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET enable_seqscan_prefetch=on"
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET effective_io_concurrency=32"
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET maintenance_io_concurrency=32"
       env:
         BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
 
@@ -292,3 +301,115 @@ jobs:
         slack-message: "Periodic perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
       env:
         SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+
+  clickbench-compare:
+    # ClichBench DB for rds-aurora and rds-Postgres deployed to the same clusters
+    # we use for performance testing in pgbench-compare.
+    # Run this job only when pgbench-compare is finished to avoid the intersection.
+    # We might change it after https://github.com/neondatabase/neon/issues/2900.
+    #
+    # *_CLICKBENCH_CONNSTR: Genuine ClickBench DB with ~100M rows
+    # *_CLICKBENCH_10M_CONNSTR: DB with the first 10M rows of ClickBench DB
+    if: success() || failure()
+    needs: [ pgbench-compare ]
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # neon-captest-prefetch: We have pre-created projects with prefetch enabled
+        # rds-aurora: Aurora Postgres Serverless v2 with autoscaling from 0.5 to 2 ACUs
+        # rds-postgres: RDS Postgres db.m5.large instance (2 vCPU, 8 GiB) with gp3 EBS storage
+        platform: [ neon-captest-prefetch, rds-postgres, rds-aurora ]
+
+    env:
+      POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
+      DEFAULT_PG_VERSION: 14
+      TEST_OUTPUT: /tmp/test_output
+      BUILD_TYPE: remote
+      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }}
+      PLATFORM: ${{ matrix.platform }}
+
+    runs-on: [ self-hosted, dev, x64 ]
+    container:
+      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:pinned
+      options: --init
+
+    timeout-minutes: 360 # 6h
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Download Neon artifact
+      uses: ./.github/actions/download
+      with:
+        name: neon-${{ runner.os }}-release-artifact
+        path: /tmp/neon/
+        prefix: latest
+
+    - name: Add Postgres binaries to PATH
+      run: |
+        ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version
+        echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH
+
+    - name: Set up Connection String
+      id: set-up-connstr
+      run: |
+        case "${PLATFORM}" in
+          neon-captest-prefetch)
+            CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CLICKBENCH_10M_CONNSTR }}
+            ;;
+          rds-aurora)
+            CONNSTR=${{ secrets.BENCHMARK_RDS_AURORA_CLICKBENCH_10M_CONNSTR }}
+            ;;
+          rds-postgres)
+            CONNSTR=${{ secrets.BENCHMARK_RDS_POSTGRES_CLICKBENCH_10M_CONNSTR }}
+            ;;
+          *)
+            echo 2>&1 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neon-captest-prefetch', 'rds-aurora', or 'rds-postgres'"
+            exit 1
+            ;;
+        esac
+
+        echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
+
+        psql ${CONNSTR} -c "SELECT version();"
+
+    - name: Set database options
+      if: matrix.platform == 'neon-captest-prefetch'
+      run: |
+        DB_NAME=$(psql ${BENCHMARK_CONNSTR} --no-align --quiet -t -c "SELECT current_database()")
+
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET enable_seqscan_prefetch=on"
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET effective_io_concurrency=32"
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET maintenance_io_concurrency=32"
+      env:
+        BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
+
+    - name: Benchmark clickbench
+      uses: ./.github/actions/run-python-test-set
+      with:
+        build_type: ${{ env.BUILD_TYPE }}
+        test_selection: performance/test_perf_olap.py
+        run_in_parallel: false
+        save_perf_report: ${{ env.SAVE_PERF_REPORT }}
+        extra_params: -m remote_cluster --timeout 21600 -k test_clickbench
+      env:
+        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
+        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
+        BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
+
+    - name: Create Allure report
+      if: success() || failure()
+      uses: ./.github/actions/allure-report
+      with:
+        action: generate
+        build_type: ${{ env.BUILD_TYPE }}
+
+    - name: Post to a Slack channel
+      if: ${{ github.event.schedule && failure() }}
+      uses: slackapi/slack-github-action@v1
+      with:
+        channel-id: "C033QLM5P7D" # dev-staging-stream
+        slack-message: "Periodic OLAP perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+      env:
+        SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

.github/workflows/build_and_test.yml

@@ -100,11 +100,11 @@ jobs:
         run: |
           if [[ $BUILD_TYPE == "debug" ]]; then
             cov_prefix="scripts/coverage --profraw-prefix=$GITHUB_JOB --dir=/tmp/coverage run"
-            CARGO_FEATURES="--features testing"
+            CARGO_FEATURES=""
             CARGO_FLAGS="--locked --timings $CARGO_FEATURES"
           elif [[ $BUILD_TYPE == "release" ]]; then
             cov_prefix=""
-            CARGO_FEATURES="--features testing,profiling"
+            CARGO_FEATURES="--features profiling"
             CARGO_FLAGS="--locked --timings --release $CARGO_FEATURES"
           fi
           echo "cov_prefix=${cov_prefix}" >> $GITHUB_ENV
@@ -539,9 +539,9 @@ jobs:
       # `neondatabase/neon` contains multiple binaries, all of them use the same input for the version into the same version formatting library.
       # Pick pageserver as currently the only binary with extra "version" features printed in the string to verify.
       # Regular pageserver version string looks like
-      #   Neon page server git-env:32d14403bd6ab4f4520a94cbfd81a6acef7a526c failpoints: true, features: []
+      #   Neon page server git-env:32d14403bd6ab4f4520a94cbfd81a6acef7a526c features: []
       # Bad versions might loop like:
-      #   Neon page server git-env:local failpoints: true, features: ["testing"]
+      #   Neon page server git-env:local features: [""]
       # Ensure that we don't have bad versions.
       - name: Verify image versions
         shell: bash # ensure no set -e for better error messages
@@ -555,11 +555,6 @@ jobs:
             exit 1
           fi
 
-          if ! echo "$pageserver_version" | grep -qv '"testing"' ; then
-            echo "Pageserver version should have no testing feature enabled"
-            exit 1
-          fi
-
       - name: Verify docker-compose example
         run: env REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com TAG=${{needs.tag.outputs.build-tag}} ./docker-compose/docker_compose_test.sh
 
@@ -668,11 +663,11 @@ jobs:
       - id: set-matrix
         run: |
           if [[ "$GITHUB_REF_NAME" == "main" ]]; then
-            STAGING='{"env_name": "staging", "proxy_job": "neon-proxy", "proxy_config": "staging.proxy", "kubeconfig_secret": "STAGING_KUBECONFIG_DATA", "console_api_key_secret": "NEON_STAGING_API_KEY"}'
-            NEON_STRESS='{"env_name": "neon-stress", "proxy_job": "neon-stress-proxy", "proxy_config": "neon-stress.proxy", "kubeconfig_secret": "NEON_STRESS_KUBECONFIG_DATA", "console_api_key_secret": "NEON_CAPTEST_API_KEY"}'
+            STAGING='{"env_name": "staging", "proxy_job": "neon-proxy", "proxy_config": "staging.proxy", "storage_broker_ns": "neon-storage-broker", "storage_broker_config": "staging.neon-storage-broker", "kubeconfig_secret": "STAGING_KUBECONFIG_DATA", "console_api_key_secret": "NEON_STAGING_API_KEY"}'
+            NEON_STRESS='{"env_name": "neon-stress", "proxy_job": "neon-stress-proxy", "proxy_config": "neon-stress.proxy", "storage_broker_ns": "neon-stress-storage-broker", "storage_broker_config": "neon-stress.neon-storage-broker", "kubeconfig_secret": "NEON_STRESS_KUBECONFIG_DATA", "console_api_key_secret": "NEON_CAPTEST_API_KEY", storage_broker_config: }'
             echo "include=[$STAGING, $NEON_STRESS]" >> $GITHUB_OUTPUT
           elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
-            PRODUCTION='{"env_name": "production", "proxy_job": "neon-proxy", "proxy_config": "production.proxy", "kubeconfig_secret": "PRODUCTION_KUBECONFIG_DATA", "console_api_key_secret": "NEON_PRODUCTION_API_KEY"}'
+            PRODUCTION='{"env_name": "production", "proxy_job": "neon-proxy", "proxy_config": "production.proxy", "storage_broker_ns": "neon-storage-broker", "storage_broker_config": "production.neon-storage-broker", "kubeconfig_secret": "PRODUCTION_KUBECONFIG_DATA", "console_api_key_secret": "NEON_PRODUCTION_API_KEY"}'
             echo "include=[$PRODUCTION]" >> $GITHUB_OUTPUT
           else
             echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
@@ -732,7 +727,7 @@ jobs:
           ssh-add ssh-key
           rm -f ssh-key ssh-key-cert.pub
           ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i ${{ matrix.env_name }}.hosts.yaml -e CONSOLE_API_TOKEN=${{ secrets[matrix.console_api_key_secret] }}
+          ansible-playbook deploy.yaml -i ${{ matrix.env_name }}.hosts.yaml -e CONSOLE_API_TOKEN=${{ secrets[matrix.console_api_key_secret] }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }}
           rm -f neon_install.tar.gz .neon_current_version
 
   deploy-new:
@@ -770,7 +765,7 @@ jobs:
             exit 1
           fi
           ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_STAGING_API_KEY}}
+          ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_STAGING_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }}
           rm -f neon_install.tar.gz .neon_current_version
 
   deploy-pr-test-new:
@@ -803,7 +798,7 @@ jobs:
           ./get_binaries.sh
 
           ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_STAGING_API_KEY}}
+          ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_STAGING_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }}
           rm -f neon_install.tar.gz .neon_current_version
 
   deploy-prod-new:
@@ -843,7 +838,7 @@ jobs:
           fi
 
           ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i prod.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_PRODUCTION_API_KEY}}
+          ansible-playbook deploy.yaml -i prod.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_PRODUCTION_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }}
           rm -f neon_install.tar.gz .neon_current_version
 
   deploy-proxy:
@@ -885,8 +880,49 @@ jobs:
       - name: Re-deploy proxy
         run: |
           DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade ${{ matrix.proxy_job }}       neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
-          helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
+          helm upgrade ${{ matrix.proxy_job }}       neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}.yaml       --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+          helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+
+  deploy-storage-broker-staging:
+    runs-on: [ self-hosted, dev, x64 ]
+    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:latest
+    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
+    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
+    if: |
+      (github.ref_name == 'main' || github.ref_name == 'release') &&
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}}
+    env:
+      KUBECONFIG: .kubeconfig
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Add curl
+        run: apt update && apt install curl -y
+
+      - name: Store kubeconfig file
+        run: |
+          echo "${{ secrets[matrix.kubeconfig_secret] }}" | base64 --decode > ${KUBECONFIG}
+          chmod 0600 ${KUBECONFIG}
+
+      - name: Setup helm v3
+        run: |
+          curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+          helm repo add neondatabase https://neondatabase.github.io/helm-charts
+
+      - name: Deploy storage-broker
+        run:
+          DOCKER_TAG=${{ needs.tag.outputs.build-tag }}
+          helm upgrade neon-storage-broker neondatabase/neon-storage-broker --namespace ${{ matrix.storage_broker_ns }} --create-namespace --install -f .github/helm-values/${{ matrix.storage_broker_config }}.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
 
   deploy-proxy-new:
     runs-on: [ self-hosted, dev, x64 ]
@@ -905,9 +941,11 @@ jobs:
           - target_region:  us-east-2
             target_cluster: dev-us-east-2-beta
             deploy_link_proxy: true
+            deploy_legacy_scram_proxy: true
           - target_region:  eu-west-1
             target_cluster: dev-eu-west-1-zeta
             deploy_link_proxy: false
+            deploy_legacy_scram_proxy: false
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -923,13 +961,54 @@ jobs:
       - name: Re-deploy scram proxy
         run: |
           DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
+          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
 
       - name: Re-deploy link proxy
         if: matrix.deploy_link_proxy
         run: |
           DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
+          helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+
+      - name: Re-deploy legacy scram proxy
+        if: matrix.deploy_legacy_scram_proxy
+        run: |
+          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
+          helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+
+  deploy-storage-broker-dev-new:
+    runs-on: [ self-hosted, dev, x64 ]
+    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
+    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
+    needs: [ push-docker-hub, tag, regress-tests ]
+    if: |
+      (github.ref_name == 'main') &&
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        include:
+          - target_region:  us-east-2
+            target_cluster: dev-us-east-2-beta
+          - target_region:  eu-west-1
+            target_cluster: dev-eu-west-1-zeta
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Configure environment
+        run: |
+          helm repo add neondatabase https://neondatabase.github.io/helm-charts
+          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
+
+      - name: Deploy storage-broker
+        run:
+          DOCKER_TAG=${{ needs.tag.outputs.build-tag }}
+          helm upgrade neon-storage-broker neondatabase/neon-storage-broker --namespace neon-storage-broker --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
 
   deploy-proxy-prod-new:
     runs-on: prod
@@ -947,6 +1026,8 @@ jobs:
         include:
           - target_region:  us-east-2
             target_cluster: prod-us-east-2-delta
+          - target_region:  us-west-2
+            target_cluster: prod-us-west-2-eta
           - target_region: eu-central-1
             target_cluster: prod-eu-central-1-gamma
           - target_region: ap-southeast-1
@@ -966,7 +1047,46 @@ jobs:
       - name: Re-deploy proxy
         run: |
           DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
+          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+
+  deploy-storage-broker-prod-new:
+    runs-on: prod
+    container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
+    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
+    needs: [ push-docker-hub, tag, regress-tests ]
+    if: |
+      (github.ref_name == 'release') &&
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        include:
+          - target_region:  us-east-2
+            target_cluster: prod-us-east-2-delta
+          - target_region:  us-west-2
+            target_cluster: prod-us-west-2-eta
+          - target_region: eu-central-1
+            target_cluster: prod-eu-central-1-gamma
+          - target_region: ap-southeast-1
+            target_cluster: prod-ap-southeast-1-epsilon
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Configure environment
+        run: |
+          helm repo add neondatabase https://neondatabase.github.io/helm-charts
+          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
+
+      - name: Deploy storage-broker
+        run:
+          DOCKER_TAG=${{ needs.tag.outputs.build-tag }}
+          helm upgrade neon-storage-broker neondatabase/neon-storage-broker --namespace neon-storage-broker --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
 
   promote-compatibility-data:
     runs-on: [ self-hosted, dev, x64 ]

README.md

@@ -2,29 +2,20 @@
 
 Neon is a serverless open-source alternative to AWS Aurora Postgres. It separates storage and compute and substitutes the PostgreSQL storage layer by redistributing data across a cluster of nodes.
 
-The project used to be called "Zenith". Many of the commands and code comments
-still refer to "zenith", but we are in the process of renaming things.
-
 ## Quick start
-[Join the waitlist](https://neon.tech/) for our free tier to receive your serverless postgres instance. Then connect to it with your preferred postgres client (psql, dbeaver, etc) or use the online SQL editor.
+Try the [Neon Free Tier](https://neon.tech/docs/introduction/technical-preview-free-tier/) to create a serverless Postgres instance. Then connect to it with your preferred Postgres client (psql, dbeaver, etc) or use the online [SQL Editor](https://neon.tech/docs/get-started-with-neon/query-with-neon-sql-editor/). See [Connect from any application](https://neon.tech/docs/connect/connect-from-any-app/) for connection instructions.
 
 Alternatively, compile and run the project [locally](#running-local-installation).
 
 ## Architecture overview
 
-A Neon installation consists of compute nodes and a Neon storage engine.
-
-Compute nodes are stateless PostgreSQL nodes backed by the Neon storage engine.
+A Neon installation consists of compute nodes and the Neon storage engine. Compute nodes are stateless PostgreSQL nodes backed by the Neon storage engine.
 
 The Neon storage engine consists of two major components:
 - Pageserver. Scalable storage backend for the compute nodes.
-- WAL service. The service receives WAL from the compute node and ensures that it is stored durably.
+- Safekeepers. The safekeepers form a redundant WAL service that received WAL from the compute node, and stores it durably until it has been processed by the pageserver and uploaded to cloud storage.
 
-Pageserver consists of:
-- Repository - Neon storage implementation.
-- WAL receiver - service that receives WAL from WAL service and stores it in the repository.
-- Page service - service that communicates with compute nodes and responds with pages from the repository.
-- WAL redo - service that builds pages from base images and WAL records on Page service request
+See developer documentation in [/docs/SUMMARY.md](/docs/SUMMARY.md) for more information.
 
 ## Running local installation
 
@@ -222,19 +213,27 @@ Ensure your dependencies are installed as described [here](https://github.com/ne
 ```sh
 git clone --recursive https://github.com/neondatabase/neon.git
 
-CARGO_BUILD_FLAGS="--features=testing" make
+make
 
 ./scripts/pytest
 ```
 
 ## Documentation
 
-Now we use README files to cover design ideas and overall architecture for each module and `rustdoc` style documentation comments. See also [/docs/](/docs/) a top-level overview of all available markdown documentation.
+[/docs/](/docs/) Contains a top-level overview of all available markdown documentation.
 
 - [/docs/sourcetree.md](/docs/sourcetree.md) contains overview of source tree layout.
 
 To view your `rustdoc` documentation in a browser, try running `cargo doc --no-deps --open`
 
+See also README files in some source directories, and `rustdoc` style documentation comments.
+
+Other resources:
+
+- [SELECT 'Hello, World'](https://neon.tech/blog/hello-world/): Blog post by Nikita Shamgunov on the high level architecture
+- [Architecture decisions in Neon](https://neon.tech/blog/architecture-decisions-in-neon/): Blog post by Heikki Linnakangas
+- [Neon: Serverless PostgreSQL!](https://www.youtube.com/watch?v=rES0yzeERns): Presentation on storage system by Heikki Linnakangas in the CMU Database Group seminar series
+
 ### Postgres-specific terms
 
 Due to Neon's very close relation with PostgreSQL internals, numerous specific terms are used.

compute_tools/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 
 [dependencies]
 anyhow = "1.0"
-chrono = "0.4"
+chrono = { version = "0.4", default-features = false, features = ["clock"] }
 clap = "4.0"
 env_logger = "0.9"
 futures = "0.3.13"

control_plane/src/background_process.rs

@@ -14,17 +14,19 @@
 
 use std::ffi::OsStr;
 use std::io::Write;
-use std::path::Path;
+use std::os::unix::prelude::AsRawFd;
+use std::os::unix::process::CommandExt;
+use std::path::{Path, PathBuf};
 use std::process::{Child, Command};
 use std::time::Duration;
 use std::{fs, io, thread};
 
-use anyhow::{anyhow, bail, Context, Result};
+use anyhow::Context;
 use nix::errno::Errno;
+use nix::fcntl::{FcntlArg, FdFlag};
 use nix::sys::signal::{kill, Signal};
 use nix::unistd::Pid;
-
-use utils::lock_file;
+use utils::pid_file::{self, PidFileRead};
 
 // These constants control the loop used to poll for process start / stop.
 //
@@ -86,6 +88,14 @@ where
     let filled_cmd = fill_aws_secrets_vars(fill_rust_env_vars(background_command));
     filled_cmd.envs(envs);
 
+    let pid_file_to_check = match initial_pid_file {
+        InitialPidFile::Create(path) => {
+            pre_exec_create_pidfile(filled_cmd, path);
+            path
+        }
+        InitialPidFile::Expect(path) => path,
+    };
+
     let mut spawned_process = filled_cmd.spawn().with_context(|| {
         format!("Could not spawn {process_name}, see console output and log files for details.")
     })?;
@@ -95,29 +105,8 @@ where
             .with_context(|| format!("Subprocess {process_name} has invalid pid {pid}"))?,
     );
 
-    let pid_file_to_check = match initial_pid_file {
-        InitialPidFile::Create(target_pid_file_path) => {
-            match lock_file::create_lock_file(target_pid_file_path, pid.to_string()) {
-                lock_file::LockCreationResult::Created { .. } => {
-                    // We use "lock" file here only to create the pid file. The lock on the pidfile will be dropped as soon
-                    // as this CLI invocation exits, so it's a bit useless, but doesn't any harm either.
-                }
-                lock_file::LockCreationResult::AlreadyLocked { .. } => {
-                    anyhow::bail!("Cannot write pid file for {process_name} at path {target_pid_file_path:?}: file is already locked by another process")
-                }
-                lock_file::LockCreationResult::CreationFailed(e) => {
-                    return Err(e.context(format!(
-                    "Failed to create pid file for {process_name} at path {target_pid_file_path:?}"
-                )))
-                }
-            }
-            None
-        }
-        InitialPidFile::Expect(pid_file_path) => Some(pid_file_path),
-    };
-
     for retries in 0..RETRIES {
-        match process_started(pid, pid_file_to_check, &process_status_check) {
+        match process_started(pid, Some(pid_file_to_check), &process_status_check) {
             Ok(true) => {
                 println!("\n{process_name} started, pid: {pid}");
                 return Ok(spawned_process);
@@ -147,14 +136,45 @@ where
     anyhow::bail!("{process_name} did not start in {RETRY_UNTIL_SECS} seconds");
 }
 
+/// Send SIGTERM to child process
+pub fn send_stop_child_process(child: &std::process::Child) -> anyhow::Result<()> {
+    let pid = child.id();
+    match kill(
+        nix::unistd::Pid::from_raw(pid.try_into().unwrap()),
+        Signal::SIGTERM,
+    ) {
+        Ok(()) => Ok(()),
+        Err(Errno::ESRCH) => {
+            println!("child process with pid {pid} does not exist");
+            Ok(())
+        }
+        Err(e) => anyhow::bail!("Failed to send signal to child process with pid {pid}: {e}"),
+    }
+}
+
 /// Stops the process, using the pid file given. Returns Ok also if the process is already not running.
 pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> anyhow::Result<()> {
-    if !pid_file.exists() {
-        println!("{process_name} is already stopped: no pid file {pid_file:?} is present");
-        return Ok(());
-    }
-    let pid = read_pidfile(pid_file)?;
+    let pid = match pid_file::read(pid_file)
+        .with_context(|| format!("read pid_file {pid_file:?}"))?
+    {
+        PidFileRead::NotExist => {
+            println!("{process_name} is already stopped: no pid file present at {pid_file:?}");
+            return Ok(());
+        }
+        PidFileRead::NotHeldByAnyProcess(_) => {
+            // Don't try to kill according to file contents beacuse the pid might have been re-used by another process.
+            // Don't delete the file either, it can race with new pid file creation.
+            // Read `pid_file` module comment for details.
+            println!(
+                "No process is holding the pidfile. The process must have already exited. Leave in place to avoid race conditions: {pid_file:?}"
+            );
+            return Ok(());
+        }
+        PidFileRead::LockedByOtherProcess(pid) => pid,
+    };
+    // XXX the pid could become invalid (and recycled) at any time before the kill() below.
 
+    // send signal
     let sig = if immediate {
         print!("Stopping {process_name} with pid {pid} immediately..");
         Signal::SIGQUIT
@@ -166,8 +186,9 @@ pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> any
     match kill(pid, sig) {
         Ok(()) => (),
         Err(Errno::ESRCH) => {
+            // Again, don't delete the pid file. The unlink can race with a new pid file being created.
             println!(
-                "{process_name} with pid {pid} does not exist, but a pid file {pid_file:?} was found"
+                "{process_name} with pid {pid} does not exist, but a pid file {pid_file:?} was found. Likely the pid got recycled. Lucky we didn't harm anyone."
             );
             return Ok(());
         }
@@ -179,11 +200,6 @@ pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> any
         match process_has_stopped(pid) {
             Ok(true) => {
                 println!("\n{process_name} stopped");
-                if let Err(e) = fs::remove_file(pid_file) {
-                    if e.kind() != io::ErrorKind::NotFound {
-                        eprintln!("Failed to remove pid file {pid_file:?} after stopping the process: {e:#}");
-                    }
-                }
                 return Ok(());
             }
             Ok(false) => {
@@ -209,7 +225,14 @@ pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> any
 }
 
 fn fill_rust_env_vars(cmd: &mut Command) -> &mut Command {
-    let mut filled_cmd = cmd.env_clear().env("RUST_BACKTRACE", "1");
+    // If RUST_BACKTRACE is set, pass it through. But if it's not set, default
+    // to RUST_BACKTRACE=1.
+    let backtrace_setting = std::env::var_os("RUST_BACKTRACE");
+    let backtrace_setting = backtrace_setting
+        .as_deref()
+        .unwrap_or_else(|| OsStr::new("1"));
+
+    let mut filled_cmd = cmd.env_clear().env("RUST_BACKTRACE", backtrace_setting);
 
     // Pass through these environment variables to the command
     for var in ["LLVM_PROFILE_FILE", "FAILPOINTS", "RUST_LOG"] {
@@ -234,6 +257,69 @@ fn fill_aws_secrets_vars(mut cmd: &mut Command) -> &mut Command {
     cmd
 }
 
+/// Add a `pre_exec` to the cmd that, inbetween fork() and exec(),
+/// 1. Claims a pidfile with a fcntl lock on it and
+/// 2. Sets up the pidfile's file descriptor so that it (and the lock)
+///    will remain held until the cmd exits.
+fn pre_exec_create_pidfile<P>(cmd: &mut Command, path: P) -> &mut Command
+where
+    P: Into<PathBuf>,
+{
+    let path: PathBuf = path.into();
+    // SAFETY
+    // pre_exec is marked unsafe because it runs between fork and exec.
+    // Why is that dangerous in various ways?
+    // Long answer:  https://github.com/rust-lang/rust/issues/39575
+    // Short answer: in a multi-threaded program, other threads may have
+    // been inside of critical sections at the time of fork. In the
+    // original process, that was allright, assuming they protected
+    // the critical sections appropriately, e.g., through locks.
+    // Fork adds another process to the mix that
+    //   1. Has a single thread T
+    //   2. In an exact copy of the address space at the time of fork.
+    // A variety of problems scan occur now:
+    //   1. T tries to grab a lock that was locked at the time of fork.
+    //      It will wait forever since in its address space, the lock
+    //      is in state 'taken' but the thread that would unlock it is
+    //      not there.
+    //   2. A rust object that represented some external resource in the
+    //      parent now got implicitly copied by the the fork, even though
+    //      the object's type is not `Copy`. The parent program may use
+    //      non-copyability as way to enforce unique ownership of an
+    //      external resource in the typesystem. The fork breaks that
+    //      assumption, as now both parent and child process have an
+    //      owned instance of the object that represents the same
+    //      underlying resource.
+    // While these seem like niche problems, (1) in particular is
+    // highly relevant. For example, `malloc()` may grab a mutex internally,
+    // and so, if we forked while another thread was mallocing' and our
+    // pre_exec closure allocates as well, it will block on the malloc
+    // mutex forever
+    //
+    // The proper solution is to only use C library functions that are marked
+    // "async-signal-safe": https://man7.org/linux/man-pages/man7/signal-safety.7.html
+    //
+    // With this specific pre_exec() closure, the non-error path doesn't allocate.
+    // The error path uses `anyhow`, and hence does allocate.
+    // We take our chances there, hoping that any potential disaster is constrained
+    // to the child process (e.g., malloc has no state ourside of the child process).
+    // Last, `expect` prints to stderr, and stdio is not async-signal-safe.
+    // Again, we take our chances, making the same assumptions as for malloc.
+    unsafe {
+        cmd.pre_exec(move || {
+            let file = pid_file::claim_for_current_process(&path).expect("claim pid file");
+            // Remove the FD_CLOEXEC flag on the pidfile descriptor so that the pidfile
+            // remains locked after exec.
+            nix::fcntl::fcntl(file.as_raw_fd(), FcntlArg::F_SETFD(FdFlag::empty()))
+                .expect("remove FD_CLOEXEC");
+            // Don't run drop(file), it would close the file before we actually exec.
+            std::mem::forget(file);
+            Ok(())
+        });
+    }
+    cmd
+}
+
 fn process_started<F>(
     pid: Pid,
     pid_file_to_check: Option<&Path>,
@@ -244,14 +330,11 @@ where
 {
     match status_check() {
         Ok(true) => match pid_file_to_check {
-            Some(pid_file_path) => {
-                if pid_file_path.exists() {
-                    let pid_in_file = read_pidfile(pid_file_path)?;
-                    Ok(pid_in_file == pid)
-                } else {
-                    Ok(false)
-                }
-            }
+            Some(pid_file_path) => match pid_file::read(pid_file_path)? {
+                PidFileRead::NotExist => Ok(false),
+                PidFileRead::LockedByOtherProcess(pid_in_file) => Ok(pid_in_file == pid),
+                PidFileRead::NotHeldByAnyProcess(_) => Ok(false),
+            },
             None => Ok(true),
         },
         Ok(false) => Ok(false),
@@ -259,21 +342,6 @@ where
     }
 }
 
-/// Read a PID file
-///
-/// We expect a file that contains a single integer.
-fn read_pidfile(pidfile: &Path) -> Result<Pid> {
-    let pid_str = fs::read_to_string(pidfile)
-        .with_context(|| format!("failed to read pidfile {pidfile:?}"))?;
-    let pid: i32 = pid_str
-        .parse()
-        .map_err(|_| anyhow!("failed to parse pidfile {pidfile:?}"))?;
-    if pid < 1 {
-        bail!("pidfile {pidfile:?} contained bad value '{pid}'");
-    }
-    Ok(Pid::from_raw(pid))
-}
-
 fn process_has_stopped(pid: Pid) -> anyhow::Result<bool> {
     match kill(pid, None) {
         // Process exists, keep waiting

control_plane/src/bin/neon_local.rs

@@ -324,7 +324,7 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result<LocalEnv> {
             pg_version,
         )
         .unwrap_or_else(|e| {
-            eprintln!("pageserver init failed: {e}");
+            eprintln!("pageserver init failed: {e:?}");
             exit(1);
         });
 

control_plane/src/local_env.rs

@@ -156,6 +156,8 @@ pub struct PageServerConf {
 
     // jwt auth token used for communication with pageserver
     pub auth_token: String,
+
+    pub testing_mode: bool,
 }
 
 impl Default for PageServerConf {
@@ -166,6 +168,7 @@ impl Default for PageServerConf {
             listen_http_addr: String::new(),
             auth_type: AuthType::Trust,
             auth_token: String::new(),
+            testing_mode: false,
         }
     }
 }

control_plane/src/pageserver.rs

@@ -1,12 +1,12 @@
 use std::collections::HashMap;
-use std::fs::{self, File};
+use std::fs::File;
 use std::io::{BufReader, Write};
 use std::num::NonZeroU64;
 use std::path::{Path, PathBuf};
 use std::process::Child;
 use std::{io, result};
 
-use anyhow::{bail, Context};
+use anyhow::{bail, ensure, Context};
 use pageserver_api::models::{
     TenantConfigRequest, TenantCreateRequest, TenantInfo, TimelineCreateRequest, TimelineInfo,
 };
@@ -141,6 +141,9 @@ impl PageServerNode {
         init_config_overrides.push(&listen_http_addr_param);
         init_config_overrides.push(&listen_pg_addr_param);
         init_config_overrides.push(&broker_endpoints_param);
+        if self.env.pageserver.testing_mode {
+            init_config_overrides.push("testing_mode=true");
+        }
 
         if let Some(broker_etcd_prefix_param) = broker_etcd_prefix_param.as_deref() {
             init_config_overrides.push(broker_etcd_prefix_param);
@@ -168,29 +171,21 @@ impl PageServerNode {
             }
             Err(e) => eprintln!("{e:#}"),
         }
-        match pageserver_process.kill() {
-            Err(e) => {
-                eprintln!(
-                    "Failed to stop pageserver {} process with pid {}: {e:#}",
-                    self.env.pageserver.id,
-                    pageserver_process.id(),
-                )
-            }
-            Ok(()) => {
-                println!(
-                    "Stopped pageserver {} process with pid {}",
-                    self.env.pageserver.id,
-                    pageserver_process.id(),
-                );
-                // cleanup after pageserver startup, since we do not call regular `stop_process` during init
-                let pid_file = self.pid_file();
-                if let Err(e) = fs::remove_file(&pid_file) {
-                    if e.kind() != io::ErrorKind::NotFound {
-                        eprintln!("Failed to remove pid file {pid_file:?} after stopping the process: {e:#}");
-                    }
-                }
-            }
-        }
+        background_process::send_stop_child_process(&pageserver_process)?;
+
+        let exit_code = pageserver_process.wait()?;
+        ensure!(
+            exit_code.success(),
+            format!(
+                "pageserver init failed with exit code {:?}",
+                exit_code.code()
+            )
+        );
+        println!(
+            "Stopped pageserver {} process with pid {}",
+            self.env.pageserver.id,
+            pageserver_process.id(),
+        );
         init_result
     }
 

docs/sourcetree.md

@@ -45,9 +45,9 @@ and create new databases and accounts (control plane API in our case).
 
 Integration tests, written in Python using the `pytest` framework.
 
-`/vendor/postgres-v14`:
+`/vendor/postgres-v14` and `/vendor/postgres-v15`:
 
-PostgreSQL source tree, with the modifications needed for Neon.
+PostgreSQL source tree per version, with the modifications needed for Neon.
 
 `/pgxn/neon`:
 

libs/pageserver_api/src/models.rs

@@ -15,20 +15,37 @@ use bytes::{BufMut, Bytes, BytesMut};
 /// A state of a tenant in pageserver's memory.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 pub enum TenantState {
-    /// Tenant is fully operational, its background jobs might be running or not.
-    Active { background_jobs_running: bool },
+    // This tenant is being loaded from local disk
+    Loading,
+    // This tenant is being downloaded from cloud storage.
+    Attaching,
+    /// Tenant is fully operational
+    Active,
     /// A tenant is recognized by pageserver, but it is being detached or the
     /// system is being shut down.
-    Paused,
+    Stopping,
     /// A tenant is recognized by the pageserver, but can no longer be used for
     /// any operations, because it failed to be activated.
     Broken,
 }
 
+impl TenantState {
+    pub fn has_in_progress_downloads(&self) -> bool {
+        match self {
+            Self::Loading => true,
+            Self::Attaching => true,
+            Self::Active => false,
+            Self::Stopping => false,
+            Self::Broken => false,
+        }
+    }
+}
+
 /// A state of a timeline in pageserver's memory.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 pub enum TimelineState {
-    /// Timeline is fully operational, its background jobs are running.
+    /// Timeline is fully operational. If the containing Tenant is Active, the timeline's
+    /// background jobs are running otherwise they will be launched when the tenant is activated.
     Active,
     /// A timeline is recognized by pageserver, but not yet ready to operate.
     /// The status indicates, that the timeline could eventually go back to Active automatically:
@@ -36,7 +53,7 @@ pub enum TimelineState {
     Suspended,
     /// A timeline is recognized by pageserver, but not yet ready to operate and not allowed to
     /// automatically become Active after certain events: only a management call can change this status.
-    Paused,
+    Stopping,
     /// A timeline is recognized by the pageserver, but can no longer be used for
     /// any operations, because it failed to be activated.
     Broken,
@@ -170,6 +187,8 @@ pub struct TimelineInfo {
     pub latest_gc_cutoff_lsn: Lsn,
     #[serde_as(as = "DisplayFromStr")]
     pub disk_consistent_lsn: Lsn,
+    #[serde_as(as = "DisplayFromStr")]
+    pub remote_consistent_lsn: Lsn,
     pub current_logical_size: Option<u64>, // is None when timeline is Unloaded
     pub current_physical_size: Option<u64>, // is None when timeline is Unloaded
     pub current_logical_size_non_incremental: Option<u64>,
@@ -182,10 +201,6 @@ pub struct TimelineInfo {
     pub last_received_msg_ts: Option<u128>,
     pub pg_version: u32,
 
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub remote_consistent_lsn: Option<Lsn>,
-    pub awaits_download: bool,
-
     pub state: TimelineState,
 
     // Some of the above fields are duplicated in 'local' and 'remote', for backwards-

libs/remote_storage/Cargo.toml

@@ -9,8 +9,11 @@ async-trait = "0.1"
 metrics = { version = "0.1", path = "../metrics" }
 utils = { version = "0.1", path = "../utils" }
 once_cell = "1.13.0"
-rusoto_core = "0.48"
-rusoto_s3 = "0.48"
+aws-smithy-http = "0.51.0"
+aws-types = "0.51.0"
+aws-config = { version = "0.51.0", default-features = false, features=["rustls"] }
+aws-sdk-s3 = "0.21.0"
+hyper = { version = "0.14", features = ["stream"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 tokio = { version = "1.17", features = ["sync", "macros", "fs", "io-util"] }

libs/remote_storage/src/lib.rs

@@ -10,7 +10,7 @@ mod s3_bucket;
 
 use std::{
     collections::HashMap,
-    fmt::{Debug, Display},
+    fmt::Debug,
     num::{NonZeroU32, NonZeroUsize},
     ops::Deref,
     path::{Path, PathBuf},
@@ -41,44 +41,27 @@ pub const DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT: usize = 100;
 
 const REMOTE_STORAGE_PREFIX_SEPARATOR: char = '/';
 
-#[derive(Clone, PartialEq, Eq)]
-pub struct RemoteObjectId(String);
+/// Path on the remote storage, relative to some inner prefix.
+/// The prefix is an implementation detail, that allows representing local paths
+/// as the remote ones, stripping the local storage prefix away.
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct RemotePath(PathBuf);
+
+impl RemotePath {
+    pub fn new(relative_path: &Path) -> anyhow::Result<Self> {
+        anyhow::ensure!(
+            relative_path.is_relative(),
+            "Path {relative_path:?} is not relative"
+        );
+        Ok(Self(relative_path.to_path_buf()))
+    }
+
+    pub fn with_base(&self, base_path: &Path) -> PathBuf {
+        base_path.join(&self.0)
+    }
 
-///
-/// A key that refers to an object in remote storage. It works much like a Path,
-/// but it's a separate datatype so that you don't accidentally mix local paths
-/// and remote keys.
-///
-impl RemoteObjectId {
-    // Needed to retrieve last component for RemoteObjectId.
-    // In other words a file name
-    /// Turn a/b/c or a/b/c/ into c
     pub fn object_name(&self) -> Option<&str> {
-        // corner case, char::to_string is not const, thats why this is more verbose than it needs to be
-        // see https://github.com/rust-lang/rust/issues/88674
-        if self.0.len() == 1 && self.0.chars().next().unwrap() == REMOTE_STORAGE_PREFIX_SEPARATOR {
-            return None;
-        }
-
-        if self.0.ends_with(REMOTE_STORAGE_PREFIX_SEPARATOR) {
-            self.0.rsplit(REMOTE_STORAGE_PREFIX_SEPARATOR).nth(1)
-        } else {
-            self.0
-                .rsplit_once(REMOTE_STORAGE_PREFIX_SEPARATOR)
-                .map(|(_, last)| last)
-        }
-    }
-}
-
-impl Debug for RemoteObjectId {
-    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
-        Debug::fmt(&self.0, fmt)
-    }
-}
-
-impl Display for RemoteObjectId {
-    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        Display::fmt(&self.0, fmt)
+        self.0.file_name().and_then(|os_str| os_str.to_str())
     }
 }
 
@@ -87,49 +70,40 @@ impl Display for RemoteObjectId {
 /// providing basic CRUD operations for storage files.
 #[async_trait::async_trait]
 pub trait RemoteStorage: Send + Sync + 'static {
-    /// Attempts to derive the storage path out of the local path, if the latter is correct.
-    fn remote_object_id(&self, local_path: &Path) -> anyhow::Result<RemoteObjectId>;
-
-    /// Gets the download path of the given storage file.
-    fn local_path(&self, remote_object_id: &RemoteObjectId) -> anyhow::Result<PathBuf>;
-
     /// Lists all items the storage has right now.
-    async fn list(&self) -> anyhow::Result<Vec<RemoteObjectId>>;
+    async fn list(&self) -> anyhow::Result<Vec<RemotePath>>;
 
     /// Lists all top level subdirectories for a given prefix
     /// Note: here we assume that if the prefix is passed it was obtained via remote_object_id
     /// which already takes into account any kind of global prefix (prefix_in_bucket for S3 or storage_root for LocalFS)
     /// so this method doesnt need to.
-    async fn list_prefixes(
-        &self,
-        prefix: Option<&RemoteObjectId>,
-    ) -> anyhow::Result<Vec<RemoteObjectId>>;
+    async fn list_prefixes(&self, prefix: Option<&RemotePath>) -> anyhow::Result<Vec<RemotePath>>;
 
     /// Streams the local file contents into remote into the remote storage entry.
     async fn upload(
         &self,
-        from: Box<(dyn io::AsyncRead + Unpin + Send + Sync + 'static)>,
+        data: Box<(dyn io::AsyncRead + Unpin + Send + Sync + 'static)>,
         // S3 PUT request requires the content length to be specified,
         // otherwise it starts to fail with the concurrent connection count increasing.
-        from_size_bytes: usize,
-        to: &RemoteObjectId,
+        data_size_bytes: usize,
+        to: &RemotePath,
         metadata: Option<StorageMetadata>,
     ) -> anyhow::Result<()>;
 
     /// Streams the remote storage entry contents into the buffered writer given, returns the filled writer.
     /// Returns the metadata, if any was stored with the file previously.
-    async fn download(&self, from: &RemoteObjectId) -> Result<Download, DownloadError>;
+    async fn download(&self, from: &RemotePath) -> Result<Download, DownloadError>;
 
     /// Streams a given byte range of the remote storage entry contents into the buffered writer given, returns the filled writer.
     /// Returns the metadata, if any was stored with the file previously.
     async fn download_byte_range(
         &self,
-        from: &RemoteObjectId,
+        from: &RemotePath,
         start_inclusive: u64,
         end_exclusive: Option<u64>,
     ) -> Result<Download, DownloadError>;
 
-    async fn delete(&self, path: &RemoteObjectId) -> anyhow::Result<()>;
+    async fn delete(&self, path: &RemotePath) -> anyhow::Result<()>;
 
     /// Downcast to LocalFs implementation. For tests.
     fn as_local(&self) -> Option<&LocalFs> {
@@ -178,34 +152,35 @@ impl std::error::Error for DownloadError {}
 /// Every storage, currently supported.
 /// Serves as a simple way to pass around the [`RemoteStorage`] without dealing with generics.
 #[derive(Clone)]
-pub struct GenericRemoteStorage(Arc<dyn RemoteStorage>);
+pub enum GenericRemoteStorage {
+    LocalFs(LocalFs),
+    AwsS3(Arc<S3Bucket>),
+}
 
 impl Deref for GenericRemoteStorage {
     type Target = dyn RemoteStorage;
 
     fn deref(&self) -> &Self::Target {
-        self.0.as_ref()
+        match self {
+            GenericRemoteStorage::LocalFs(local_fs) => local_fs,
+            GenericRemoteStorage::AwsS3(s3_bucket) => s3_bucket.as_ref(),
+        }
     }
 }
 
 impl GenericRemoteStorage {
-    pub fn new(storage: impl RemoteStorage) -> Self {
-        Self(Arc::new(storage))
-    }
-
     pub fn from_config(
-        working_directory: PathBuf,
         storage_config: &RemoteStorageConfig,
     ) -> anyhow::Result<GenericRemoteStorage> {
         Ok(match &storage_config.storage {
             RemoteStorageKind::LocalFs(root) => {
                 info!("Using fs root '{}' as a remote storage", root.display());
-                GenericRemoteStorage::new(LocalFs::new(root.clone(), working_directory)?)
+                GenericRemoteStorage::LocalFs(LocalFs::new(root.clone())?)
             }
             RemoteStorageKind::AwsS3(s3_config) => {
                 info!("Using s3 bucket '{}' in region '{}' as a remote storage, prefix in bucket: '{:?}', bucket endpoint: '{:?}'",
                       s3_config.bucket_name, s3_config.bucket_region, s3_config.prefix_in_bucket, s3_config.endpoint);
-                GenericRemoteStorage::new(S3Bucket::new(s3_config, working_directory)?)
+                GenericRemoteStorage::AwsS3(Arc::new(S3Bucket::new(s3_config)?))
             }
         })
     }
@@ -219,23 +194,12 @@ impl GenericRemoteStorage {
         &self,
         from: Box<dyn tokio::io::AsyncRead + Unpin + Send + Sync + 'static>,
         from_size_bytes: usize,
-        from_path: &Path,
+        to: &RemotePath,
     ) -> anyhow::Result<()> {
-        let target_storage_path = self.remote_object_id(from_path).with_context(|| {
-            format!(
-                "Failed to get the storage path for source local path '{}'",
-                from_path.display()
-            )
-        })?;
-
-        self.upload(from, from_size_bytes, &target_storage_path, None)
+        self.upload(from, from_size_bytes, to, None)
             .await
             .with_context(|| {
-                format!(
-                    "Failed to upload from '{}' to storage path '{:?}'",
-                    from_path.display(),
-                    target_storage_path
-                )
+                format!("Failed to upload data of length {from_size_bytes} to storage path {to:?}")
             })
     }
 
@@ -244,24 +208,11 @@ impl GenericRemoteStorage {
     pub async fn download_storage_object(
         &self,
         byte_range: Option<(u64, Option<u64>)>,
-        to_path: &Path,
+        from: &RemotePath,
     ) -> Result<Download, DownloadError> {
-        let remote_object_path = self
-            .remote_object_id(to_path)
-            .with_context(|| {
-                format!(
-                    "Failed to get the storage path for target local path '{}'",
-                    to_path.display()
-                )
-            })
-            .map_err(DownloadError::BadInput)?;
-
         match byte_range {
-            Some((start, end)) => {
-                self.download_byte_range(&remote_object_path, start, end)
-                    .await
-            }
-            None => self.download(&remote_object_path).await,
+            Some((start, end)) => self.download_byte_range(from, start, end).await,
+            None => self.download(from).await,
         }
     }
 }
@@ -271,23 +222,6 @@ impl GenericRemoteStorage {
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct StorageMetadata(HashMap<String, String>);
 
-fn strip_path_prefix<'a>(prefix: &'a Path, path: &'a Path) -> anyhow::Result<&'a Path> {
-    if prefix == path {
-        anyhow::bail!(
-            "Prefix and the path are equal, cannot strip: '{}'",
-            prefix.display()
-        )
-    } else {
-        path.strip_prefix(prefix).with_context(|| {
-            format!(
-                "Path '{}' is not prefixed with '{}'",
-                path.display(),
-                prefix.display(),
-            )
-        })
-    }
-}
-
 /// External backup storage configuration, enough for creating a client for that storage.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct RemoteStorageConfig {
@@ -431,21 +365,24 @@ mod tests {
     use super::*;
 
     #[test]
-    fn object_name() {
-        let k = RemoteObjectId("a/b/c".to_owned());
+    fn test_object_name() {
+        let k = RemotePath::new(Path::new("a/b/c")).unwrap();
         assert_eq!(k.object_name(), Some("c"));
 
-        let k = RemoteObjectId("a/b/c/".to_owned());
+        let k = RemotePath::new(Path::new("a/b/c/")).unwrap();
         assert_eq!(k.object_name(), Some("c"));
 
-        let k = RemoteObjectId("a/".to_owned());
+        let k = RemotePath::new(Path::new("a/")).unwrap();
         assert_eq!(k.object_name(), Some("a"));
 
         // XXX is it impossible to have an empty key?
-        let k = RemoteObjectId("".to_owned());
-        assert_eq!(k.object_name(), None);
-
-        let k = RemoteObjectId("/".to_owned());
+        let k = RemotePath::new(Path::new("")).unwrap();
         assert_eq!(k.object_name(), None);
     }
+
+    #[test]
+    fn rempte_path_cannot_be_created_from_absolute_ones() {
+        let err = RemotePath::new(Path::new("/")).expect_err("Should fail on absolute paths");
+        assert_eq!(err.to_string(), "Path \"/\" is not relative");
+    }
 }

libs/remote_storage/src/local_fs.rs

@@ -5,6 +5,7 @@
 //! volume is mounted to the local FS.
 
 use std::{
+    borrow::Cow,
     future::Future,
     path::{Path, PathBuf},
     pin::Pin,
@@ -18,60 +19,33 @@ use tokio::{
 use tracing::*;
 use utils::crashsafe::path_with_suffix_extension;
 
-use crate::{Download, DownloadError, RemoteObjectId};
+use crate::{Download, DownloadError, RemotePath};
 
-use super::{strip_path_prefix, RemoteStorage, StorageMetadata};
+use super::{RemoteStorage, StorageMetadata};
 
 const LOCAL_FS_TEMP_FILE_SUFFIX: &str = "___temp";
 
-/// Convert a Path in the remote storage into a RemoteObjectId
-fn remote_object_id_from_path(path: &Path) -> anyhow::Result<RemoteObjectId> {
-    Ok(RemoteObjectId(
-        path.to_str()
-            .ok_or_else(|| anyhow::anyhow!("unexpected characters found in path"))?
-            .to_string(),
-    ))
-}
-
+#[derive(Debug, Clone)]
 pub struct LocalFs {
-    working_directory: PathBuf,
     storage_root: PathBuf,
 }
 
 impl LocalFs {
     /// Attempts to create local FS storage, along with its root directory.
-    pub fn new(root: PathBuf, working_directory: PathBuf) -> anyhow::Result<Self> {
-        if !root.exists() {
-            std::fs::create_dir_all(&root).with_context(|| {
-                format!(
-                    "Failed to create all directories in the given root path '{}'",
-                    root.display(),
-                )
+    /// Storage root will be created (if does not exist) and transformed into an absolute path (if passed as relative).
+    pub fn new(mut storage_root: PathBuf) -> anyhow::Result<Self> {
+        if !storage_root.exists() {
+            std::fs::create_dir_all(&storage_root).with_context(|| {
+                format!("Failed to create all directories in the given root path {storage_root:?}")
             })?;
         }
-        Ok(Self {
-            working_directory,
-            storage_root: root,
-        })
-    }
-
-    ///
-    /// Get the absolute path in the local filesystem to given remote object.
-    ///
-    /// This is public so that it can be used in tests. Should not be used elsewhere.
-    ///
-    pub fn resolve_in_storage(&self, remote_object_id: &RemoteObjectId) -> anyhow::Result<PathBuf> {
-        let path = PathBuf::from(&remote_object_id.0);
-        if path.is_relative() {
-            Ok(self.storage_root.join(path))
-        } else if path.starts_with(&self.storage_root) {
-            Ok(path)
-        } else {
-            bail!(
-                "Path '{}' does not belong to the current storage",
-                path.display()
-            )
+        if !storage_root.is_absolute() {
+            storage_root = storage_root.canonicalize().with_context(|| {
+                format!("Failed to represent path {storage_root:?} as an absolute path")
+            })?;
         }
+
+        Ok(Self { storage_root })
     }
 
     async fn read_storage_metadata(
@@ -103,45 +77,48 @@ impl LocalFs {
 
 #[async_trait::async_trait]
 impl RemoteStorage for LocalFs {
-    /// Convert a "local" path into a "remote path"
-    fn remote_object_id(&self, local_path: &Path) -> anyhow::Result<RemoteObjectId> {
-        let path = self.storage_root.join(
-            strip_path_prefix(&self.working_directory, local_path)
-                .context("local path does not belong to this storage")?,
-        );
-        remote_object_id_from_path(&path)
+    async fn list(&self) -> anyhow::Result<Vec<RemotePath>> {
+        Ok(get_all_files(&self.storage_root, true)
+            .await?
+            .into_iter()
+            .map(|path| {
+                path.strip_prefix(&self.storage_root)
+                    .context("Failed to strip storage root prefix")
+                    .and_then(RemotePath::new)
+                    .expect(
+                        "We list files for storage root, hence should be able to remote the prefix",
+                    )
+            })
+            .collect())
     }
 
-    fn local_path(&self, remote_object_id: &RemoteObjectId) -> anyhow::Result<PathBuf> {
-        let storage_path = PathBuf::from(&remote_object_id.0);
-        let relative_path = strip_path_prefix(&self.storage_root, &storage_path)
-            .context("local path does not belong to this storage")?;
-        Ok(self.working_directory.join(relative_path))
-    }
-
-    async fn list(&self) -> anyhow::Result<Vec<RemoteObjectId>> {
-        get_all_files(&self.storage_root, true).await
-    }
-
-    async fn list_prefixes(
-        &self,
-        prefix: Option<&RemoteObjectId>,
-    ) -> anyhow::Result<Vec<RemoteObjectId>> {
+    async fn list_prefixes(&self, prefix: Option<&RemotePath>) -> anyhow::Result<Vec<RemotePath>> {
         let path = match prefix {
-            Some(prefix) => Path::new(&prefix.0),
-            None => &self.storage_root,
+            Some(prefix) => Cow::Owned(prefix.with_base(&self.storage_root)),
+            None => Cow::Borrowed(&self.storage_root),
         };
-        get_all_files(path, false).await
+        Ok(get_all_files(path.as_ref(), false)
+            .await?
+            .into_iter()
+            .map(|path| {
+                path.strip_prefix(&self.storage_root)
+                    .context("Failed to strip preifix")
+                    .and_then(RemotePath::new)
+                    .expect(
+                        "We list files for storage root, hence should be able to remote the prefix",
+                    )
+            })
+            .collect())
     }
 
     async fn upload(
         &self,
-        from: Box<(dyn io::AsyncRead + Unpin + Send + Sync + 'static)>,
-        from_size_bytes: usize,
-        to: &RemoteObjectId,
+        data: Box<(dyn io::AsyncRead + Unpin + Send + Sync + 'static)>,
+        data_size_bytes: usize,
+        to: &RemotePath,
         metadata: Option<StorageMetadata>,
     ) -> anyhow::Result<()> {
-        let target_file_path = self.resolve_in_storage(to)?;
+        let target_file_path = to.with_base(&self.storage_root);
         create_target_directory(&target_file_path).await?;
         // We need this dance with sort of durable rename (without fsyncs)
         // to prevent partial uploads. This was really hit when pageserver shutdown
@@ -162,8 +139,8 @@ impl RemoteStorage for LocalFs {
                 })?,
         );
 
-        let from_size_bytes = from_size_bytes as u64;
-        let mut buffer_to_read = from.take(from_size_bytes);
+        let from_size_bytes = data_size_bytes as u64;
+        let mut buffer_to_read = data.take(from_size_bytes);
 
         let bytes_read = io::copy(&mut buffer_to_read, &mut destination)
             .await
@@ -220,27 +197,22 @@ impl RemoteStorage for LocalFs {
         Ok(())
     }
 
-    async fn download(&self, from: &RemoteObjectId) -> Result<Download, DownloadError> {
-        let file_path = self
-            .resolve_in_storage(from)
-            .map_err(DownloadError::BadInput)?;
-        if file_exists(&file_path).map_err(DownloadError::BadInput)? {
+    async fn download(&self, from: &RemotePath) -> Result<Download, DownloadError> {
+        let target_path = from.with_base(&self.storage_root);
+        if file_exists(&target_path).map_err(DownloadError::BadInput)? {
             let source = io::BufReader::new(
                 fs::OpenOptions::new()
                     .read(true)
-                    .open(&file_path)
+                    .open(&target_path)
                     .await
                     .with_context(|| {
-                        format!(
-                            "Failed to open source file '{}' to use in the download",
-                            file_path.display()
-                        )
+                        format!("Failed to open source file {target_path:?} to use in the download")
                     })
                     .map_err(DownloadError::Other)?,
             );
 
             let metadata = self
-                .read_storage_metadata(&file_path)
+                .read_storage_metadata(&target_path)
                 .await
                 .map_err(DownloadError::Other)?;
             Ok(Download {
@@ -254,7 +226,7 @@ impl RemoteStorage for LocalFs {
 
     async fn download_byte_range(
         &self,
-        from: &RemoteObjectId,
+        from: &RemotePath,
         start_inclusive: u64,
         end_exclusive: Option<u64>,
     ) -> Result<Download, DownloadError> {
@@ -266,20 +238,15 @@ impl RemoteStorage for LocalFs {
                 return Err(DownloadError::Other(anyhow::anyhow!("Invalid range, start ({start_inclusive}) and end_exclusive ({end_exclusive:?}) difference is zero bytes")));
             }
         }
-        let file_path = self
-            .resolve_in_storage(from)
-            .map_err(DownloadError::BadInput)?;
-        if file_exists(&file_path).map_err(DownloadError::BadInput)? {
+        let target_path = from.with_base(&self.storage_root);
+        if file_exists(&target_path).map_err(DownloadError::BadInput)? {
             let mut source = io::BufReader::new(
                 fs::OpenOptions::new()
                     .read(true)
-                    .open(&file_path)
+                    .open(&target_path)
                     .await
                     .with_context(|| {
-                        format!(
-                            "Failed to open source file '{}' to use in the download",
-                            file_path.display()
-                        )
+                        format!("Failed to open source file {target_path:?} to use in the download")
                     })
                     .map_err(DownloadError::Other)?,
             );
@@ -289,7 +256,7 @@ impl RemoteStorage for LocalFs {
                 .context("Failed to seek to the range start in a local storage file")
                 .map_err(DownloadError::Other)?;
             let metadata = self
-                .read_storage_metadata(&file_path)
+                .read_storage_metadata(&target_path)
                 .await
                 .map_err(DownloadError::Other)?;
 
@@ -308,15 +275,12 @@ impl RemoteStorage for LocalFs {
         }
     }
 
-    async fn delete(&self, path: &RemoteObjectId) -> anyhow::Result<()> {
-        let file_path = self.resolve_in_storage(path)?;
+    async fn delete(&self, path: &RemotePath) -> anyhow::Result<()> {
+        let file_path = path.with_base(&self.storage_root);
         if file_path.exists() && file_path.is_file() {
             Ok(fs::remove_file(file_path).await?)
         } else {
-            bail!(
-                "File '{}' either does not exist or is not a file",
-                file_path.display()
-            )
+            bail!("File {file_path:?} either does not exist or is not a file")
         }
     }
 
@@ -332,7 +296,7 @@ fn storage_metadata_path(original_path: &Path) -> PathBuf {
 fn get_all_files<'a, P>(
     directory_path: P,
     recursive: bool,
-) -> Pin<Box<dyn Future<Output = anyhow::Result<Vec<RemoteObjectId>>> + Send + Sync + 'a>>
+) -> Pin<Box<dyn Future<Output = anyhow::Result<Vec<PathBuf>>> + Send + Sync + 'a>>
 where
     P: AsRef<Path> + Send + Sync + 'a,
 {
@@ -346,20 +310,20 @@ where
                     let file_type = dir_entry.file_type().await?;
                     let entry_path = dir_entry.path();
                     if file_type.is_symlink() {
-                        debug!("{:?} us a symlink, skipping", entry_path)
+                        debug!("{entry_path:?} us a symlink, skipping")
                     } else if file_type.is_dir() {
                         if recursive {
                             paths.extend(get_all_files(&entry_path, true).await?.into_iter())
                         } else {
-                            paths.push(remote_object_id_from_path(&dir_entry.path())?)
+                            paths.push(entry_path)
                         }
                     } else {
-                        paths.push(remote_object_id_from_path(&dir_entry.path())?);
+                        paths.push(entry_path);
                     }
                 }
                 Ok(paths)
             } else {
-                bail!("Path '{}' is not a directory", directory_path.display())
+                bail!("Path {directory_path:?} is not a directory")
             }
         } else {
             Ok(Vec::new())
@@ -394,173 +358,6 @@ fn file_exists(file_path: &Path) -> anyhow::Result<bool> {
     }
 }
 
-#[cfg(test)]
-mod pure_tests {
-    use tempfile::tempdir;
-
-    use super::*;
-
-    #[test]
-    fn storage_path_positive() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
-        let storage_root = PathBuf::from("somewhere").join("else");
-        let storage = LocalFs {
-            working_directory: workdir.clone(),
-            storage_root: storage_root.clone(),
-        };
-
-        let local_path = workdir
-            .join("timelines")
-            .join("some_timeline")
-            .join("file_name");
-        let expected_path = storage_root.join(local_path.strip_prefix(&workdir)?);
-
-        let actual_path = PathBuf::from(
-            storage
-                .remote_object_id(&local_path)
-                .expect("Matching path should map to storage path normally")
-                .0,
-        );
-        assert_eq!(
-            expected_path,
-            actual_path,
-            "File paths from workdir should be stored in local fs storage with the same path they have relative to the workdir"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn storage_path_negatives() -> anyhow::Result<()> {
-        #[track_caller]
-        fn storage_path_error(storage: &LocalFs, mismatching_path: &Path) -> String {
-            match storage.remote_object_id(mismatching_path) {
-                Ok(wrong_path) => panic!(
-                    "Expected path '{}' to error, but got storage path: {:?}",
-                    mismatching_path.display(),
-                    wrong_path,
-                ),
-                Err(e) => format!("{:?}", e),
-            }
-        }
-
-        let workdir = tempdir()?.path().to_owned();
-        let storage_root = PathBuf::from("somewhere").join("else");
-        let storage = LocalFs {
-            working_directory: workdir.clone(),
-            storage_root,
-        };
-
-        let error_string = storage_path_error(&storage, &workdir);
-        assert!(error_string.contains("does not belong to this storage"));
-        assert!(error_string.contains(workdir.to_str().unwrap()));
-
-        let mismatching_path_str = "/something/else";
-        let error_message = storage_path_error(&storage, Path::new(mismatching_path_str));
-        assert!(
-            error_message.contains(mismatching_path_str),
-            "Error should mention wrong path"
-        );
-        assert!(
-            error_message.contains(workdir.to_str().unwrap()),
-            "Error should mention server workdir"
-        );
-        assert!(error_message.contains("does not belong to this storage"));
-
-        Ok(())
-    }
-
-    #[test]
-    fn local_path_positive() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-        let storage_root = PathBuf::from("somewhere").join("else");
-        let storage = LocalFs {
-            working_directory: workdir.clone(),
-            storage_root: storage_root.clone(),
-        };
-
-        let name = "not a metadata";
-        let local_path = workdir.join("timelines").join("some_timeline").join(name);
-        assert_eq!(
-            local_path,
-            storage
-                .local_path(&remote_object_id_from_path(
-                    &storage_root.join(local_path.strip_prefix(&workdir)?)
-                )?)
-                .expect("For a valid input, valid local path should be parsed"),
-            "Should be able to parse metadata out of the correctly named remote delta file"
-        );
-
-        let local_metadata_path = workdir
-            .join("timelines")
-            .join("some_timeline")
-            .join("metadata");
-        let remote_metadata_path = storage.remote_object_id(&local_metadata_path)?;
-        assert_eq!(
-            local_metadata_path,
-            storage
-                .local_path(&remote_metadata_path)
-                .expect("For a valid input, valid local path should be parsed"),
-            "Should be able to parse metadata out of the correctly named remote metadata file"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn local_path_negatives() -> anyhow::Result<()> {
-        #[track_caller]
-        fn local_path_error(storage: &LocalFs, storage_path: &RemoteObjectId) -> String {
-            match storage.local_path(storage_path) {
-                Ok(wrong_path) => panic!(
-                    "Expected local path input {:?} to cause an error, but got file path: {:?}",
-                    storage_path, wrong_path,
-                ),
-                Err(e) => format!("{:?}", e),
-            }
-        }
-
-        let storage_root = PathBuf::from("somewhere").join("else");
-        let storage = LocalFs {
-            working_directory: tempdir()?.path().to_owned(),
-            storage_root,
-        };
-
-        let totally_wrong_path = "wrong_wrong_wrong";
-        let error_message =
-            local_path_error(&storage, &RemoteObjectId(totally_wrong_path.to_string()));
-        assert!(error_message.contains(totally_wrong_path));
-
-        Ok(())
-    }
-
-    #[test]
-    fn download_destination_matches_original_path() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-        let original_path = workdir
-            .join("timelines")
-            .join("some_timeline")
-            .join("some name");
-
-        let storage_root = PathBuf::from("somewhere").join("else");
-        let dummy_storage = LocalFs {
-            working_directory: workdir,
-            storage_root,
-        };
-
-        let storage_path = dummy_storage.remote_object_id(&original_path)?;
-        let download_destination = dummy_storage.local_path(&storage_path)?;
-
-        assert_eq!(
-            original_path, download_destination,
-            "'original path -> storage path -> matching fs path' transformation should produce the same path as the input one for the correct path"
-        );
-
-        Ok(())
-    }
-}
-
 #[cfg(test)]
 mod fs_tests {
     use super::*;
@@ -572,7 +369,7 @@ mod fs_tests {
         storage: &LocalFs,
         #[allow(clippy::ptr_arg)]
         // have to use &PathBuf due to `storage.local_path` parameter requirements
-        remote_storage_path: &RemoteObjectId,
+        remote_storage_path: &RemotePath,
         expected_metadata: Option<&StorageMetadata>,
     ) -> anyhow::Result<String> {
         let mut download = storage
@@ -595,41 +392,16 @@ mod fs_tests {
 
     #[tokio::test]
     async fn upload_file() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
         let storage = create_storage()?;
 
-        let (file, size) = create_file_for_upload(
-            &storage.working_directory.join("whatever"),
-            "whatever_contents",
-        )
-        .await?;
-        let target_path = "/somewhere/else";
-        match storage
-            .upload(
-                Box::new(file),
-                size,
-                &RemoteObjectId(target_path.to_string()),
-                None,
-            )
-            .await
-        {
-            Ok(()) => panic!("Should not allow storing files with wrong target path"),
-            Err(e) => {
-                let message = format!("{:?}", e);
-                assert!(message.contains(target_path));
-                assert!(message.contains("does not belong to the current storage"));
-            }
-        }
-        assert!(storage.list().await?.is_empty());
-
-        let target_path_1 = upload_dummy_file(&workdir, &storage, "upload_1", None).await?;
+        let target_path_1 = upload_dummy_file(&storage, "upload_1", None).await?;
         assert_eq!(
             storage.list().await?,
             vec![target_path_1.clone()],
             "Should list a single file after first upload"
         );
 
-        let target_path_2 = upload_dummy_file(&workdir, &storage, "upload_2", None).await?;
+        let target_path_2 = upload_dummy_file(&storage, "upload_2", None).await?;
         assert_eq!(
             list_files_sorted(&storage).await?,
             vec![target_path_1.clone(), target_path_2.clone()],
@@ -643,7 +415,7 @@ mod fs_tests {
     async fn upload_file_negatives() -> anyhow::Result<()> {
         let storage = create_storage()?;
 
-        let id = storage.remote_object_id(&storage.working_directory.join("dummy"))?;
+        let id = RemotePath::new(Path::new("dummy"))?;
         let content = std::io::Cursor::new(b"12345");
 
         // Check that you get an error if the size parameter doesn't match the actual
@@ -668,16 +440,14 @@ mod fs_tests {
     }
 
     fn create_storage() -> anyhow::Result<LocalFs> {
-        LocalFs::new(tempdir()?.path().to_owned(), tempdir()?.path().to_owned())
+        LocalFs::new(tempdir()?.path().to_owned())
     }
 
     #[tokio::test]
     async fn download_file() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&workdir, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&storage, upload_name, None).await?;
 
         let contents = read_and_assert_remote_file_contents(&storage, &upload_target, None).await?;
         assert_eq!(
@@ -687,7 +457,7 @@ mod fs_tests {
         );
 
         let non_existing_path = "somewhere/else";
-        match storage.download(&RemoteObjectId(non_existing_path.to_string())).await {
+        match storage.download(&RemotePath::new(Path::new(non_existing_path))?).await {
             Err(DownloadError::NotFound) => {} // Should get NotFound for non existing keys
             other => panic!("Should get a NotFound error when downloading non-existing storage files, but got: {other:?}"),
         }
@@ -696,11 +466,9 @@ mod fs_tests {
 
     #[tokio::test]
     async fn download_file_range_positive() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&workdir, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&storage, upload_name, None).await?;
 
         let full_range_download_contents =
             read_and_assert_remote_file_contents(&storage, &upload_target, None).await?;
@@ -766,11 +534,9 @@ mod fs_tests {
 
     #[tokio::test]
     async fn download_file_range_negative() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&workdir, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&storage, upload_name, None).await?;
 
         let start = 1_000_000_000;
         let end = start + 1;
@@ -812,11 +578,9 @@ mod fs_tests {
 
     #[tokio::test]
     async fn delete_file() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&workdir, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&storage, upload_name, None).await?;
 
         storage.delete(&upload_target).await?;
         assert!(storage.list().await?.is_empty());
@@ -826,7 +590,8 @@ mod fs_tests {
             Err(e) => {
                 let error_string = e.to_string();
                 assert!(error_string.contains("does not exist"));
-                assert!(error_string.contains(&upload_target.0));
+                let expected_path = upload_target.with_base(&storage.storage_root);
+                assert!(error_string.contains(expected_path.to_str().unwrap()));
             }
         }
         Ok(())
@@ -834,8 +599,6 @@ mod fs_tests {
 
     #[tokio::test]
     async fn file_with_metadata() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
         let storage = create_storage()?;
         let upload_name = "upload_1";
         let metadata = StorageMetadata(HashMap::from([
@@ -843,7 +606,7 @@ mod fs_tests {
             ("two".to_string(), "2".to_string()),
         ]));
         let upload_target =
-            upload_dummy_file(&workdir, &storage, upload_name, Some(metadata.clone())).await?;
+            upload_dummy_file(&storage, upload_name, Some(metadata.clone())).await?;
 
         let full_range_download_contents =
             read_and_assert_remote_file_contents(&storage, &upload_target, Some(&metadata)).await?;
@@ -883,23 +646,32 @@ mod fs_tests {
     }
 
     async fn upload_dummy_file(
-        workdir: &Path,
         storage: &LocalFs,
         name: &str,
         metadata: Option<StorageMetadata>,
-    ) -> anyhow::Result<RemoteObjectId> {
-        let timeline_path = workdir.join("timelines").join("some_timeline");
-        let relative_timeline_path = timeline_path.strip_prefix(&workdir)?;
-        let storage_path = storage.storage_root.join(relative_timeline_path).join(name);
-        let remote_object_id = RemoteObjectId(storage_path.to_str().unwrap().to_string());
-
-        let from_path = storage.working_directory.join(name);
+    ) -> anyhow::Result<RemotePath> {
+        let from_path = storage
+            .storage_root
+            .join("timelines")
+            .join("some_timeline")
+            .join(name);
         let (file, size) = create_file_for_upload(&from_path, &dummy_contents(name)).await?;
 
+        let relative_path = from_path
+            .strip_prefix(&storage.storage_root)
+            .context("Failed to strip storage root prefix")
+            .and_then(RemotePath::new)
+            .with_context(|| {
+                format!(
+                    "Failed to resolve remote part of path {:?} for base {:?}",
+                    from_path, storage.storage_root
+                )
+            })?;
+
         storage
-            .upload(Box::new(file), size, &remote_object_id, metadata)
+            .upload(Box::new(file), size, &relative_path, metadata)
             .await?;
-        remote_object_id_from_path(&storage_path)
+        Ok(relative_path)
     }
 
     async fn create_file_for_upload(
@@ -924,7 +696,7 @@ mod fs_tests {
         format!("contents for {name}")
     }
 
-    async fn list_files_sorted(storage: &LocalFs) -> anyhow::Result<Vec<RemoteObjectId>> {
+    async fn list_files_sorted(storage: &LocalFs) -> anyhow::Result<Vec<RemotePath>> {
         let mut files = storage.list().await?;
         files.sort_by(|a, b| a.0.cmp(&b.0));
         Ok(files)

libs/remote_storage/src/s3_bucket.rs

@@ -4,27 +4,34 @@
 //! allowing multiple api users to independently work with the same S3 bucket, if
 //! their bucket prefixes are both specified and different.
 
-use std::path::{Path, PathBuf};
+use std::env::var;
+use std::sync::Arc;
+use std::time::Duration;
 
 use anyhow::Context;
-use rusoto_core::{
-    credential::{InstanceMetadataProvider, StaticProvider},
-    HttpClient, Region, RusotoError,
+use aws_config::{
+    environment::credentials::EnvironmentVariableCredentialsProvider, imds,
+    imds::credentials::ImdsCredentialsProvider, meta::credentials::provide_credentials_fn,
 };
-use rusoto_s3::{
-    DeleteObjectRequest, GetObjectError, GetObjectRequest, ListObjectsV2Request, PutObjectRequest,
-    S3Client, StreamingBody, S3,
+use aws_sdk_s3::{
+    config::Config,
+    error::{GetObjectError, GetObjectErrorKind},
+    types::{ByteStream, SdkError},
+    Client, Endpoint, Region,
 };
+use aws_smithy_http::body::SdkBody;
+use aws_types::credentials::{CredentialsError, ProvideCredentials};
+use hyper::Body;
 use tokio::{io, sync::Semaphore};
 use tokio_util::io::ReaderStream;
 use tracing::debug;
 
+use super::StorageMetadata;
 use crate::{
-    strip_path_prefix, Download, DownloadError, RemoteObjectId, RemoteStorage, S3Config,
-    REMOTE_STORAGE_PREFIX_SEPARATOR,
+    Download, DownloadError, RemotePath, RemoteStorage, S3Config, REMOTE_STORAGE_PREFIX_SEPARATOR,
 };
 
-use super::StorageMetadata;
+const DEFAULT_IMDS_TIMEOUT: Duration = Duration::from_secs(10);
 
 pub(super) mod metrics {
     use metrics::{register_int_counter_vec, IntCounterVec};
@@ -91,32 +98,9 @@ pub(super) mod metrics {
     }
 }
 
-fn download_destination(
-    id: &RemoteObjectId,
-    workdir: &Path,
-    prefix_to_strip: Option<&str>,
-) -> PathBuf {
-    let path_without_prefix = match prefix_to_strip {
-        Some(prefix) => id.0.strip_prefix(prefix).unwrap_or_else(|| {
-            panic!(
-                "Could not strip prefix '{}' from S3 object key '{}'",
-                prefix, id.0
-            )
-        }),
-        None => &id.0,
-    };
-
-    workdir.join(
-        path_without_prefix
-            .split(REMOTE_STORAGE_PREFIX_SEPARATOR)
-            .collect::<PathBuf>(),
-    )
-}
-
 /// AWS S3 storage.
 pub struct S3Bucket {
-    workdir: PathBuf,
-    client: S3Client,
+    client: Client,
     bucket_name: String,
     prefix_in_bucket: Option<String>,
     // Every request to S3 can be throttled or cancelled, if a certain number of requests per second is exceeded.
@@ -125,50 +109,53 @@ pub struct S3Bucket {
     concurrency_limiter: Semaphore,
 }
 
+#[derive(Default)]
+struct GetObjectRequest {
+    bucket: String,
+    key: String,
+    range: Option<String>,
+}
 impl S3Bucket {
     /// Creates the S3 storage, errors if incorrect AWS S3 configuration provided.
-    pub fn new(aws_config: &S3Config, workdir: PathBuf) -> anyhow::Result<Self> {
+    pub fn new(aws_config: &S3Config) -> anyhow::Result<Self> {
         debug!(
             "Creating s3 remote storage for S3 bucket {}",
             aws_config.bucket_name
         );
-        let region = match aws_config.endpoint.clone() {
-            Some(custom_endpoint) => Region::Custom {
-                name: aws_config.bucket_region.clone(),
-                endpoint: custom_endpoint,
-            },
-            None => aws_config
-                .bucket_region
-                .parse::<Region>()
-                .context("Failed to parse the s3 region from config")?,
-        };
-        let request_dispatcher = HttpClient::new().context("Failed to create S3 http client")?;
+        let mut config_builder = Config::builder()
+            .region(Region::new(aws_config.bucket_region.clone()))
+            .credentials_provider(provide_credentials_fn(|| async {
+                match var("AWS_ACCESS_KEY_ID").is_ok() && var("AWS_SECRET_ACCESS_KEY").is_ok() {
+                    true => {
+                        EnvironmentVariableCredentialsProvider::new()
+                            .provide_credentials()
+                            .await
+                    }
+                    false => {
+                        let imds_client = imds::Client::builder()
+                            .connect_timeout(DEFAULT_IMDS_TIMEOUT)
+                            .read_timeout(DEFAULT_IMDS_TIMEOUT)
+                            .build()
+                            .await
+                            .map_err(CredentialsError::unhandled)?;
+                        ImdsCredentialsProvider::builder()
+                            .imds_client(imds_client)
+                            .build()
+                            .provide_credentials()
+                            .await
+                    }
+                }
+            }));
 
-        let access_key_id = std::env::var("AWS_ACCESS_KEY_ID").ok();
-        let secret_access_key = std::env::var("AWS_SECRET_ACCESS_KEY").ok();
-        // session token is used when authorizing through sso
-        // which is typically the case when testing locally on developer machine
-        let session_token = std::env::var("AWS_SESSION_TOKEN").ok();
-
-        let client = if access_key_id.is_none() && secret_access_key.is_none() {
-            debug!("Using IAM-based AWS access");
-            S3Client::new_with(request_dispatcher, InstanceMetadataProvider::new(), region)
-        } else {
-            debug!(
-                "Using credentials-based AWS access. Session token is set: {}",
-                session_token.is_some()
+        if let Some(custom_endpoint) = aws_config.endpoint.clone() {
+            let endpoint = Endpoint::immutable(
+                custom_endpoint
+                    .parse()
+                    .expect("Failed to parse S3 custom endpoint"),
             );
-            S3Client::new_with(
-                request_dispatcher,
-                StaticProvider::new(
-                    access_key_id.unwrap_or_default(),
-                    secret_access_key.unwrap_or_default(),
-                    session_token,
-                    None,
-                ),
-                region,
-            )
-        };
+            config_builder.set_endpoint_resolver(Some(Arc::new(endpoint)));
+        }
+        let client = Client::from_conf(config_builder.build());
 
         let prefix_in_bucket = aws_config.prefix_in_bucket.as_deref().map(|prefix| {
             let mut prefix = prefix;
@@ -182,16 +169,41 @@ impl S3Bucket {
             }
             prefix
         });
-
         Ok(Self {
             client,
-            workdir,
             bucket_name: aws_config.bucket_name.clone(),
             prefix_in_bucket,
             concurrency_limiter: Semaphore::new(aws_config.concurrency_limit.get()),
         })
     }
 
+    fn s3_object_to_relative_path(&self, key: &str) -> RemotePath {
+        let relative_path =
+            match key.strip_prefix(self.prefix_in_bucket.as_deref().unwrap_or_default()) {
+                Some(stripped) => stripped,
+                // we rely on AWS to return properly prefixed paths
+                // for requests with a certain prefix
+                None => panic!(
+                    "Key {} does not start with bucket prefix {:?}",
+                    key, self.prefix_in_bucket
+                ),
+            };
+        RemotePath(
+            relative_path
+                .split(REMOTE_STORAGE_PREFIX_SEPARATOR)
+                .collect(),
+        )
+    }
+
+    fn relative_path_to_s3_object(&self, path: &RemotePath) -> String {
+        let mut full_path = self.prefix_in_bucket.clone().unwrap_or_default();
+        for segment in path.0.iter() {
+            full_path.push(REMOTE_STORAGE_PREFIX_SEPARATOR);
+            full_path.push_str(segment.to_str().unwrap_or_default());
+        }
+        full_path
+    }
+
     async fn download_object(&self, request: GetObjectRequest) -> Result<Download, DownloadError> {
         let _guard = self
             .concurrency_limiter
@@ -202,20 +214,33 @@ impl S3Bucket {
 
         metrics::inc_get_object();
 
-        match self.client.get_object(request).await {
-            Ok(object_output) => match object_output.body {
-                None => {
-                    metrics::inc_get_object_fail();
-                    Err(DownloadError::Other(anyhow::anyhow!(
-                        "Got no body for the S3 object given"
-                    )))
-                }
-                Some(body) => Ok(Download {
-                    metadata: object_output.metadata.map(StorageMetadata),
-                    download_stream: Box::pin(io::BufReader::new(body.into_async_read())),
-                }),
-            },
-            Err(RusotoError::Service(GetObjectError::NoSuchKey(_))) => Err(DownloadError::NotFound),
+        let get_object = self
+            .client
+            .get_object()
+            .bucket(request.bucket)
+            .key(request.key)
+            .set_range(request.range)
+            .send()
+            .await;
+
+        match get_object {
+            Ok(object_output) => {
+                let metadata = object_output.metadata().cloned().map(StorageMetadata);
+                Ok(Download {
+                    metadata,
+                    download_stream: Box::pin(io::BufReader::new(
+                        object_output.body.into_async_read(),
+                    )),
+                })
+            }
+            Err(SdkError::ServiceError {
+                err:
+                    GetObjectError {
+                        kind: GetObjectErrorKind::NoSuchKey(..),
+                        ..
+                    },
+                ..
+            }) => Err(DownloadError::NotFound),
             Err(e) => {
                 metrics::inc_get_object_fail();
                 Err(DownloadError::Other(anyhow::anyhow!(
@@ -228,25 +253,7 @@ impl S3Bucket {
 
 #[async_trait::async_trait]
 impl RemoteStorage for S3Bucket {
-    fn remote_object_id(&self, local_path: &Path) -> anyhow::Result<RemoteObjectId> {
-        let relative_path = strip_path_prefix(&self.workdir, local_path)?;
-        let mut key = self.prefix_in_bucket.clone().unwrap_or_default();
-        for segment in relative_path {
-            key.push(REMOTE_STORAGE_PREFIX_SEPARATOR);
-            key.push_str(&segment.to_string_lossy());
-        }
-        Ok(RemoteObjectId(key))
-    }
-
-    fn local_path(&self, storage_path: &RemoteObjectId) -> anyhow::Result<PathBuf> {
-        Ok(download_destination(
-            storage_path,
-            &self.workdir,
-            self.prefix_in_bucket.as_deref(),
-        ))
-    }
-
-    async fn list(&self) -> anyhow::Result<Vec<RemoteObjectId>> {
+    async fn list(&self) -> anyhow::Result<Vec<RemotePath>> {
         let mut document_keys = Vec::new();
 
         let mut continuation_token = None;
@@ -261,12 +268,11 @@ impl RemoteStorage for S3Bucket {
 
             let fetch_response = self
                 .client
-                .list_objects_v2(ListObjectsV2Request {
-                    bucket: self.bucket_name.clone(),
-                    prefix: self.prefix_in_bucket.clone(),
-                    continuation_token,
-                    ..ListObjectsV2Request::default()
-                })
+                .list_objects_v2()
+                .bucket(self.bucket_name.clone())
+                .set_prefix(self.prefix_in_bucket.clone())
+                .set_continuation_token(continuation_token)
+                .send()
                 .await
                 .map_err(|e| {
                     metrics::inc_list_objects_fail();
@@ -277,7 +283,7 @@ impl RemoteStorage for S3Bucket {
                     .contents
                     .unwrap_or_default()
                     .into_iter()
-                    .filter_map(|o| Some(RemoteObjectId(o.key?))),
+                    .filter_map(|o| Some(self.s3_object_to_relative_path(o.key()?))),
             );
 
             match fetch_response.continuation_token {
@@ -291,13 +297,10 @@ impl RemoteStorage for S3Bucket {
 
     /// See the doc for `RemoteStorage::list_prefixes`
     /// Note: it wont include empty "directories"
-    async fn list_prefixes(
-        &self,
-        prefix: Option<&RemoteObjectId>,
-    ) -> anyhow::Result<Vec<RemoteObjectId>> {
+    async fn list_prefixes(&self, prefix: Option<&RemotePath>) -> anyhow::Result<Vec<RemotePath>> {
         // get the passed prefix or if it is not set use prefix_in_bucket value
         let list_prefix = prefix
-            .map(|p| p.0.clone())
+            .map(|p| self.relative_path_to_s3_object(p))
             .or_else(|| self.prefix_in_bucket.clone())
             .map(|mut p| {
                 // required to end with a separator
@@ -322,13 +325,12 @@ impl RemoteStorage for S3Bucket {
 
             let fetch_response = self
                 .client
-                .list_objects_v2(ListObjectsV2Request {
-                    bucket: self.bucket_name.clone(),
-                    prefix: list_prefix.clone(),
-                    continuation_token,
-                    delimiter: Some(REMOTE_STORAGE_PREFIX_SEPARATOR.to_string()),
-                    ..ListObjectsV2Request::default()
-                })
+                .list_objects_v2()
+                .bucket(self.bucket_name.clone())
+                .set_prefix(list_prefix.clone())
+                .set_continuation_token(continuation_token)
+                .delimiter(REMOTE_STORAGE_PREFIX_SEPARATOR.to_string())
+                .send()
                 .await
                 .map_err(|e| {
                     metrics::inc_list_objects_fail();
@@ -340,7 +342,7 @@ impl RemoteStorage for S3Bucket {
                     .common_prefixes
                     .unwrap_or_default()
                     .into_iter()
-                    .filter_map(|o| Some(RemoteObjectId(o.prefix?))),
+                    .filter_map(|o| Some(self.s3_object_to_relative_path(o.prefix()?))),
             );
 
             match fetch_response.continuation_token {
@@ -356,7 +358,7 @@ impl RemoteStorage for S3Bucket {
         &self,
         from: Box<(dyn io::AsyncRead + Unpin + Send + Sync + 'static)>,
         from_size_bytes: usize,
-        to: &RemoteObjectId,
+        to: &RemotePath,
         metadata: Option<StorageMetadata>,
     ) -> anyhow::Result<()> {
         let _guard = self
@@ -366,17 +368,18 @@ impl RemoteStorage for S3Bucket {
             .context("Concurrency limiter semaphore got closed during S3 upload")?;
 
         metrics::inc_put_object();
+
+        let body = Body::wrap_stream(ReaderStream::new(from));
+        let bytes_stream = ByteStream::new(SdkBody::from(body));
+
         self.client
-            .put_object(PutObjectRequest {
-                body: Some(StreamingBody::new_with_size(
-                    ReaderStream::new(from),
-                    from_size_bytes,
-                )),
-                bucket: self.bucket_name.clone(),
-                key: to.0.to_owned(),
-                metadata: metadata.map(|m| m.0),
-                ..PutObjectRequest::default()
-            })
+            .put_object()
+            .bucket(self.bucket_name.clone())
+            .key(self.relative_path_to_s3_object(to))
+            .set_metadata(metadata.map(|m| m.0))
+            .content_length(from_size_bytes.try_into()?)
+            .body(bytes_stream)
+            .send()
             .await
             .map_err(|e| {
                 metrics::inc_put_object_fail();
@@ -385,10 +388,10 @@ impl RemoteStorage for S3Bucket {
         Ok(())
     }
 
-    async fn download(&self, from: &RemoteObjectId) -> Result<Download, DownloadError> {
+    async fn download(&self, from: &RemotePath) -> Result<Download, DownloadError> {
         self.download_object(GetObjectRequest {
             bucket: self.bucket_name.clone(),
-            key: from.0.to_owned(),
+            key: self.relative_path_to_s3_object(from),
             ..GetObjectRequest::default()
         })
         .await
@@ -396,7 +399,7 @@ impl RemoteStorage for S3Bucket {
 
     async fn download_byte_range(
         &self,
-        from: &RemoteObjectId,
+        from: &RemotePath,
         start_inclusive: u64,
         end_exclusive: Option<u64>,
     ) -> Result<Download, DownloadError> {
@@ -404,20 +407,19 @@ impl RemoteStorage for S3Bucket {
         // and needs both ends to be exclusive
         let end_inclusive = end_exclusive.map(|end| end.saturating_sub(1));
         let range = Some(match end_inclusive {
-            Some(end_inclusive) => format!("bytes={}-{}", start_inclusive, end_inclusive),
-            None => format!("bytes={}-", start_inclusive),
+            Some(end_inclusive) => format!("bytes={start_inclusive}-{end_inclusive}"),
+            None => format!("bytes={start_inclusive}-"),
         });
 
         self.download_object(GetObjectRequest {
             bucket: self.bucket_name.clone(),
-            key: from.0.to_owned(),
+            key: self.relative_path_to_s3_object(from),
             range,
-            ..GetObjectRequest::default()
         })
         .await
     }
 
-    async fn delete(&self, remote_object_id: &RemoteObjectId) -> anyhow::Result<()> {
+    async fn delete(&self, path: &RemotePath) -> anyhow::Result<()> {
         let _guard = self
             .concurrency_limiter
             .acquire()
@@ -427,11 +429,10 @@ impl RemoteStorage for S3Bucket {
         metrics::inc_delete_object();
 
         self.client
-            .delete_object(DeleteObjectRequest {
-                bucket: self.bucket_name.clone(),
-                key: remote_object_id.0.to_owned(),
-                ..DeleteObjectRequest::default()
-            })
+            .delete_object()
+            .bucket(self.bucket_name.clone())
+            .key(self.relative_path_to_s3_object(path))
+            .send()
             .await
             .map_err(|e| {
                 metrics::inc_delete_object_fail();
@@ -440,181 +441,3 @@ impl RemoteStorage for S3Bucket {
         Ok(())
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use tempfile::tempdir;
-
-    use super::*;
-
-    #[test]
-    fn test_download_destination() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-        let local_path = workdir.join("one").join("two").join("test_name");
-        let relative_path = local_path.strip_prefix(&workdir)?;
-
-        let key = RemoteObjectId(format!(
-            "{}{}",
-            REMOTE_STORAGE_PREFIX_SEPARATOR,
-            relative_path
-                .iter()
-                .map(|segment| segment.to_str().unwrap())
-                .collect::<Vec<_>>()
-                .join(&REMOTE_STORAGE_PREFIX_SEPARATOR.to_string()),
-        ));
-
-        assert_eq!(
-            local_path,
-            download_destination(&key, &workdir, None),
-            "Download destination should consist of s3 path joined with the workdir prefix"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn storage_path_positive() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
-        let segment_1 = "matching";
-        let segment_2 = "file";
-        let local_path = &workdir.join(segment_1).join(segment_2);
-
-        let storage = dummy_storage(workdir);
-
-        let expected_key = RemoteObjectId(format!(
-            "{}{REMOTE_STORAGE_PREFIX_SEPARATOR}{segment_1}{REMOTE_STORAGE_PREFIX_SEPARATOR}{segment_2}",
-            storage.prefix_in_bucket.as_deref().unwrap_or_default(),
-        ));
-
-        let actual_key = storage
-            .remote_object_id(local_path)
-            .expect("Matching path should map to S3 path normally");
-        assert_eq!(
-            expected_key,
-            actual_key,
-            "S3 key from the matching path should contain all segments after the workspace prefix, separated with S3 separator"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn storage_path_negatives() -> anyhow::Result<()> {
-        #[track_caller]
-        fn storage_path_error(storage: &S3Bucket, mismatching_path: &Path) -> String {
-            match storage.remote_object_id(mismatching_path) {
-                Ok(wrong_key) => panic!(
-                    "Expected path '{}' to error, but got S3 key: {:?}",
-                    mismatching_path.display(),
-                    wrong_key,
-                ),
-                Err(e) => e.to_string(),
-            }
-        }
-
-        let workdir = tempdir()?.path().to_owned();
-        let storage = dummy_storage(workdir.clone());
-
-        let error_message = storage_path_error(&storage, &workdir);
-        assert!(
-            error_message.contains("Prefix and the path are equal"),
-            "Message '{}' does not contain the required string",
-            error_message
-        );
-
-        let mismatching_path = PathBuf::from("somewhere").join("else");
-        let error_message = storage_path_error(&storage, &mismatching_path);
-        assert!(
-            error_message.contains(mismatching_path.to_str().unwrap()),
-            "Error should mention wrong path"
-        );
-        assert!(
-            error_message.contains(workdir.to_str().unwrap()),
-            "Error should mention server workdir"
-        );
-        assert!(
-            error_message.contains("is not prefixed with"),
-            "Message '{}' does not contain a required string",
-            error_message
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn local_path_positive() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-        let storage = dummy_storage(workdir.clone());
-        let timeline_dir = workdir.join("timelines").join("test_timeline");
-        let relative_timeline_path = timeline_dir.strip_prefix(&workdir)?;
-
-        let s3_key = create_s3_key(
-            &relative_timeline_path.join("not a metadata"),
-            storage.prefix_in_bucket.as_deref(),
-        );
-        assert_eq!(
-            download_destination(&s3_key, &workdir, storage.prefix_in_bucket.as_deref()),
-            storage
-                .local_path(&s3_key)
-                .expect("For a valid input, valid S3 info should be parsed"),
-            "Should be able to parse metadata out of the correctly named remote delta file"
-        );
-
-        let s3_key = create_s3_key(
-            &relative_timeline_path.join("metadata"),
-            storage.prefix_in_bucket.as_deref(),
-        );
-        assert_eq!(
-            download_destination(&s3_key, &workdir, storage.prefix_in_bucket.as_deref()),
-            storage
-                .local_path(&s3_key)
-                .expect("For a valid input, valid S3 info should be parsed"),
-            "Should be able to parse metadata out of the correctly named remote metadata file"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn download_destination_matches_original_path() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-        let original_path = workdir
-            .join("timelines")
-            .join("some_timeline")
-            .join("some name");
-
-        let dummy_storage = dummy_storage(workdir);
-
-        let key = dummy_storage.remote_object_id(&original_path)?;
-        let download_destination = dummy_storage.local_path(&key)?;
-
-        assert_eq!(
-            original_path, download_destination,
-            "'original path -> storage key -> matching fs path' transformation should produce the same path as the input one for the correct path"
-        );
-
-        Ok(())
-    }
-
-    fn dummy_storage(workdir: PathBuf) -> S3Bucket {
-        S3Bucket {
-            workdir,
-            client: S3Client::new("us-east-1".parse().unwrap()),
-            bucket_name: "dummy-bucket".to_string(),
-            prefix_in_bucket: Some("dummy_prefix/".to_string()),
-            concurrency_limiter: Semaphore::new(1),
-        }
-    }
-
-    fn create_s3_key(relative_file_path: &Path, prefix: Option<&str>) -> RemoteObjectId {
-        RemoteObjectId(relative_file_path.iter().fold(
-            prefix.unwrap_or_default().to_string(),
-            |mut path_string, segment| {
-                path_string.push(REMOTE_STORAGE_PREFIX_SEPARATOR);
-                path_string.push_str(segment.to_str().unwrap());
-                path_string
-            },
-        ))
-    }
-}

libs/utils/Cargo.toml

@@ -4,6 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
+sentry = "0.29.0"
 async-trait = "0.1"
 anyhow = "1.0"
 bincode = "1.3"

libs/utils/src/fs_ext.rs

@@ -0,0 +1,45 @@
+/// Extensions to `std::fs` types.
+use std::{fs, io, path::Path};
+
+pub trait PathExt {
+    /// Returns an error if `self` is not a directory.
+    fn is_empty_dir(&self) -> io::Result<bool>;
+}
+
+impl<P> PathExt for P
+where
+    P: AsRef<Path>,
+{
+    fn is_empty_dir(&self) -> io::Result<bool> {
+        Ok(fs::read_dir(self)?.into_iter().next().is_none())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::path::PathBuf;
+
+    #[test]
+    fn is_empty_dir() {
+        use super::PathExt;
+
+        let dir = tempfile::tempdir().unwrap();
+        let dir_path = dir.path();
+
+        // test positive case
+        assert!(
+            dir_path.is_empty_dir().expect("test failure"),
+            "new tempdir should be empty"
+        );
+
+        // invoke on a file to ensure it returns an error
+        let file_path: PathBuf = dir_path.join("testfile");
+        let f = std::fs::File::create(&file_path).unwrap();
+        drop(f);
+        assert!(file_path.is_empty_dir().is_err());
+
+        // do it again on a path, we know to be nonexistent
+        std::fs::remove_file(&file_path).unwrap();
+        assert!(file_path.is_empty_dir().is_err());
+    }
+}

libs/utils/src/lib.rs

@@ -34,6 +34,7 @@ pub mod sock_split;
 pub mod logging;
 
 pub mod lock_file;
+pub mod pid_file;
 
 // Misc
 pub mod accum;
@@ -46,8 +47,11 @@ pub mod tcp_listener;
 pub mod nonblock;
 
 // Default signal handling
+pub mod sentry_init;
 pub mod signals;
 
+pub mod fs_ext;
+
 /// use with fail::cfg("$name", "return(2000)")
 #[macro_export]
 macro_rules! failpoint_sleep_millis_async {

libs/utils/src/lock_file.rs

@@ -1,81 +1,133 @@
-//! A module to create and read lock files. A lock file ensures that only one
-//! process is running at a time, in a particular directory.
+//! A module to create and read lock files.
 //!
-//! File locking is done using [`fcntl::flock`], which means that holding the
-//! lock on file only prevents acquiring another lock on it; all other
-//! operations are still possible on files. Other process can still open, read,
-//! write, or remove the file, for example.
-//! If the file is removed while a process is holding a lock on it,
-//! the process that holds the lock does not get any error or notification.
-//! Furthermore, you can create a new file with the same name and lock the new file,
-//! while the old process is still running.
-//! Deleting the lock file while the locking process is still running is a bad idea!
+//! File locking is done using [`fcntl::flock`] exclusive locks.
+//! The only consumer of this module is currently [`pid_file`].
+//! See the module-level comment there for potential pitfalls
+//! with lock files that are used to store PIDs (pidfiles).
 
-use std::{fs, os::unix::prelude::AsRawFd, path::Path};
+use std::{
+    fs,
+    io::{Read, Write},
+    ops::Deref,
+    os::unix::prelude::AsRawFd,
+    path::{Path, PathBuf},
+};
 
 use anyhow::Context;
-use nix::fcntl;
+use nix::{errno::Errno::EAGAIN, fcntl};
 
 use crate::crashsafe;
 
-pub enum LockCreationResult {
-    Created {
-        new_lock_contents: String,
-        file: fs::File,
-    },
-    AlreadyLocked {
-        existing_lock_contents: String,
-    },
-    CreationFailed(anyhow::Error),
+/// A handle to an open and unlocked, but not-yet-written lock file.
+/// Returned by [`create_exclusive`].
+#[must_use]
+pub struct UnwrittenLockFile {
+    path: PathBuf,
+    file: fs::File,
 }
 
-/// Creates a lock file in the path given and writes the given contents into the file.
-/// Note: The lock is automatically released when the file closed. You might want to use Box::leak to make sure it lives until the end of the program.
-pub fn create_lock_file(lock_file_path: &Path, contents: String) -> LockCreationResult {
-    let lock_file = match fs::OpenOptions::new()
+/// Returned by [`UnwrittenLockFile::write_content`].
+#[must_use]
+pub struct LockFileGuard(fs::File);
+
+impl Deref for LockFileGuard {
+    type Target = fs::File;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl UnwrittenLockFile {
+    /// Replace the content of this lock file with the byte representation of `contents`.
+    pub fn write_content(mut self, contents: String) -> anyhow::Result<LockFileGuard> {
+        self.file
+            .set_len(0)
+            .context("Failed to truncate lockfile")?;
+        self.file
+            .write_all(contents.as_bytes())
+            .with_context(|| format!("Failed to write '{contents}' contents into lockfile"))?;
+        crashsafe::fsync_file_and_parent(&self.path).context("fsync lockfile")?;
+        Ok(LockFileGuard(self.file))
+    }
+}
+
+/// Creates and opens a lock file in the path, grabs an exclusive flock on it, and returns
+/// a handle that allows overwriting the locked file's content.
+///
+/// The exclusive lock is released when dropping the returned handle.
+///
+/// It is not an error if the file already exists.
+/// It is an error if the file is already locked.
+pub fn create_exclusive(lock_file_path: &Path) -> anyhow::Result<UnwrittenLockFile> {
+    let lock_file = fs::OpenOptions::new()
         .create(true) // O_CREAT
         .write(true)
         .open(lock_file_path)
-        .context("Failed to open lock file")
-    {
-        Ok(file) => file,
-        Err(e) => return LockCreationResult::CreationFailed(e),
-    };
+        .context("open lock file")?;
 
-    match fcntl::flock(
+    let res = fcntl::flock(
         lock_file.as_raw_fd(),
         fcntl::FlockArg::LockExclusiveNonblock,
-    ) {
-        Ok(()) => {
-            match lock_file
-                .set_len(0)
-                .context("Failed to truncate lockfile")
-                .and_then(|()| {
-                    fs::write(lock_file_path, &contents).with_context(|| {
-                        format!("Failed to write '{contents}' contents into lockfile")
-                    })
-                })
-                .and_then(|()| {
-                    crashsafe::fsync_file_and_parent(lock_file_path)
-                        .context("Failed to fsync lockfile")
-                }) {
-                Ok(()) => LockCreationResult::Created {
-                    new_lock_contents: contents,
-                    file: lock_file,
-                },
-                Err(e) => LockCreationResult::CreationFailed(e),
-            }
-        }
-        Err(nix::errno::Errno::EAGAIN) => {
-            match fs::read_to_string(lock_file_path).context("Failed to read lockfile contents") {
-                Ok(existing_lock_contents) => LockCreationResult::AlreadyLocked {
-                    existing_lock_contents,
-                },
-                Err(e) => LockCreationResult::CreationFailed(e),
-            }
-        }
-        Err(e) => {
-            LockCreationResult::CreationFailed(anyhow::anyhow!("Failed to lock lockfile: {e}"))
-        }
+    );
+    match res {
+        Ok(()) => Ok(UnwrittenLockFile {
+            path: lock_file_path.to_owned(),
+            file: lock_file,
+        }),
+        Err(EAGAIN) => anyhow::bail!("file is already locked"),
+        Err(e) => Err(e).context("flock error"),
+    }
+}
+
+/// Returned by [`read_and_hold_lock_file`].
+/// Check out the [`pid_file`] module for what the variants mean
+/// and potential caveats if the lock files that are used to store PIDs.
+pub enum LockFileRead {
+    /// No file exists at the given path.
+    NotExist,
+    /// No other process held the lock file, so we grabbed an flock
+    /// on it and read its contents.
+    /// Release the flock by dropping the [`LockFileGuard`].
+    NotHeldByAnyProcess(LockFileGuard, String),
+    /// The file exists but another process was holding an flock on it.
+    LockedByOtherProcess {
+        not_locked_file: fs::File,
+        content: String,
+    },
+}
+
+/// Open & try to lock the lock file at the given `path`, returning a [handle][`LockFileRead`] to
+/// inspect its content. It is not an `Err(...)` if the file does not exist or is already locked.
+/// Check the [`LockFileRead`] variants for details.
+pub fn read_and_hold_lock_file(path: &Path) -> anyhow::Result<LockFileRead> {
+    let res = fs::OpenOptions::new().read(true).open(path);
+    let mut lock_file = match res {
+        Ok(f) => f,
+        Err(e) => match e.kind() {
+            std::io::ErrorKind::NotFound => return Ok(LockFileRead::NotExist),
+            _ => return Err(e).context("open lock file"),
+        },
+    };
+    let res = fcntl::flock(
+        lock_file.as_raw_fd(),
+        fcntl::FlockArg::LockExclusiveNonblock,
+    );
+    // We need the content regardless of lock success / failure.
+    // But, read it after flock so that, if it succeeded, the content is consistent.
+    let mut content = String::new();
+    lock_file
+        .read_to_string(&mut content)
+        .context("read lock file")?;
+    match res {
+        Ok(()) => Ok(LockFileRead::NotHeldByAnyProcess(
+            LockFileGuard(lock_file),
+            content,
+        )),
+        Err(EAGAIN) => Ok(LockFileRead::LockedByOtherProcess {
+            not_locked_file: lock_file,
+            content,
+        }),
+        Err(e) => Err(e).context("flock error"),
     }
 }

libs/utils/src/pid_file.rs

@@ -0,0 +1,165 @@
+//! Abstraction to create & read pidfiles.
+//!
+//! A pidfile is a file in the filesystem that stores a process's PID.
+//! Its purpose is to implement a singleton behavior where only
+//! one process of some "kind" is supposed to be running at a given time.
+//! The "kind" is identified by the pidfile.
+//!
+//! During process startup, the process that is supposed to be a singleton
+//! must [claim][`claim_for_current_process`] the pidfile first.
+//! If that is unsuccessful, the process must not act as the singleton, i.e.,
+//! it must not access any of the resources that only the singleton may access.
+//!
+//! A common need is to signal a running singleton process, e.g., to make
+//! it shut down and exit.
+//! For that, we have to [`read`] the pidfile. The result of the `read` operation
+//! tells us if there is any singleton process, and if so, what PID it has.
+//! We can then proceed to signal it, although some caveats still apply.
+//! Read the function-level documentation of [`read`] for that.
+//!
+//! ## Never Remove Pidfiles
+//!
+//! It would be natural to assume that the process who claimed the pidfile
+//! should remove it upon exit to avoid leaving a stale pidfile in place.
+//! However, we already have a reliable way to detect staleness of the pidfile,
+//! i.e., the `flock` that [claiming][`claim_for_current_process`] puts on it.
+//!
+//! And further, removing pidfiles would introduce a **catastrophic race condition**
+//! where two processes are running that are supposed to be singletons.
+//! Suppose we were to remove our pidfile during process shutdown.
+//! Here is how the race plays out:
+//! - Suppose we have a service called `myservice` with pidfile `myservice.pidfile`.
+//! - Process `A` starts to shut down.
+//! - Process `B` is just starting up
+//!     - It `open("myservice.pid", O_WRONLY|O_CREAT)` the file
+//!     - It blocks on `flock`
+//! - Process `A` removes the pidfile as the last step of its shutdown procedure
+//!     - `unlink("myservice.pid")
+//! - Process `A` exits
+//!     - This releases its `flock` and unblocks `B`
+//! - Process `B` still has the file descriptor for `myservice.pid` open
+//! - Process `B` writes its PID into `myservice.pid`.
+//! - But the `myservice.pid` file has been unlinked, so, there is `myservice.pid`
+//!   in the directory.
+//! - Process `C` starts
+//!     - It `open("myservice.pid", O_WRONLY|O_CREAT)` which creates a new file (new inode)
+//!     - It `flock`s the file, which, since it's a different file, does not block
+//!     - It writes its PID into the file
+//!
+//! At this point, `B` and `C` are running, which is hazardous.
+//! Morale of the story: don't unlink pidfiles, ever.
+
+use std::{ops::Deref, path::Path};
+
+use anyhow::Context;
+use nix::unistd::Pid;
+
+use crate::lock_file::{self, LockFileRead};
+
+/// Keeps a claim on a pidfile alive until it is dropped.
+/// Returned by [`claim_for_current_process`].
+#[must_use]
+pub struct PidFileGuard(lock_file::LockFileGuard);
+
+impl Deref for PidFileGuard {
+    type Target = lock_file::LockFileGuard;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+/// Try to claim `path` as a pidfile for the current process.
+///
+/// If another process has already claimed the pidfile, and it is still running,
+/// this function returns ane error.
+/// Otherwise, the function `flock`s the file and updates its contents to the
+/// current process's PID.
+/// If the update fails, the flock is released and an error returned.
+/// On success, the function returns a [`PidFileGuard`] to keep the flock alive.
+///
+/// ### Maintaining A Claim
+///
+/// It is the caller's responsibility to maintain the claim.
+/// The claim ends as soon as the returned guard object is dropped.
+/// To maintain the claim for the remaining lifetime of the current process,
+/// use [`std::mem::forget`] or similar.
+pub fn claim_for_current_process(path: &Path) -> anyhow::Result<PidFileGuard> {
+    let unwritten_lock_file = lock_file::create_exclusive(path).context("lock file")?;
+    // if any of the next steps fail, we drop the file descriptor and thereby release the lock
+    let guard = unwritten_lock_file
+        .write_content(Pid::this().to_string())
+        .context("write pid to lock file")?;
+    Ok(PidFileGuard(guard))
+}
+
+/// Returned by [`read`].
+pub enum PidFileRead {
+    /// No file exists at the given path.
+    NotExist,
+    /// The given pidfile is currently not claimed by any process.
+    /// To determine this, the [`read`] operation acquired
+    /// an exclusive flock on the file. The lock is still held and responsibility
+    /// to release it is returned through the guard object.
+    /// Before releasing it, other [`claim_for_current_process`] or [`read`] calls
+    /// will fail.
+    ///
+    /// ### Caveats
+    ///
+    /// Do not unlink the pidfile from the filesystem. See module-comment for why.
+    NotHeldByAnyProcess(PidFileGuard),
+    /// The given pidfile is still claimed by another process whose PID is given
+    /// as part of this variant.
+    ///
+    /// ### Caveats
+    ///
+    /// 1. The other process might exit at any time, turning the given PID stale.
+    /// 2. There is a small window in which `claim_for_current_process` has already
+    ///    locked the file but not yet updates its contents. [`read`] will return
+    ///    this variant here, but with the old file contents, i.e., a stale PID.
+    ///
+    /// The kernel is free to recycle PID once it has been `wait(2)`ed upon by
+    /// its creator. Thus, acting upon a stale PID, e.g., by issuing a `kill`
+    /// system call on it, bears the risk of killing an unrelated process.
+    /// This is an inherent limitation of using pidfiles.
+    /// The only race-free solution is to have a supervisor-process with a lifetime
+    /// that exceeds that of all of its child-processes (e.g., `runit`, `supervisord`).
+    LockedByOtherProcess(Pid),
+}
+
+/// Try to read the file at the given path as a pidfile that was previously created
+/// through [`claim_for_current_process`].
+///
+/// On success, this function returns a [`PidFileRead`].
+/// Check its docs for a description of the meaning of its different variants.
+pub fn read(pidfile: &Path) -> anyhow::Result<PidFileRead> {
+    let res = lock_file::read_and_hold_lock_file(pidfile).context("read and hold pid file")?;
+    let ret = match res {
+        LockFileRead::NotExist => PidFileRead::NotExist,
+        LockFileRead::NotHeldByAnyProcess(guard, _) => {
+            PidFileRead::NotHeldByAnyProcess(PidFileGuard(guard))
+        }
+        LockFileRead::LockedByOtherProcess {
+            not_locked_file: _not_locked_file,
+            content,
+        } => {
+            // XXX the read races with the write in claim_pid_file_for_pid().
+            // But pids are smaller than a page, so the kernel page cache will lock for us.
+            // The only problem is that we might get the old contents here.
+            // Can only fix that by implementing some scheme that downgrades the
+            // exclusive lock to shared lock in claim_pid_file_for_pid().
+            PidFileRead::LockedByOtherProcess(parse_pidfile_content(&content)?)
+        }
+    };
+    Ok(ret)
+}
+
+fn parse_pidfile_content(content: &str) -> anyhow::Result<Pid> {
+    let pid: i32 = content
+        .parse()
+        .map_err(|_| anyhow::anyhow!("parse pidfile content to PID"))?;
+    if pid < 1 {
+        anyhow::bail!("bad value in pidfile '{pid}'");
+    }
+    Ok(Pid::from_raw(pid))
+}

libs/utils/src/sentry_init.rs

@@ -0,0 +1,27 @@
+use sentry::ClientInitGuard;
+use std::borrow::Cow;
+use std::env;
+
+pub use sentry::release_name;
+
+#[must_use]
+pub fn init_sentry(
+    release_name: Option<Cow<'static, str>>,
+    extra_options: &[(&str, &str)],
+) -> Option<ClientInitGuard> {
+    let dsn = env::var("SENTRY_DSN").ok()?;
+
+    let guard = sentry::init((
+        dsn,
+        sentry::ClientOptions {
+            release: release_name,
+            ..Default::default()
+        },
+    ));
+    sentry::configure_scope(|scope| {
+        for &(key, value) in extra_options {
+            scope.set_extra(key, value.into());
+        }
+    });
+    Some(guard)
+}

pageserver/Cargo.toml

@@ -5,10 +5,6 @@ edition = "2021"
 
 [features]
 default = []
-# Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro,
-# which adds some runtime cost to run tests on outage conditions
-testing = ["fail/failpoints"]
-
 profiling = ["pprof"]
 
 [dependencies]
@@ -18,13 +14,13 @@ async-stream = "0.3"
 async-trait = "0.1"
 byteorder = "1.4.3"
 bytes = "1.0.1"
-chrono = "0.4.19"
+chrono = { version = "0.4.23", default-features = false, features = ["clock"] }
 clap = { version = "4.0", features = ["string"] }
 close_fds = "0.3.2"
 const_format = "0.2.21"
 crc32c = "0.6.0"
 crossbeam-utils = "0.8.5"
-fail = "0.5.0"
+fail = { version = "0.5", default-features = false, features = ["failpoints"] }
 futures = "0.3.13"
 git-version = "0.3.5"
 hex = "0.4.3"
@@ -35,6 +31,7 @@ itertools = "0.10.3"
 nix = "0.25"
 num-traits = "0.2.15"
 once_cell = "1.13.0"
+pin-project-lite = "0.2.7"
 postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
 postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
 postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }

pageserver/benches/README.md

@@ -0,0 +1,12 @@
+## Pageserver Benchmarks
+
+# How to run
+
+To run all benchmarks:
+`cargo bench`
+
+To run a specific file:
+`cargo bench --bench bench_layer_map`
+
+To run a specific function:
+`cargo bench --bench bench_layer_map -- real_map_uniform_queries`

pageserver/benches/bench_walredo.rs

@@ -431,7 +431,7 @@ fn pg_record(will_init: bool, bytes: &'static [u8]) -> NeonWalRecord {
 struct Request {
     key: Key,
     lsn: Lsn,
-    base_img: Option<Bytes>,
+    base_img: Option<(Lsn, Bytes)>,
     records: Vec<(Lsn, NeonWalRecord)>,
     pg_version: u32,
 }

pageserver/src/basebackup.rs

@@ -12,7 +12,6 @@
 //!
 use anyhow::{anyhow, bail, ensure, Context, Result};
 use bytes::{BufMut, BytesMut};
-use fail::fail_point;
 use itertools::Itertools;
 use std::fmt::Write as FmtWrite;
 use std::io;
@@ -22,6 +21,7 @@ use std::time::SystemTime;
 use tar::{Builder, EntryType, Header};
 use tracing::*;
 
+use crate::fail_point;
 use crate::tenant::Timeline;
 use pageserver_api::reltag::{RelTag, SlruKind};
 

pageserver/src/bin/draw_timeline_dir.rs

@@ -11,8 +11,8 @@
 //!
 //! Example use:
 //! ```
-//! $ cd test_output/test_pgbench\[neon-45-684\]/repo/tenants/$TENANT/timelines/$TIMELINE
-//! $ ls | grep "__" | cargo run --release --bin draw_timeline_dir > out.svg
+//! $ ls test_output/test_pgbench\[neon-45-684\]/repo/tenants/$TENANT/timelines/$TIMELINE | \
+//! $   grep "__" | cargo run --release --bin draw_timeline_dir > out.svg
 //! $ firefox out.svg
 //! ```
 //!
@@ -25,6 +25,8 @@ use anyhow::Result;
 use pageserver::repository::Key;
 use std::cmp::Ordering;
 use std::io::{self, BufRead};
+use std::path::PathBuf;
+use std::str::FromStr;
 use std::{
     collections::{BTreeMap, BTreeSet},
     ops::Range,
@@ -65,7 +67,11 @@ fn main() -> Result<()> {
     let mut ranges: Vec<(Range<Key>, Range<Lsn>)> = vec![];
     let stdin = io::stdin();
     for line in stdin.lock().lines() {
-        let range = parse_filename(&line.unwrap());
+        let line = line.unwrap();
+        let line = PathBuf::from_str(&line).unwrap();
+        let filename = line.file_name().unwrap();
+        let filename = filename.to_str().unwrap();
+        let range = parse_filename(filename);
         ranges.push(range);
     }
 

pageserver/src/bin/pageserver.rs

@@ -7,7 +7,6 @@ use std::{env, ops::ControlFlow, path::Path, str::FromStr};
 use anyhow::{anyhow, Context};
 use clap::{Arg, ArgAction, Command};
 use fail::FailScenario;
-use nix::unistd::Pid;
 use tracing::*;
 
 use metrics::set_build_info_metric;
@@ -23,9 +22,10 @@ use pageserver::{
 use remote_storage::GenericRemoteStorage;
 use utils::{
     auth::JwtAuth,
-    lock_file, logging,
+    logging,
     postgres_backend::AuthType,
     project_git_version,
+    sentry_init::{init_sentry, release_name},
     signals::{self, Signal},
     tcp_listener,
 };
@@ -35,10 +35,6 @@ project_git_version!(GIT_VERSION);
 const PID_FILE_NAME: &str = "pageserver.pid";
 
 const FEATURES: &[&str] = &[
-    #[cfg(feature = "testing")]
-    "testing",
-    #[cfg(feature = "fail/failpoints")]
-    "fail/failpoints",
     #[cfg(feature = "profiling")]
     "profiling",
 ];
@@ -85,6 +81,9 @@ fn main() -> anyhow::Result<()> {
         }
     };
 
+    // initialize sentry if SENTRY_DSN is provided
+    let _sentry_guard = init_sentry(release_name!(), &[("node_id", &conf.id.to_string())]);
+
     let tenants_path = conf.tenants_path();
     if !tenants_path.exists() {
         utils::crashsafe::create_dir_all(conf.tenants_path()).with_context(|| {
@@ -175,6 +174,10 @@ fn initialize_config(
     let conf = PageServerConf::parse_and_validate(&toml, workdir)
         .context("Failed to parse pageserver configuration")?;
 
+    if pageserver::TESTING_MODE.set(conf.testing_mode).is_err() {
+        anyhow::bail!("testing_mode was already initialized");
+    }
+
     if update_config {
         info!("Writing pageserver config to '{}'", cfg_file_path.display());
 
@@ -203,41 +206,32 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
 
     // If any failpoints were set from FAILPOINTS environment variable,
     // print them to the log for debugging purposes
-    let failpoints = fail::list();
-    if !failpoints.is_empty() {
-        info!(
-            "started with failpoints: {}",
-            failpoints
-                .iter()
-                .map(|(name, actions)| format!("{name}={actions}"))
-                .collect::<Vec<String>>()
-                .join(";")
-        )
+    if *pageserver::TESTING_MODE.get().unwrap() {
+        let failpoints = fail::list();
+        if !failpoints.is_empty() {
+            info!(
+                "started with testing mode enabled, failpoints: {}",
+                failpoints
+                    .iter()
+                    .map(|(name, actions)| format!("{name}={actions}"))
+                    .collect::<Vec<String>>()
+                    .join(";")
+            )
+        } else {
+            info!("started with testing mode enabled");
+        }
+    } else {
+        info!("started with testing mode disabled");
     }
 
     let lock_file_path = conf.workdir.join(PID_FILE_NAME);
-    let lock_file = match lock_file::create_lock_file(&lock_file_path, Pid::this().to_string()) {
-        lock_file::LockCreationResult::Created {
-            new_lock_contents,
-            file,
-        } => {
-            info!("Created lock file at {lock_file_path:?} with contenst {new_lock_contents}");
-            file
-        }
-        lock_file::LockCreationResult::AlreadyLocked {
-            existing_lock_contents,
-        } => anyhow::bail!(
-            "Could not lock pid file; pageserver is already running in {:?} with PID {}",
-            conf.workdir,
-            existing_lock_contents
-        ),
-        lock_file::LockCreationResult::CreationFailed(e) => {
-            return Err(e.context(format!("Failed to create lock file at {lock_file_path:?}")))
-        }
-    };
+    let lock_file =
+        utils::pid_file::claim_for_current_process(&lock_file_path).context("claim pid file")?;
+    info!("Claimed pid file at {lock_file_path:?}");
+
     // ensure that the lock file is held even if the main thread of the process is panics
     // we need to release the lock file only when the current process is gone
-    let _ = Box::leak(Box::new(lock_file));
+    std::mem::forget(lock_file);
 
     // TODO: Check that it looks like a valid repository before going further
 
@@ -292,15 +286,23 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
     let remote_storage = conf
         .remote_storage_config
         .as_ref()
-        .map(|storage_config| {
-            GenericRemoteStorage::from_config(conf.workdir.clone(), storage_config)
-        })
+        .map(GenericRemoteStorage::from_config)
         .transpose()
         .context("Failed to init generic remote storage")?;
-    let remote_index = {
-        let _rt_guard = BACKGROUND_RUNTIME.enter();
-        tenant_mgr::init_tenant_mgr(conf, remote_storage.clone())?
-    };
+
+    let (init_result_sender, init_result_receiver) =
+        std::sync::mpsc::channel::<anyhow::Result<()>>();
+    let storage_for_spawn = remote_storage.clone();
+    let _handler = BACKGROUND_RUNTIME.spawn(async move {
+        let result = tenant_mgr::init_tenant_mgr(conf, storage_for_spawn).await;
+        init_result_sender.send(result)
+    });
+    match init_result_receiver.recv() {
+        Ok(init_result) => init_result.context("Failed to init tenant_mgr")?,
+        Err(_sender_dropped_err) => {
+            anyhow::bail!("Failed to init tenant_mgr: no init status was returned");
+        }
+    }
 
     // Spawn all HTTP related tasks in the MGMT_REQUEST_RUNTIME.
     // bind before launching separate thread so the error reported before startup exits
@@ -309,7 +311,7 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
     {
         let _rt_guard = MGMT_REQUEST_RUNTIME.enter();
 
-        let router = http::make_router(conf, auth.clone(), remote_index, remote_storage)?;
+        let router = http::make_router(conf, auth.clone(), remote_storage)?;
         let service =
             utils::http::RouterService::new(router.build().map_err(|err| anyhow!(err))?).unwrap();
         let server = hyper::Server::from_tcp(http_listener)?

pageserver/src/config.rs

@@ -5,7 +5,7 @@
 //! See also `settings.md` for better description on every parameter.
 
 use anyhow::{anyhow, bail, ensure, Context, Result};
-use remote_storage::RemoteStorageConfig;
+use remote_storage::{RemotePath, RemoteStorageConfig};
 use std::env;
 use utils::crashsafe::path_with_suffix_extension;
 use utils::id::ConnectionId;
@@ -25,9 +25,11 @@ use utils::{
     postgres_backend::AuthType,
 };
 
-use crate::tenant::TIMELINES_SEGMENT_NAME;
+use crate::tenant::{TENANT_ATTACHING_MARKER_FILENAME, TIMELINES_SEGMENT_NAME};
 use crate::tenant_config::{TenantConf, TenantConfOpt};
-use crate::{METADATA_FILE_NAME, TENANT_CONFIG_NAME, TIMELINE_UNINIT_MARK_SUFFIX};
+use crate::{
+    IGNORED_TENANT_FILE_NAME, METADATA_FILE_NAME, TENANT_CONFIG_NAME, TIMELINE_UNINIT_MARK_SUFFIX,
+};
 
 pub mod defaults {
     use crate::tenant_config::defaults::*;
@@ -51,6 +53,8 @@ pub mod defaults {
     pub const DEFAULT_CONCURRENT_TENANT_SIZE_LOGICAL_SIZE_QUERIES: usize =
         super::ConfigurableSemaphore::DEFAULT_INITIAL.get();
 
+    pub const DEFAULT_TESTING_MODE: bool = false;
+
     ///
     /// Default built-in configuration file.
     ///
@@ -73,6 +77,8 @@ pub mod defaults {
 
 #concurrent_tenant_size_logical_size_queries = '{DEFAULT_CONCURRENT_TENANT_SIZE_LOGICAL_SIZE_QUERIES}'
 
+testing_mode = false
+
 # [tenant_config]
 #checkpoint_distance = {DEFAULT_CHECKPOINT_DISTANCE} # in bytes
 #checkpoint_timeout = {DEFAULT_CHECKPOINT_TIMEOUT}
@@ -141,6 +147,9 @@ pub struct PageServerConf {
 
     /// Number of concurrent [`Tenant::gather_size_inputs`] allowed.
     pub concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore,
+
+    /// Enables failpoint support and extra mgmt APIs useful for testing.
+    pub testing_mode: bool,
 }
 
 /// We do not want to store this in a PageServerConf because the latter may be logged
@@ -220,6 +229,8 @@ struct PageServerConfigBuilder {
     log_format: BuilderValue<LogFormat>,
 
     concurrent_tenant_size_logical_size_queries: BuilderValue<ConfigurableSemaphore>,
+
+    testing_mode: BuilderValue<bool>,
 }
 
 impl Default for PageServerConfigBuilder {
@@ -250,6 +261,8 @@ impl Default for PageServerConfigBuilder {
             log_format: Set(LogFormat::from_str(DEFAULT_LOG_FORMAT).unwrap()),
 
             concurrent_tenant_size_logical_size_queries: Set(ConfigurableSemaphore::default()),
+
+            testing_mode: Set(DEFAULT_TESTING_MODE),
         }
     }
 }
@@ -330,11 +343,11 @@ impl PageServerConfigBuilder {
         self.concurrent_tenant_size_logical_size_queries = BuilderValue::Set(u);
     }
 
-    pub fn build(self) -> anyhow::Result<PageServerConf> {
-        let broker_endpoints = self
-            .broker_endpoints
-            .ok_or(anyhow!("No broker endpoints provided"))?;
+    pub fn testing_mode(&mut self, testing_mode: bool) {
+        self.testing_mode = BuilderValue::Set(testing_mode);
+    }
 
+    pub fn build(self) -> anyhow::Result<PageServerConf> {
         Ok(PageServerConf {
             listen_pg_addr: self
                 .listen_pg_addr
@@ -370,7 +383,9 @@ impl PageServerConfigBuilder {
             profiling: self.profiling.ok_or(anyhow!("missing profiling"))?,
             // TenantConf is handled separately
             default_tenant_conf: TenantConf::default(),
-            broker_endpoints,
+            broker_endpoints: self
+                .broker_endpoints
+                .ok_or(anyhow!("No broker endpoints provided"))?,
             broker_etcd_prefix: self
                 .broker_etcd_prefix
                 .ok_or(anyhow!("missing broker_etcd_prefix"))?,
@@ -380,6 +395,7 @@ impl PageServerConfigBuilder {
                 .ok_or(anyhow!(
                     "missing concurrent_tenant_size_logical_size_queries"
                 ))?,
+            testing_mode: self.testing_mode.ok_or(anyhow!("missing testing_mode"))?,
         })
     }
 }
@@ -397,6 +413,15 @@ impl PageServerConf {
         self.tenants_path().join(tenant_id.to_string())
     }
 
+    pub fn tenant_attaching_mark_file_path(&self, tenant_id: &TenantId) -> PathBuf {
+        self.tenant_path(tenant_id)
+            .join(TENANT_ATTACHING_MARKER_FILENAME)
+    }
+
+    pub fn tenant_ignore_mark_file_path(&self, tenant_id: TenantId) -> PathBuf {
+        self.tenant_path(&tenant_id).join(IGNORED_TENANT_FILE_NAME)
+    }
+
     /// Points to a place in pageserver's local directory,
     /// where certain tenant's tenantconf file should be located.
     pub fn tenant_config_path(&self, tenant_id: TenantId) -> PathBuf {
@@ -445,6 +470,28 @@ impl PageServerConf {
             .join(METADATA_FILE_NAME)
     }
 
+    /// Files on the remote storage are stored with paths, relative to the workdir.
+    /// That path includes in itself both tenant and timeline ids, allowing to have a unique remote storage path.
+    ///
+    /// Errors if the path provided does not start from pageserver's workdir.
+    pub fn remote_path(&self, local_path: &Path) -> anyhow::Result<RemotePath> {
+        local_path
+            .strip_prefix(&self.workdir)
+            .context("Failed to strip workdir prefix")
+            .and_then(RemotePath::new)
+            .with_context(|| {
+                format!(
+                    "Failed to resolve remote part of path {:?} for base {:?}",
+                    local_path, self.workdir
+                )
+            })
+    }
+
+    /// Turns storage remote path of a file into its local path.
+    pub fn local_path(&self, remote_path: &RemotePath) -> PathBuf {
+        remote_path.with_base(&self.workdir)
+    }
+
     //
     // Postgres distribution paths
     //
@@ -481,7 +528,7 @@ impl PageServerConf {
         let mut builder = PageServerConfigBuilder::default();
         builder.workdir(workdir.to_owned());
 
-        let mut t_conf: TenantConfOpt = Default::default();
+        let mut t_conf = TenantConfOpt::default();
 
         for (key, item) in toml.iter() {
             match key {
@@ -529,6 +576,7 @@ impl PageServerConf {
                     let permits = NonZeroUsize::new(permits).context("initial semaphore permits out of range: 0, use other configuration to disable a feature")?;
                     ConfigurableSemaphore::new(permits)
                 }),
+                "testing_mode" => builder.testing_mode(parse_toml_bool(key, item)?),
                 _ => bail!("unrecognized pageserver option '{key}'"),
             }
         }
@@ -612,6 +660,10 @@ impl PageServerConf {
         if let Some(max_lsn_wal_lag) = item.get("max_lsn_wal_lag") {
             t_conf.max_lsn_wal_lag = Some(parse_toml_from_str("max_lsn_wal_lag", max_lsn_wal_lag)?);
         }
+        if let Some(trace_read_requests) = item.get("trace_read_requests") {
+            t_conf.trace_read_requests =
+                Some(parse_toml_bool("trace_read_requests", trace_read_requests)?);
+        }
 
         Ok(t_conf)
     }
@@ -644,6 +696,7 @@ impl PageServerConf {
             broker_etcd_prefix: etcd_broker::DEFAULT_NEON_BROKER_ETCD_PREFIX.to_string(),
             log_format: LogFormat::from_str(defaults::DEFAULT_LOG_FORMAT).unwrap(),
             concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(),
+            testing_mode: true,
         }
     }
 }
@@ -657,6 +710,11 @@ fn parse_toml_string(name: &str, item: &Item) -> Result<String> {
     Ok(s.to_string())
 }
 
+fn parse_toml_bool(name: &str, item: &Item) -> Result<bool> {
+    item.as_bool()
+        .with_context(|| format!("configure option {name} is not a boolean"))
+}
+
 fn parse_toml_u64(name: &str, item: &Item) -> Result<u64> {
     // A toml integer is signed, so it cannot represent the full range of an u64. That's OK
     // for our use, though.
@@ -833,6 +891,7 @@ log_format = 'json'
                 broker_etcd_prefix: etcd_broker::DEFAULT_NEON_BROKER_ETCD_PREFIX.to_string(),
                 log_format: LogFormat::from_str(defaults::DEFAULT_LOG_FORMAT).unwrap(),
                 concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(),
+                testing_mode: defaults::DEFAULT_TESTING_MODE,
             },
             "Correct defaults should be used when no config values are provided"
         );
@@ -879,6 +938,7 @@ log_format = 'json'
                 broker_etcd_prefix: etcd_broker::DEFAULT_NEON_BROKER_ETCD_PREFIX.to_string(),
                 log_format: LogFormat::Json,
                 concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(),
+                testing_mode: defaults::DEFAULT_TESTING_MODE,
             },
             "Should be able to parse all basic config values correctly"
         );
@@ -1011,6 +1071,35 @@ broker_endpoints = ['{broker_endpoint}']
         Ok(())
     }
 
+    #[test]
+    fn parse_tenant_config() -> anyhow::Result<()> {
+        let tempdir = tempdir()?;
+        let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?;
+
+        let broker_endpoint = "http://127.0.0.1:7777";
+        let trace_read_requests = true;
+
+        let config_string = format!(
+            r#"{ALL_BASE_VALUES_TOML}
+pg_distrib_dir='{}'
+broker_endpoints = ['{broker_endpoint}']
+
+[tenant_config]
+trace_read_requests = {trace_read_requests}"#,
+            pg_distrib_dir.display(),
+        );
+
+        let toml = config_string.parse()?;
+
+        let conf = PageServerConf::parse_and_validate(&toml, &workdir)?;
+        assert_eq!(
+            conf.default_tenant_conf.trace_read_requests, trace_read_requests,
+            "Tenant config from pageserver config file should be parsed and udpated values used as defaults for all tenants",
+        );
+
+        Ok(())
+    }
+
     fn prepare_fs(tempdir: &TempDir) -> anyhow::Result<(PathBuf, PathBuf)> {
         let tempdir_path = tempdir.path();
 

pageserver/src/http/openapi_spec.yml

@@ -274,6 +274,7 @@ paths:
         schema:
           type: string
           format: hex
+
     post:
       description: Schedules attach operation to happen in the background for given tenant
       responses:
@@ -325,7 +326,9 @@ paths:
           type: string
           format: hex
     post:
-      description: Detach local tenant
+      description: |
+        Remove tenant data (including all corresponding timelines) from pageserver's memory and file system.
+        Files on the remote storage are not affected.
       responses:
         "200":
           description: Tenant detached
@@ -354,6 +357,92 @@ paths:
               schema:
                 $ref: "#/components/schemas/Error"
 
+  /v1/tenant/{tenant_id}/ignore:
+    parameters:
+      - name: tenant_id
+        in: path
+        required: true
+        schema:
+          type: string
+          format: hex
+    post:
+      description: |
+        Remove tenant data (including all corresponding timelines) from pageserver's memory.
+        Files on local disk and remote storage are not affected.
+
+        Future pageserver restarts won't load the data back until `load` is called on such tenant.
+      responses:
+        "200":
+          description: Tenant ignored
+        "400":
+          description: Error when no tenant id found in path parameters
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+
+  /v1/tenant/{tenant_id}/load:
+    parameters:
+      - name: tenant_id
+        in: path
+        required: true
+        schema:
+          type: string
+          format: hex
+    post:
+      description: |
+        Schedules an operation that attempts to load a tenant from the local disk and
+        synchronise it with the remote storage (if enabled), repeating pageserver's restart logic for tenant load.
+        If the tenant was ignored before, removes the ignore mark and continues with load scheduling.
+
+        Errors if the tenant is absent on disk, already present in memory or fails to schedule its load.
+        Scheduling a load does not mean that the tenant would load successfully, check tenant status to ensure load correctness.
+      responses:
+        "202":
+          description: Tenant scheduled to load successfully
+        "400":
+          description: Error when no tenant id found in path parameters
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+
   /v1/tenant/{tenant_id}/size:
     parameters:
       - name: tenant_id
@@ -604,13 +693,7 @@ components:
         id:
           type: string
         state:
-            oneOf:
-              - type: string
-              - type: object
-                properties:
-                  background_jobs_running:
-                    type: boolean
-
+          type: string
         current_physical_size:
           type: integer
         has_in_progress_downloads:
@@ -665,7 +748,6 @@ components:
         - tenant_id
         - last_record_lsn
         - disk_consistent_lsn
-        - awaits_download
         - state
         - latest_gc_cutoff_lsn
       properties:
@@ -708,8 +790,6 @@ components:
           format: hex
         last_received_msg_ts:
           type: integer
-        awaits_download:
-          type: boolean
         state:
           type: string
         latest_gc_cutoff_lsn:

pageserver/src/http/routes.rs

@@ -4,19 +4,17 @@ use anyhow::{anyhow, Context, Result};
 use hyper::StatusCode;
 use hyper::{Body, Request, Response, Uri};
 use remote_storage::GenericRemoteStorage;
-use tokio::task::JoinError;
 use tracing::*;
 
-use super::models::{LocalTimelineInfo, RemoteTimelineInfo, TimelineInfo};
 use super::models::{
-    StatusResponse, TenantConfigRequest, TenantCreateRequest, TenantCreateResponse, TenantInfo,
-    TimelineCreateRequest,
+    ConfigureFailpointsRequest, LocalTimelineInfo, RemoteTimelineInfo, StatusResponse,
+    TenantConfigRequest, TenantCreateRequest, TenantCreateResponse, TenantInfo,
+    TimelineCreateRequest, TimelineGcRequest, TimelineInfo,
 };
 use crate::pgdatadir_mapping::LsnForTimestamp;
-use crate::storage_sync;
-use crate::storage_sync::index::{RemoteIndex, RemoteTimeline};
-use crate::tenant::{TenantState, Timeline};
+use crate::tenant::Timeline;
 use crate::tenant_config::TenantConfOpt;
+use crate::CheckpointConfig;
 use crate::{config::PageServerConf, tenant_mgr};
 use utils::{
     auth::JwtAuth,
@@ -27,20 +25,13 @@ use utils::{
         request::parse_request_param,
         RequestExt, RouterBuilder,
     },
-    id::{TenantId, TenantTimelineId, TimelineId},
+    id::{TenantId, TimelineId},
     lsn::Lsn,
 };
 
-// Imports only used for testing APIs
-#[cfg(feature = "testing")]
-use super::models::{ConfigureFailpointsRequest, TimelineGcRequest};
-#[cfg(feature = "testing")]
-use crate::CheckpointConfig;
-
 struct State {
     conf: &'static PageServerConf,
     auth: Option<Arc<JwtAuth>>,
-    remote_index: RemoteIndex,
     allowlist_routes: Vec<Uri>,
     remote_storage: Option<GenericRemoteStorage>,
 }
@@ -49,7 +40,6 @@ impl State {
     fn new(
         conf: &'static PageServerConf,
         auth: Option<Arc<JwtAuth>>,
-        remote_index: RemoteIndex,
         remote_storage: Option<GenericRemoteStorage>,
     ) -> anyhow::Result<Self> {
         let allowlist_routes = ["/v1/status", "/v1/doc", "/swagger.yml"]
@@ -60,7 +50,6 @@ impl State {
             conf,
             auth,
             allowlist_routes,
-            remote_index,
             remote_storage,
         })
     }
@@ -86,12 +75,24 @@ fn check_permission(request: &Request<Body>, tenant_id: Option<TenantId>) -> Res
 }
 
 // Helper function to construct a TimelineInfo struct for a timeline
-async fn build_timeline_info(
-    state: &State,
+fn build_timeline_info(
     timeline: &Arc<Timeline>,
     include_non_incremental_logical_size: bool,
     include_non_incremental_physical_size: bool,
 ) -> anyhow::Result<TimelineInfo> {
+    let mut info = build_timeline_info_common(timeline)?;
+    if include_non_incremental_logical_size {
+        info.current_logical_size_non_incremental =
+            Some(timeline.get_current_logical_size_non_incremental(info.last_record_lsn)?);
+    }
+    if include_non_incremental_physical_size {
+        info.current_physical_size_non_incremental =
+            Some(timeline.get_physical_size_non_incremental()?)
+    }
+    Ok(info)
+}
+
+fn build_timeline_info_common(timeline: &Arc<Timeline>) -> anyhow::Result<TimelineInfo> {
     let last_record_lsn = timeline.get_last_record_lsn();
     let (wal_source_connstr, last_received_msg_lsn, last_received_msg_ts) = {
         let guard = timeline.last_received_wal.lock().unwrap();
@@ -106,22 +107,6 @@ async fn build_timeline_info(
         }
     };
 
-    let (remote_consistent_lsn, awaits_download) = if let Some(remote_entry) = state
-        .remote_index
-        .read()
-        .await
-        .timeline_entry(&TenantTimelineId {
-            tenant_id: timeline.tenant_id,
-            timeline_id: timeline.timeline_id,
-        }) {
-        (
-            Some(remote_entry.metadata.disk_consistent_lsn()),
-            remote_entry.awaits_download,
-        )
-    } else {
-        (None, false)
-    };
-
     let ancestor_timeline_id = timeline.get_ancestor_timeline_id();
     let ancestor_lsn = match timeline.get_ancestor_lsn() {
         Lsn(0) => None,
@@ -136,6 +121,7 @@ async fn build_timeline_info(
     };
     let current_physical_size = Some(timeline.get_physical_size());
     let state = timeline.current_state();
+    let remote_consistent_lsn = timeline.get_remote_consistent_lsn().unwrap_or(Lsn(0));
 
     let info = TimelineInfo {
         tenant_id: timeline.tenant_id,
@@ -143,28 +129,19 @@ async fn build_timeline_info(
         ancestor_timeline_id,
         ancestor_lsn,
         disk_consistent_lsn: timeline.get_disk_consistent_lsn(),
+        remote_consistent_lsn,
         last_record_lsn,
         prev_record_lsn: Some(timeline.get_prev_record_lsn()),
         latest_gc_cutoff_lsn: *timeline.get_latest_gc_cutoff_lsn(),
         current_logical_size,
         current_physical_size,
-        current_logical_size_non_incremental: if include_non_incremental_logical_size {
-            Some(timeline.get_current_logical_size_non_incremental(last_record_lsn)?)
-        } else {
-            None
-        },
-        current_physical_size_non_incremental: if include_non_incremental_physical_size {
-            Some(timeline.get_physical_size_non_incremental()?)
-        } else {
-            None
-        },
+        current_logical_size_non_incremental: None,
+        current_physical_size_non_incremental: None,
         wal_source_connstr,
         last_received_msg_lsn,
         last_received_msg_ts,
         pg_version: timeline.pg_version,
 
-        remote_consistent_lsn,
-        awaits_download,
         state,
 
         // Duplicate some fields in 'local' and 'remote' fields, for backwards-compatility
@@ -176,7 +153,7 @@ async fn build_timeline_info(
             current_physical_size,
         },
         remote: RemoteTimelineInfo {
-            remote_consistent_lsn,
+            remote_consistent_lsn: Some(remote_consistent_lsn),
         },
     };
     Ok(info)
@@ -194,34 +171,30 @@ async fn timeline_create_handler(mut request: Request<Body>) -> Result<Response<
     let request_data: TimelineCreateRequest = json_request(&mut request).await?;
     check_permission(&request, Some(tenant_id))?;
 
-    let state = get_state(&request);
+    let new_timeline_id = request_data
+        .new_timeline_id
+        .unwrap_or_else(TimelineId::generate);
 
-    let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
-    let new_timeline_info = async {
-        match tenant.create_timeline(
-            request_data.new_timeline_id.map(TimelineId::from),
-            request_data.ancestor_timeline_id.map(TimelineId::from),
-            request_data.ancestor_start_lsn,
-            request_data.pg_version.unwrap_or(crate::DEFAULT_PG_VERSION)
-        ).await {
-            Ok(Some(new_timeline)) => {
-                // Created. Construct a TimelineInfo for it.
-                let timeline_info = build_timeline_info(state, &new_timeline, false, false)
-                    .await
-                    .map_err(ApiError::InternalServerError)?;
-                Ok(Some(timeline_info))
-            }
-            Ok(None) => Ok(None), // timeline already exists
-            Err(err) => Err(ApiError::InternalServerError(err)),
+    let tenant = tenant_mgr::get_tenant(tenant_id, true)
+        .await
+        .map_err(ApiError::NotFound)?;
+    match tenant.create_timeline(
+        new_timeline_id,
+        request_data.ancestor_timeline_id.map(TimelineId::from),
+        request_data.ancestor_start_lsn,
+        request_data.pg_version.unwrap_or(crate::DEFAULT_PG_VERSION)
+    )
+    .instrument(info_span!("timeline_create", tenant = %tenant_id, new_timeline = ?request_data.new_timeline_id, timeline_id = %new_timeline_id, lsn=?request_data.ancestor_start_lsn, pg_version=?request_data.pg_version))
+    .await {
+        Ok(Some(new_timeline)) => {
+            // Created. Construct a TimelineInfo for it.
+            let timeline_info = build_timeline_info_common(&new_timeline)
+                .map_err(ApiError::InternalServerError)?;
+            json_response(StatusCode::CREATED, timeline_info)
         }
+        Ok(None) => json_response(StatusCode::CONFLICT, ()), // timeline already exists
+        Err(err) => Err(ApiError::InternalServerError(err)),
     }
-    .instrument(info_span!("timeline_create", tenant = %tenant_id, new_timeline = ?request_data.new_timeline_id, lsn=?request_data.ancestor_start_lsn, pg_version=?request_data.pg_version))
-        .await?;
-
-    Ok(match new_timeline_info {
-        Some(info) => json_response(StatusCode::CREATED, info)?,
-        None => json_response(StatusCode::CONFLICT, ())?,
-    })
 }
 
 async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -232,27 +205,29 @@ async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>,
         query_param_present(&request, "include-non-incremental-physical-size");
     check_permission(&request, Some(tenant_id))?;
 
-    let state = get_state(&request);
+    let response_data = async {
+        let tenant = tenant_mgr::get_tenant(tenant_id, true)
+            .await
+            .map_err(ApiError::NotFound)?;
+        let timelines = tenant.list_timelines();
 
-    let timelines = info_span!("timeline_list", tenant = %tenant_id).in_scope(|| {
-        let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
-        Ok(tenant.list_timelines())
-    })?;
+        let mut response_data = Vec::with_capacity(timelines.len());
+        for timeline in timelines {
+            let timeline_info = build_timeline_info(
+                &timeline,
+                include_non_incremental_logical_size,
+                include_non_incremental_physical_size,
+            )
+            .context("Failed to convert tenant timeline {timeline_id} into the local one: {e:?}")
+            .map_err(ApiError::InternalServerError)?;
 
-    let mut response_data = Vec::with_capacity(timelines.len());
-    for timeline in timelines {
-        let timeline_info = build_timeline_info(
-            state,
-            &timeline,
-            include_non_incremental_logical_size,
-            include_non_incremental_physical_size,
-        )
-        .await
-        .context("Failed to convert tenant timeline {timeline_id} into the local one: {e:?}")
-        .map_err(ApiError::InternalServerError)?;
+            response_data.push(timeline_info);
+        }
 
-        response_data.push(timeline_info);
+        Ok(response_data)
     }
+    .instrument(info_span!("timeline_list", tenant = %tenant_id))
+    .await?;
 
     json_response(StatusCode::OK, response_data)
 }
@@ -296,24 +271,20 @@ async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body
         query_param_present(&request, "include-non-incremental-physical-size");
     check_permission(&request, Some(tenant_id))?;
 
-    let state = get_state(&request);
-
     let timeline_info = async {
-        let timeline = tokio::task::spawn_blocking(move || {
-            tenant_mgr::get_tenant(tenant_id, true)?.get_timeline(timeline_id, false)
-        })
-        .await
-        .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))?;
+        let tenant = tenant_mgr::get_tenant(tenant_id, true)
+            .await
+            .map_err(ApiError::NotFound)?;
 
-        let timeline = timeline.map_err(ApiError::NotFound)?;
+        let timeline = tenant
+            .get_timeline(timeline_id, false)
+            .map_err(ApiError::NotFound)?;
 
         let timeline_info = build_timeline_info(
-            state,
             &timeline,
             include_non_incremental_logical_size,
             include_non_incremental_physical_size,
         )
-        .await
         .context("Failed to get local timeline info: {e:#}")
         .map_err(ApiError::InternalServerError)?;
 
@@ -337,6 +308,7 @@ async fn get_lsn_by_timestamp_handler(request: Request<Body>) -> Result<Response
     let timestamp_pg = postgres_ffi::to_pg_timestamp(timestamp);
 
     let timeline = tenant_mgr::get_tenant(tenant_id, true)
+        .await
         .and_then(|tenant| tenant.get_timeline(timeline_id, true))
         .map_err(ApiError::NotFound)?;
     let result = match timeline
@@ -358,117 +330,28 @@ async fn tenant_attach_handler(request: Request<Body>) -> Result<Response<Body>,
 
     info!("Handling tenant attach {tenant_id}");
 
-    tokio::task::spawn_blocking(move || match tenant_mgr::get_tenant(tenant_id, false) {
-        Ok(tenant) => {
-            if tenant.list_timelines().is_empty() {
-                info!("Attaching to tenant {tenant_id} with zero timelines");
-                Ok(())
-            } else {
-                Err(ApiError::Conflict(
-                    "Tenant is already present locally".to_owned(),
-                ))
-            }
-        }
-        Err(_) => Ok(()),
-    })
-    .await
-    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??;
-
     let state = get_state(&request);
-    let remote_index = &state.remote_index;
 
-    let mut index_accessor = remote_index.write().await;
-    if let Some(tenant_entry) = index_accessor.tenant_entry_mut(&tenant_id) {
-        if tenant_entry.has_in_progress_downloads() {
-            return Err(ApiError::Conflict(
-                "Tenant download is already in progress".to_string(),
-            ));
-        }
-
-        for (timeline_id, remote_timeline) in tenant_entry.iter_mut() {
-            storage_sync::schedule_layer_download(tenant_id, *timeline_id);
-            remote_timeline.awaits_download = true;
-        }
-        return json_response(StatusCode::ACCEPTED, ());
-    }
-    // no tenant in the index, release the lock to make the potentially lengthy download operation
-    drop(index_accessor);
-
-    // download index parts for every tenant timeline
-    let remote_timelines = match gather_tenant_timelines_index_parts(state, tenant_id).await {
-        Ok(Some(remote_timelines)) => remote_timelines,
-        Ok(None) => return Err(ApiError::NotFound(anyhow!("Unknown remote tenant"))),
-        Err(e) => {
-            error!("Failed to retrieve remote tenant data: {:?}", e);
-            return Err(ApiError::NotFound(anyhow!(
-                "Failed to retrieve remote tenant"
-            )));
-        }
-    };
-
-    // recheck that download is not in progress because
-    // we've released the lock to avoid holding it during the download
-    let mut index_accessor = remote_index.write().await;
-    let tenant_entry = match index_accessor.tenant_entry_mut(&tenant_id) {
-        Some(tenant_entry) => {
-            if tenant_entry.has_in_progress_downloads() {
-                return Err(ApiError::Conflict(
-                    "Tenant download is already in progress".to_string(),
-                ));
-            }
-            tenant_entry
-        }
-        None => index_accessor.add_tenant_entry(tenant_id),
-    };
-
-    // populate remote index with the data from index part and create directories on the local filesystem
-    for (timeline_id, mut remote_timeline) in remote_timelines {
-        tokio::fs::create_dir_all(state.conf.timeline_path(&timeline_id, &tenant_id))
+    if let Some(remote_storage) = &state.remote_storage {
+        // FIXME: distinguish between "Tenant already exists" and other errors
+        tenant_mgr::attach_tenant(state.conf, tenant_id, remote_storage.clone())
+            .instrument(info_span!("tenant_attach", tenant = %tenant_id))
             .await
-            .context("Failed to create new timeline directory")
             .map_err(ApiError::InternalServerError)?;
-
-        remote_timeline.awaits_download = true;
-        tenant_entry.insert(timeline_id, remote_timeline);
-        // schedule actual download
-        storage_sync::schedule_layer_download(tenant_id, timeline_id);
+    } else {
+        return Err(ApiError::BadRequest(anyhow!(
+            "attach_tenant is not possible because pageserver was configured without remote storage"
+        )));
     }
 
     json_response(StatusCode::ACCEPTED, ())
 }
 
-/// Note: is expensive from s3 access perspective,
-/// for details see comment to `storage_sync::gather_tenant_timelines_index_parts`
-async fn gather_tenant_timelines_index_parts(
-    state: &State,
-    tenant_id: TenantId,
-) -> anyhow::Result<Option<Vec<(TimelineId, RemoteTimeline)>>> {
-    let index_parts = match state.remote_storage.as_ref() {
-        Some(storage) => {
-            storage_sync::gather_tenant_timelines_index_parts(state.conf, storage, tenant_id).await
-        }
-        None => return Ok(None),
-    }
-    .with_context(|| format!("Failed to download index parts for tenant {tenant_id}"))?;
-
-    let mut remote_timelines = Vec::with_capacity(index_parts.len());
-    for (timeline_id, index_part) in index_parts {
-        let timeline_path = state.conf.timeline_path(&timeline_id, &tenant_id);
-        let remote_timeline = RemoteTimeline::from_index_part(&timeline_path, index_part)
-            .with_context(|| {
-                format!("Failed to convert index part into remote timeline for timeline {tenant_id}/{timeline_id}")
-            })?;
-        remote_timelines.push((timeline_id, remote_timeline));
-    }
-    Ok(Some(remote_timelines))
-}
-
 async fn timeline_delete_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     check_permission(&request, Some(tenant_id))?;
 
-    let state = get_state(&request);
     tenant_mgr::delete_timeline(tenant_id, timeline_id)
         .instrument(info_span!("timeline_delete", tenant = %tenant_id, timeline = %timeline_id))
         .await
@@ -477,12 +360,6 @@ async fn timeline_delete_handler(request: Request<Body>) -> Result<Response<Body
         // it.
         .map_err(ApiError::InternalServerError)?;
 
-    let mut remote_index = state.remote_index.write().await;
-    remote_index.remove_timeline_entry(TenantTimelineId {
-        tenant_id,
-        timeline_id,
-    });
-
     json_response(StatusCode::OK, ())
 }
 
@@ -499,8 +376,34 @@ async fn tenant_detach_handler(request: Request<Body>) -> Result<Response<Body>,
         // Replace this with better handling once the error type permits it.
         .map_err(ApiError::InternalServerError)?;
 
-    let mut remote_index = state.remote_index.write().await;
-    remote_index.remove_tenant_entry(&tenant_id);
+    json_response(StatusCode::OK, ())
+}
+
+async fn tenant_load_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
+    check_permission(&request, Some(tenant_id))?;
+
+    let state = get_state(&request);
+    tenant_mgr::load_tenant(state.conf, tenant_id, state.remote_storage.clone())
+        .instrument(info_span!("load", tenant = %tenant_id))
+        .await
+        .map_err(ApiError::InternalServerError)?;
+
+    json_response(StatusCode::ACCEPTED, ())
+}
+
+async fn tenant_ignore_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
+    check_permission(&request, Some(tenant_id))?;
+
+    let state = get_state(&request);
+    let conf = state.conf;
+    tenant_mgr::ignore_tenant(conf, tenant_id)
+        .instrument(info_span!("ignore_tenant", tenant = %tenant_id))
+        .await
+        // FIXME: Errors from `ignore_tenant` can be caused by both both user and internal errors.
+        // Replace this with better handling once the error type permits it.
+        .map_err(ApiError::InternalServerError)?;
 
     json_response(StatusCode::OK, ())
 }
@@ -508,16 +411,17 @@ async fn tenant_detach_handler(request: Request<Body>) -> Result<Response<Body>,
 async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     check_permission(&request, None)?;
 
-    let state = get_state(&request);
-    // clone to avoid holding the lock while awaiting for blocking task
-    let remote_index = state.remote_index.read().await.clone();
-
-    let response_data = tokio::task::spawn_blocking(move || {
-        let _enter = info_span!("tenant_list").entered();
-        crate::tenant_mgr::list_tenant_info(&remote_index)
-    })
-    .await
-    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))?;
+    let response_data = tenant_mgr::list_tenants()
+        .instrument(info_span!("tenant_list"))
+        .await
+        .iter()
+        .map(|(id, state)| TenantInfo {
+            id: *id,
+            state: *state,
+            current_physical_size: None,
+            has_in_progress_downloads: Some(state.has_in_progress_downloads()),
+        })
+        .collect::<Vec<TenantInfo>>();
 
     json_response(StatusCode::OK, response_data)
 }
@@ -526,58 +430,37 @@ async fn tenant_status(request: Request<Body>) -> Result<Response<Body>, ApiErro
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
     check_permission(&request, Some(tenant_id))?;
 
-    // if tenant is in progress of downloading it can be absent in global tenant map
-    let tenant = tenant_mgr::get_tenant(tenant_id, false);
+    let tenant_info = async {
+        let tenant = tenant_mgr::get_tenant(tenant_id, false).await?;
 
-    let state = get_state(&request);
-    let remote_index = &state.remote_index;
-
-    let index_accessor = remote_index.read().await;
-    let has_in_progress_downloads = index_accessor
-        .tenant_entry(&tenant_id)
-        .map(|t| t.has_in_progress_downloads())
-        .unwrap_or_else(|| {
-            info!("Tenant {tenant_id} not found in remote index");
-            false
-        });
-
-    let (tenant_state, current_physical_size) = match tenant {
-        Ok(tenant) => {
-            let timelines = tenant.list_timelines();
-            // Calculate total physical size of all timelines
-            let mut current_physical_size = 0;
-            for timeline in timelines {
-                current_physical_size += timeline.get_physical_size();
-            }
-
-            (tenant.current_state(), Some(current_physical_size))
+        // Calculate total physical size of all timelines
+        let mut current_physical_size = 0;
+        for timeline in tenant.list_timelines().iter() {
+            current_physical_size += timeline.get_physical_size();
         }
-        Err(e) => {
-            error!("Failed to get local tenant state: {e:#}");
-            if has_in_progress_downloads {
-                (TenantState::Paused, None)
-            } else {
-                (TenantState::Broken, None)
-            }
-        }
-    };
 
-    json_response(
-        StatusCode::OK,
-        TenantInfo {
+        let state = tenant.current_state();
+        Ok(TenantInfo {
             id: tenant_id,
-            state: tenant_state,
-            current_physical_size,
-            has_in_progress_downloads: Some(has_in_progress_downloads),
-        },
-    )
+            state,
+            current_physical_size: Some(current_physical_size),
+            has_in_progress_downloads: Some(state.has_in_progress_downloads()),
+        })
+    }
+    .instrument(info_span!("tenant_status_handler", tenant = %tenant_id))
+    .await
+    .map_err(ApiError::InternalServerError)?;
+
+    json_response(StatusCode::OK, tenant_info)
 }
 
 async fn tenant_size_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
     check_permission(&request, Some(tenant_id))?;
 
-    let tenant = tenant_mgr::get_tenant(tenant_id, false).map_err(ApiError::InternalServerError)?;
+    let tenant = tenant_mgr::get_tenant(tenant_id, true)
+        .await
+        .map_err(ApiError::InternalServerError)?;
 
     // this can be long operation, it currently is not backed by any request coalescing or similar
     let inputs = tenant
@@ -625,8 +508,6 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
     check_permission(&request, None)?;
 
     let request_data: TenantCreateRequest = json_request(&mut request).await?;
-    println!("tenant create: {:?}", request_data.trace_read_requests);
-    let remote_index = get_state(&request).remote_index.clone();
 
     let mut tenant_conf = TenantConfOpt::default();
     if let Some(gc_period) = request_data.gc_period {
@@ -696,20 +577,39 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
         .map(TenantId::from)
         .unwrap_or_else(TenantId::generate);
 
-    let new_tenant_id = tokio::task::spawn_blocking(move || {
-        let _enter = info_span!("tenant_create", tenant = ?target_tenant_id).entered();
-        let conf = get_config(&request);
+    let state = get_state(&request);
 
-        tenant_mgr::create_tenant(conf, tenant_conf, target_tenant_id, remote_index)
-            // FIXME: `create_tenant` can fail from both user and internal errors. Replace this
-            // with better error handling once the type permits it
-            .map_err(ApiError::InternalServerError)
-    })
+    let new_tenant = tenant_mgr::create_tenant(
+        state.conf,
+        tenant_conf,
+        target_tenant_id,
+        state.remote_storage.clone(),
+    )
+    .instrument(info_span!("tenant_create", tenant = ?target_tenant_id))
     .await
-    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??;
+    // FIXME: `create_tenant` can fail from both user and internal errors. Replace this
+    // with better error handling once the type permits it
+    .map_err(ApiError::InternalServerError)?;
 
-    Ok(match new_tenant_id {
-        Some(id) => json_response(StatusCode::CREATED, TenantCreateResponse(id))?,
+    Ok(match new_tenant {
+        Some(tenant) => {
+            // We created the tenant. Existing API semantics are that the tenant
+            // is Active when this function returns.
+            if let res @ Err(_) = tenant.wait_to_become_active().await {
+                // This shouldn't happen because we just created the tenant directory
+                // in tenant_mgr::create_tenant, and there aren't any remote timelines
+                // to load, so, nothing can really fail during load.
+                // Don't do cleanup because we don't know how we got here.
+                // The tenant will likely be in `Broken` state and subsequent
+                // calls will fail.
+                res.context("created tenant failed to become active")
+                    .map_err(ApiError::InternalServerError)?;
+            }
+            json_response(
+                StatusCode::CREATED,
+                TenantCreateResponse(tenant.tenant_id()),
+            )?
+        }
         None => json_response(StatusCode::CONFLICT, ())?,
     })
 }
@@ -780,22 +680,17 @@ async fn tenant_config_handler(mut request: Request<Body>) -> Result<Response<Bo
         );
     }
 
-    tokio::task::spawn_blocking(move || {
-        let _enter = info_span!("tenant_config", tenant = ?tenant_id).entered();
-
-        let state = get_state(&request);
-        tenant_mgr::update_tenant_config(state.conf, tenant_conf, tenant_id)
-            // FIXME: `update_tenant_config` can fail because of both user and internal errors.
-            // Replace this `map_err` with better error handling once the type permits it
-            .map_err(ApiError::InternalServerError)
-    })
-    .await
-    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??;
+    let state = get_state(&request);
+    tenant_mgr::update_tenant_config(state.conf, tenant_conf, tenant_id)
+        .instrument(info_span!("tenant_config", tenant = ?tenant_id))
+        .await
+        // FIXME: `update_tenant_config` can fail because of both user and internal errors.
+        // Replace this `map_err` with better error handling once the type permits it
+        .map_err(ApiError::InternalServerError)?;
 
     json_response(StatusCode::OK, ())
 }
 
-#[cfg(feature = "testing")]
 async fn failpoints_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
     if !fail::has_failpoints() {
         return Err(ApiError::BadRequest(anyhow!(
@@ -829,54 +724,52 @@ async fn failpoints_handler(mut request: Request<Body>) -> Result<Response<Body>
 }
 
 // Run GC immediately on given timeline.
-#[cfg(feature = "testing")]
 async fn timeline_gc_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     check_permission(&request, Some(tenant_id))?;
 
-    // FIXME: currently this will return a 500 error on bad tenant id; it should be 4XX
-    let tenant = tenant_mgr::get_tenant(tenant_id, false).map_err(ApiError::NotFound)?;
     let gc_req: TimelineGcRequest = json_request(&mut request).await?;
 
-    let gc_horizon = gc_req.gc_horizon.unwrap_or_else(|| tenant.get_gc_horizon());
-
-    // Use tenant's pitr setting
-    let pitr = tenant.get_pitr_interval();
-    let result = tenant
-        .gc_iteration(Some(timeline_id), gc_horizon, pitr, true)
-        .instrument(info_span!("manual_gc", tenant = %tenant_id, timeline = %timeline_id))
+    let wait_task_done = tenant_mgr::immediate_gc(tenant_id, timeline_id, gc_req).await?;
+    let gc_result = wait_task_done
         .await
-        // FIXME: `gc_iteration` can return an error for multiple reasons; we should handle it
-        // better once the types support it.
+        .context("wait for gc task")
+        .map_err(ApiError::InternalServerError)?
         .map_err(ApiError::InternalServerError)?;
-    json_response(StatusCode::OK, result)
+
+    json_response(StatusCode::OK, gc_result)
 }
 
 // Run compaction immediately on given timeline.
-#[cfg(feature = "testing")]
 async fn timeline_compact_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     check_permission(&request, Some(tenant_id))?;
 
-    let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
+    let tenant = tenant_mgr::get_tenant(tenant_id, true)
+        .await
+        .map_err(ApiError::NotFound)?;
     let timeline = tenant
         .get_timeline(timeline_id, true)
         .map_err(ApiError::NotFound)?;
-    timeline.compact().map_err(ApiError::InternalServerError)?;
+    timeline
+        .compact()
+        .await
+        .map_err(ApiError::InternalServerError)?;
 
     json_response(StatusCode::OK, ())
 }
 
 // Run checkpoint immediately on given timeline.
-#[cfg(feature = "testing")]
 async fn timeline_checkpoint_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     check_permission(&request, Some(tenant_id))?;
 
-    let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
+    let tenant = tenant_mgr::get_tenant(tenant_id, true)
+        .await
+        .map_err(ApiError::NotFound)?;
     let timeline = tenant
         .get_timeline(timeline_id, true)
         .map_err(ApiError::NotFound)?;
@@ -898,7 +791,6 @@ async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
 pub fn make_router(
     conf: &'static PageServerConf,
     auth: Option<Arc<JwtAuth>>,
-    remote_index: RemoteIndex,
     remote_storage: Option<GenericRemoteStorage>,
 ) -> anyhow::Result<RouterBuilder<hyper::Body, ApiError>> {
     let spec = include_bytes!("openapi_spec.yml");
@@ -914,29 +806,32 @@ pub fn make_router(
         }))
     }
 
+    // A wrapper around a handler function that returns an error if the server
+    // was not configured with testing_mode enabled. This is used to gate API
+    // functions that should only be used in tests, never in production.
     macro_rules! testing_api {
         ($handler_desc:literal, $handler:path $(,)?) => {{
-            #[cfg(not(feature = "testing"))]
-            async fn cfg_disabled(_req: Request<Body>) -> Result<Response<Body>, ApiError> {
-                Err(ApiError::BadRequest(anyhow!(concat!(
-                    "Cannot ",
-                    $handler_desc,
-                    " because pageserver was compiled without testing APIs",
-                ))))
+            use futures::FutureExt;
+            |req: Request<Body>| {
+                if conf.testing_mode {
+                    $handler(req).left_future()
+                } else {
+                    async {
+                        Err(ApiError::BadRequest(anyhow!(concat!(
+                            "Cannot ",
+                            $handler_desc,
+                            " because pageserver was configured without testing APIs",
+                        ))))
+                    }
+                    .right_future()
+                }
             }
-
-            #[cfg(feature = "testing")]
-            let handler = $handler;
-            #[cfg(not(feature = "testing"))]
-            let handler = cfg_disabled;
-            handler
         }};
     }
 
     Ok(router
         .data(Arc::new(
-            State::new(conf, auth, remote_index, remote_storage)
-                .context("Failed to initialize router state")?,
+            State::new(conf, auth, remote_storage).context("Failed to initialize router state")?,
         ))
         .get("/v1/status", status_handler)
         .put(
@@ -952,6 +847,8 @@ pub fn make_router(
         .post("/v1/tenant/:tenant_id/timeline", timeline_create_handler)
         .post("/v1/tenant/:tenant_id/attach", tenant_attach_handler)
         .post("/v1/tenant/:tenant_id/detach", tenant_detach_handler)
+        .post("/v1/tenant/:tenant_id/load", tenant_load_handler)
+        .post("/v1/tenant/:tenant_id/ignore", tenant_ignore_handler)
         .get(
             "/v1/tenant/:tenant_id/timeline/:timeline_id",
             timeline_detail_handler,

pageserver/src/lib.rs

@@ -4,13 +4,14 @@ pub mod config;
 pub mod http;
 pub mod import_datadir;
 pub mod keyspace;
-pub mod metrics;
+pub(crate) mod metrics;
 pub mod page_cache;
 pub mod page_service;
 pub mod pgdatadir_mapping;
 pub mod profiling;
 pub mod repository;
-pub mod storage_sync;
+pub mod storage_sync2;
+pub use storage_sync2 as storage_sync;
 pub mod task_mgr;
 pub mod tenant;
 pub mod tenant_config;
@@ -23,11 +24,9 @@ pub mod walreceiver;
 pub mod walrecord;
 pub mod walredo;
 
-use std::collections::HashMap;
 use std::path::Path;
 
 use tracing::info;
-use utils::id::{TenantId, TimelineId};
 
 use crate::task_mgr::TaskKind;
 
@@ -73,7 +72,7 @@ pub async fn shutdown_pageserver(exit_code: i32) {
     //
     // FIXME: Does this wait for the sync tasks to finish syncing what's queued up?
     // Should it?
-    task_mgr::shutdown_tasks(Some(TaskKind::StorageSync), None, None).await;
+    task_mgr::shutdown_tasks(Some(TaskKind::RemoteUploadTask), None, None).await;
 
     // Shut down the HTTP endpoint last, so that you can still check the server's
     // status while it's shutting down.
@@ -108,20 +107,6 @@ fn exponential_backoff_duration_seconds(n: u32, base_increment: f64, max_seconds
     }
 }
 
-/// A newtype to store arbitrary data grouped by tenant and timeline ids.
-/// One could use [`utils::id::TenantTimelineId`] for grouping, but that would
-/// not include the cases where a certain tenant has zero timelines.
-/// This is sometimes important: a tenant could be registered during initial load from FS,
-/// even if he has no timelines on disk.
-#[derive(Debug)]
-pub struct TenantTimelineValues<T>(HashMap<TenantId, HashMap<TimelineId, T>>);
-
-impl<T> TenantTimelineValues<T> {
-    fn new() -> Self {
-        Self(HashMap::new())
-    }
-}
-
 /// The name of the metadata file pageserver creates per timeline.
 /// Full path: `tenants/<tenant_id>/timelines/<timeline_id>/metadata`.
 pub const METADATA_FILE_NAME: &str = "metadata";
@@ -140,6 +125,13 @@ pub const TEMP_FILE_SUFFIX: &str = "___temp";
 /// Full path: `tenants/<tenant_id>/timelines/<timeline_id>___uninit`.
 pub const TIMELINE_UNINIT_MARK_SUFFIX: &str = "___uninit";
 
+/// A marker file to prevent pageserver from loading a certain tenant on restart.
+/// Different from [`TIMELINE_UNINIT_MARK_SUFFIX`] due to semantics of the corresponding
+/// `ignore` management API command, that expects the ignored tenant to be properly loaded
+/// into pageserver's memory before being ignored.
+/// Full path: `tenants/<tenant_id>/___ignored_tenant`.
+pub const IGNORED_TENANT_FILE_NAME: &str = "___ignored_tenant";
+
 pub fn is_temporary(path: &Path) -> bool {
     match path.file_name() {
         Some(name) => name.to_string_lossy().ends_with(TEMP_FILE_SUFFIX),
@@ -156,6 +148,28 @@ pub fn is_uninit_mark(path: &Path) -> bool {
     }
 }
 
+///
+/// Wrapper around fail::fail_point! macro that returns quickly if testing_mode was
+/// disabled in the pageserver config. Also enabled in unit tests.
+///
+/// fail::fail_point! is fairly quick, but it does acquire an RwLock and perform a HashMap
+/// lookup. This macro is hopefully cheap enough that we don't need to worry about the
+/// overhead even in production, and even if the macro is used in hot spots. (This check
+/// compiles to two cmp instructions; get_unchecked() would shrink it to one.)
+///
+#[macro_export]
+macro_rules! fail_point {
+    ($($name:expr),*) => {{
+        if cfg!(test) || *$crate::TESTING_MODE.get().expect("testing_mode not initialized") {
+            fail::fail_point!($($name), *)
+        }
+    }};
+}
+
+/// This is set early in the pageserver startup, from the "testing_mode" setting in the
+/// config file.
+pub static TESTING_MODE: once_cell::sync::OnceCell<bool> = once_cell::sync::OnceCell::new();
+
 #[cfg(test)]
 mod backoff_defaults_tests {
     use super::*;

pageserver/src/metrics.rs

@@ -1,9 +1,8 @@
 use metrics::core::{AtomicU64, GenericCounter};
 use metrics::{
-    register_gauge_vec, register_histogram, register_histogram_vec, register_int_counter,
-    register_int_counter_vec, register_int_gauge, register_int_gauge_vec, register_uint_gauge_vec,
-    GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge,
-    UIntGaugeVec,
+    register_histogram, register_histogram_vec, register_int_counter, register_int_counter_vec,
+    register_int_gauge, register_int_gauge_vec, register_uint_gauge_vec, Histogram, HistogramVec,
+    IntCounter, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
 };
 use once_cell::sync::Lazy;
 use utils::id::{TenantId, TimelineId};
@@ -200,63 +199,59 @@ pub static NUM_ONDISK_LAYERS: Lazy<IntGauge> = Lazy::new(|| {
         .expect("failed to define a metric")
 });
 
-pub static REMAINING_SYNC_ITEMS: Lazy<IntGauge> = Lazy::new(|| {
-    register_int_gauge!(
-        "pageserver_remote_storage_remaining_sync_items",
-        "Number of storage sync items left in the queue"
+// remote storage metrics
+
+pub static REMOTE_UPLOAD_QUEUE_UNFINISHED_TASKS: Lazy<IntGaugeVec> = Lazy::new(|| {
+    register_int_gauge_vec!(
+        "pageserver_remote_upload_queue_unfinished_tasks",
+        "Number of tasks in the upload queue that are not finished yet.",
+        &["tenant_id", "timeline_id", "file_kind", "op_kind"],
     )
-    .expect("failed to register pageserver remote storage remaining sync items int gauge")
+    .expect("failed to define a metric")
 });
 
-pub static IMAGE_SYNC_TIME: Lazy<GaugeVec> = Lazy::new(|| {
-    register_gauge_vec!(
-        "pageserver_remote_storage_image_sync_duration",
-        "Time spent to synchronize (up/download) a whole pageserver image",
-        &["tenant_id", "timeline_id"],
-    )
-    .expect("failed to register per-timeline pageserver image sync time vec")
-});
+#[derive(Debug, Clone, Copy)]
+pub enum RemoteOpKind {
+    Upload,
+    Download,
+    Delete,
+}
+impl RemoteOpKind {
+    pub fn as_str(&self) -> &str {
+        match self {
+            Self::Upload => "upload",
+            Self::Download => "download",
+            Self::Delete => "delete",
+        }
+    }
+}
 
-pub static IMAGE_SYNC_OPERATION_KINDS: &[&str] = &["upload", "download", "delete"];
-pub static IMAGE_SYNC_STATUS: &[&str] = &["success", "failure", "abort"];
+#[derive(Debug, Clone, Copy)]
+pub enum RemoteOpFileKind {
+    Layer,
+    Index,
+}
+impl RemoteOpFileKind {
+    pub fn as_str(&self) -> &str {
+        match self {
+            Self::Layer => "layer",
+            Self::Index => "index",
+        }
+    }
+}
 
-pub static IMAGE_SYNC_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
-    register_int_counter_vec!(
-        "pageserver_remote_storage_image_sync_count",
-        "Number of synchronization operations executed for pageserver images. \
-        Grouped by tenant, timeline, operation_kind and status",
-        &["tenant_id", "timeline_id", "operation_kind", "status"]
-    )
-    .expect("failed to register pageserver image sync count vec")
-});
+pub static REMOTE_OPERATION_KINDS: &[&str] = &["upload", "download", "delete"];
+pub static REMOTE_OPERATION_FILE_KINDS: &[&str] = &["layer", "index"];
+pub static REMOTE_OPERATION_STATUSES: &[&str] = &["success", "failure"];
 
-pub static IMAGE_SYNC_TIME_HISTOGRAM: Lazy<HistogramVec> = Lazy::new(|| {
+pub static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
     register_histogram_vec!(
-        "pageserver_remote_storage_image_sync_seconds",
-        "Time took to synchronize (download or upload) a whole pageserver image. \
-        Grouped by operation_kind and status",
-        &["operation_kind", "status"],
-        vec![0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 3.0, 10.0, 20.0]
+        "pageserver_remote_operation_seconds",
+        "Time spent on remote storage operations. \
+        Grouped by tenant, timeline, operation_kind and status",
+        &["tenant_id", "timeline_id", "file_kind", "op_kind", "status"]
     )
-    .expect("failed to register pageserver image sync time histogram vec")
-});
-
-pub static REMOTE_INDEX_UPLOAD: Lazy<IntCounterVec> = Lazy::new(|| {
-    register_int_counter_vec!(
-        "pageserver_remote_storage_remote_index_uploads_total",
-        "Number of remote index uploads",
-        &["tenant_id", "timeline_id"],
-    )
-    .expect("failed to register pageserver remote index upload vec")
-});
-
-pub static NO_LAYERS_UPLOAD: Lazy<IntCounterVec> = Lazy::new(|| {
-    register_int_counter_vec!(
-        "pageserver_remote_storage_no_layers_uploads_total",
-        "Number of skipped uploads due to no layers",
-        &["tenant_id", "timeline_id"],
-    )
-    .expect("failed to register pageserver no layers upload vec")
+    .expect("failed to define a metric")
 });
 
 pub static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
@@ -473,16 +468,90 @@ impl Drop for TimelineMetrics {
             let _ = SMGR_QUERY_TIME.remove_label_values(&[op, tenant_id, timeline_id]);
         }
 
-        for op in IMAGE_SYNC_OPERATION_KINDS {
-            for status in IMAGE_SYNC_STATUS {
-                let _ = IMAGE_SYNC_COUNT.remove_label_values(&[tenant_id, timeline_id, op, status]);
+        let _ = REMOTE_UPLOAD_QUEUE_UNFINISHED_TASKS.remove_label_values(&[tenant_id, timeline_id]);
+        for file_kind in REMOTE_OPERATION_FILE_KINDS {
+            for op in REMOTE_OPERATION_KINDS {
+                for status in REMOTE_OPERATION_STATUSES {
+                    let _ = REMOTE_OPERATION_TIME.remove_label_values(&[
+                        tenant_id,
+                        timeline_id,
+                        file_kind,
+                        op,
+                        status,
+                    ]);
+                }
             }
         }
-
-        let _ = IMAGE_SYNC_TIME.remove_label_values(&[tenant_id, timeline_id]);
     }
 }
 
 pub fn remove_tenant_metrics(tenant_id: &TenantId) {
     let _ = STORAGE_TIME.remove_label_values(&["gc", &tenant_id.to_string(), "-"]);
 }
+
+use futures::Future;
+use pin_project_lite::pin_project;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use std::time::Instant;
+
+/// Wrapper future that measures the time spent by a remote storage operation,
+/// and records the time and success/failure as a prometheus metric.
+pub trait MeasureRemoteOp: Sized {
+    fn measure_remote_op(
+        self,
+        tenant_id: TenantId,
+        timeline_id: TimelineId,
+        file_kind: RemoteOpFileKind,
+        op: RemoteOpKind,
+    ) -> MeasuredRemoteOp<Self> {
+        let start = Instant::now();
+        MeasuredRemoteOp {
+            inner: self,
+            tenant_id,
+            timeline_id,
+            file_kind,
+            op,
+            start,
+        }
+    }
+}
+
+impl<T: Sized> MeasureRemoteOp for T {}
+
+pin_project! {
+    pub struct MeasuredRemoteOp<F>
+    {
+        #[pin]
+        inner: F,
+        tenant_id: TenantId,
+        timeline_id: TimelineId,
+        file_kind: RemoteOpFileKind,
+        op: RemoteOpKind,
+        start: Instant,
+    }
+}
+
+impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
+    type Output = Result<O, E>;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        let this = self.project();
+        let poll_result = this.inner.poll(cx);
+        if let Poll::Ready(ref res) = poll_result {
+            let duration = this.start.elapsed();
+            let status = if res.is_ok() { &"success" } else { &"failure" };
+            REMOTE_OPERATION_TIME
+                .get_metric_with_label_values(&[
+                    &this.tenant_id.to_string(),
+                    &this.timeline_id.to_string(),
+                    this.file_kind.as_str(),
+                    this.op.as_str(),
+                    status,
+                ])
+                .unwrap()
+                .observe(duration.as_secs_f64());
+        }
+        poll_result
+    }
+}

pageserver/src/page_service.rs

@@ -25,6 +25,7 @@ use std::net::TcpListener;
 use std::str;
 use std::str::FromStr;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::pin;
 use tokio_util::io::StreamReader;
 use tokio_util::io::SyncIoBridge;
@@ -47,7 +48,7 @@ use crate::metrics::{LIVE_CONNECTIONS_COUNT, SMGR_QUERY_TIME};
 use crate::profiling::profpoint_start;
 use crate::task_mgr;
 use crate::task_mgr::TaskKind;
-use crate::tenant::Timeline;
+use crate::tenant::{Tenant, Timeline};
 use crate::tenant_mgr;
 use crate::trace::Tracer;
 use crate::CheckpointConfig;
@@ -279,7 +280,7 @@ impl PageServerHandler {
         task_mgr::associate_with(Some(tenant_id), Some(timeline_id));
 
         // Make request tracer if needed
-        let tenant = tenant_mgr::get_tenant(tenant_id, true)?;
+        let tenant = get_active_tenant_with_timeout(tenant_id).await?;
         let mut tracer = if tenant.get_trace_read_requests() {
             let connection_id = ConnectionId::generate();
             let path = tenant
@@ -291,7 +292,7 @@ impl PageServerHandler {
         };
 
         // Check that the timeline exists
-        let timeline = get_local_timeline(tenant_id, timeline_id)?;
+        let timeline = tenant.get_timeline(timeline_id, true)?;
 
         // switch client to COPYBOTH
         pgb.write_message(&BeMessage::CopyBothResponse)?;
@@ -314,6 +315,7 @@ impl PageServerHandler {
 
             let copy_data_bytes = match msg? {
                 Some(FeMessage::CopyData(bytes)) => bytes,
+                Some(FeMessage::Terminate) => break,
                 Some(m) => {
                     bail!("unexpected message: {m:?} during COPY");
                 }
@@ -376,7 +378,7 @@ impl PageServerHandler {
         task_mgr::associate_with(Some(tenant_id), Some(timeline_id));
         // Create empty timeline
         info!("creating new timeline");
-        let tenant = tenant_mgr::get_tenant(tenant_id, true)?;
+        let tenant = get_active_tenant_with_timeout(tenant_id).await?;
         let timeline = tenant.create_empty_timeline(timeline_id, base_lsn, pg_version)?;
 
         // TODO mark timeline as not ready until it reaches end_lsn.
@@ -431,7 +433,7 @@ impl PageServerHandler {
     ) -> anyhow::Result<()> {
         task_mgr::associate_with(Some(tenant_id), Some(timeline_id));
 
-        let timeline = get_local_timeline(tenant_id, timeline_id)?;
+        let timeline = get_active_timeline_with_timeout(tenant_id, timeline_id).await?;
         ensure!(timeline.get_last_record_lsn() == start_lsn);
 
         // TODO leave clean state on error. For now you can use detach to clean
@@ -624,7 +626,7 @@ impl PageServerHandler {
         full_backup: bool,
     ) -> anyhow::Result<()> {
         // check that the timeline exists
-        let timeline = get_local_timeline(tenant_id, timeline_id)?;
+        let timeline = get_active_timeline_with_timeout(tenant_id, timeline_id).await?;
         let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
         if let Some(lsn) = lsn {
             // Backup was requested at a particular LSN. Wait for it to arrive.
@@ -766,7 +768,7 @@ impl postgres_backend_async::Handler for PageServerHandler {
             let timeline_id = TimelineId::from_str(params[1])?;
 
             self.check_permission(Some(tenant_id))?;
-            let timeline = get_local_timeline(tenant_id, timeline_id)?;
+            let timeline = get_active_timeline_with_timeout(tenant_id, timeline_id).await?;
 
             let end_of_timeline = timeline.get_last_record_rlsn();
 
@@ -889,7 +891,7 @@ impl postgres_backend_async::Handler for PageServerHandler {
 
             self.check_permission(Some(tenant_id))?;
 
-            let tenant = tenant_mgr::get_tenant(tenant_id, true)?;
+            let tenant = get_active_tenant_with_timeout(tenant_id).await?;
             pgb.write_message(&BeMessage::RowDescription(&[
                 RowDescriptor::int8_col(b"checkpoint_distance"),
                 RowDescriptor::int8_col(b"checkpoint_timeout"),
@@ -933,8 +935,28 @@ impl postgres_backend_async::Handler for PageServerHandler {
     }
 }
 
-fn get_local_timeline(tenant_id: TenantId, timeline_id: TimelineId) -> Result<Arc<Timeline>> {
-    tenant_mgr::get_tenant(tenant_id, true)
+/// Get active tenant.
+///
+/// If the tenant is Loading, waits for it to become Active, for up to 30 s. That
+/// ensures that queries don't fail immediately after pageserver startup, because
+/// all tenants are still loading.
+async fn get_active_tenant_with_timeout(tenant_id: TenantId) -> Result<Arc<Tenant>> {
+    let tenant = tenant_mgr::get_tenant(tenant_id, false).await?;
+    match tokio::time::timeout(Duration::from_secs(30), tenant.wait_to_become_active()).await {
+        Ok(wait_result) => wait_result
+            // no .context(), the error message is good enough and some tests depend on it
+            .map(move |()| tenant),
+        Err(_) => anyhow::bail!("Timeout waiting for tenant {tenant_id} to become Active"),
+    }
+}
+
+/// Shorthand for getting a reference to a Timeline of an Active tenant.
+async fn get_active_timeline_with_timeout(
+    tenant_id: TenantId,
+    timeline_id: TimelineId,
+) -> Result<Arc<Timeline>> {
+    get_active_tenant_with_timeout(tenant_id)
+        .await
         .and_then(|tenant| tenant.get_timeline(timeline_id, true))
 }
 

pageserver/src/repository.rs

@@ -189,7 +189,7 @@ impl Value {
 ///
 /// Result of performing GC
 ///
-#[derive(Default, Serialize)]
+#[derive(Default, Serialize, Debug)]
 pub struct GcResult {
     pub layers_total: u64,
     pub layers_needed_by_cutoff: u64,

pageserver/src/storage_sync/delete.rs

@@ -1,235 +0,0 @@
-//! Timeline synchronization logic to delete a bulk of timeline's remote files from the remote storage.
-
-use std::path::Path;
-
-use anyhow::Context;
-use futures::stream::{FuturesUnordered, StreamExt};
-use tracing::{debug, error, info};
-
-use crate::storage_sync::{SyncQueue, SyncTask};
-use remote_storage::GenericRemoteStorage;
-use utils::id::TenantTimelineId;
-
-use super::{LayersDeletion, SyncData};
-
-/// Attempts to remove the timleline layers from the remote storage.
-/// If the task had not adjusted the metadata before, the deletion will fail.
-pub(super) async fn delete_timeline_layers(
-    storage: &GenericRemoteStorage,
-    sync_queue: &SyncQueue,
-    sync_id: TenantTimelineId,
-    mut delete_data: SyncData<LayersDeletion>,
-) -> bool {
-    if !delete_data.data.deletion_registered {
-        error!("Cannot delete timeline layers before the deletion metadata is not registered, reenqueueing");
-        delete_data.retries += 1;
-        sync_queue.push(sync_id, SyncTask::Delete(delete_data));
-        return false;
-    }
-
-    if delete_data.data.layers_to_delete.is_empty() {
-        info!("No layers to delete, skipping");
-        return true;
-    }
-
-    let layers_to_delete = delete_data
-        .data
-        .layers_to_delete
-        .drain()
-        .collect::<Vec<_>>();
-    debug!("Layers to delete: {layers_to_delete:?}");
-    info!("Deleting {} timeline layers", layers_to_delete.len());
-
-    let mut delete_tasks = layers_to_delete
-        .into_iter()
-        .map(|local_layer_path| async {
-            match remove_storage_object(storage, &local_layer_path).await {
-                Ok(()) => Ok(local_layer_path),
-                Err(e) => Err((e, local_layer_path)),
-            }
-        })
-        .collect::<FuturesUnordered<_>>();
-
-    let mut errored = false;
-    while let Some(deletion_result) = delete_tasks.next().await {
-        match deletion_result {
-            Ok(local_layer_path) => {
-                debug!(
-                    "Successfully deleted layer {} for timeline {sync_id}",
-                    local_layer_path.display()
-                );
-                delete_data.data.deleted_layers.insert(local_layer_path);
-            }
-            Err((e, local_layer_path)) => {
-                errored = true;
-                error!(
-                    "Failed to delete layer {} for timeline {sync_id}: {e:?}",
-                    local_layer_path.display()
-                );
-                delete_data.data.layers_to_delete.insert(local_layer_path);
-            }
-        }
-    }
-
-    if errored {
-        debug!("Reenqueuing failed delete task for timeline {sync_id}");
-        delete_data.retries += 1;
-        sync_queue.push(sync_id, SyncTask::Delete(delete_data));
-    } else {
-        info!("Successfully deleted all layers");
-    }
-    errored
-}
-
-async fn remove_storage_object(
-    storage: &GenericRemoteStorage,
-    local_layer_path: &Path,
-) -> anyhow::Result<()> {
-    let storage_path = storage
-        .remote_object_id(local_layer_path)
-        .with_context(|| {
-            format!(
-                "Failed to get the layer storage path for local path '{}'",
-                local_layer_path.display()
-            )
-        })?;
-
-    storage.delete(&storage_path).await.with_context(|| {
-        format!(
-            "Failed to delete remote layer from storage at '{:?}'",
-            storage_path
-        )
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use std::{collections::HashSet, num::NonZeroUsize};
-
-    use itertools::Itertools;
-    use tempfile::tempdir;
-    use tokio::fs;
-    use utils::lsn::Lsn;
-
-    use crate::{
-        storage_sync::test_utils::{create_local_timeline, dummy_metadata},
-        tenant::harness::{TenantHarness, TIMELINE_ID},
-    };
-    use remote_storage::{LocalFs, RemoteStorage};
-
-    use super::*;
-
-    #[tokio::test]
-    async fn delete_timeline_negative() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("delete_timeline_negative")?;
-        let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_path_buf(),
-            harness.conf.workdir.clone(),
-        )?);
-
-        let deleted = delete_timeline_layers(
-            &storage,
-            &sync_queue,
-            sync_id,
-            SyncData {
-                retries: 1,
-                data: LayersDeletion {
-                    deleted_layers: HashSet::new(),
-                    layers_to_delete: HashSet::new(),
-                    deletion_registered: false,
-                },
-            },
-        )
-        .await;
-
-        assert!(
-            !deleted,
-            "Should not start the deletion for task with delete metadata unregistered"
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn delete_timeline() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("delete_timeline")?;
-        let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
-
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-        let layer_files = ["a", "b", "c", "d"];
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_path_buf(),
-            harness.conf.workdir.clone(),
-        )?);
-
-        let local_storage = storage.as_local().unwrap();
-
-        let current_retries = 3;
-        let metadata = dummy_metadata(Lsn(0x30));
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let timeline_upload =
-            create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?;
-        for (local_path, _metadata) in timeline_upload.layers_to_upload {
-            let remote_path =
-                local_storage.resolve_in_storage(&local_storage.remote_object_id(&local_path)?)?;
-            let remote_parent_dir = remote_path.parent().unwrap();
-            if !remote_parent_dir.exists() {
-                fs::create_dir_all(&remote_parent_dir).await?;
-            }
-            fs::copy(&local_path, &remote_path).await?;
-        }
-        assert_eq!(
-            local_storage
-                .list()
-                .await?
-                .into_iter()
-                .map(|remote_path| local_storage.local_path(&remote_path).unwrap())
-                .filter_map(|local_path| { Some(local_path.file_name()?.to_str()?.to_owned()) })
-                .sorted()
-                .collect::<Vec<_>>(),
-            layer_files
-                .iter()
-                .map(|layer_str| layer_str.to_string())
-                .sorted()
-                .collect::<Vec<_>>(),
-            "Expect to have all layer files remotely before deletion"
-        );
-
-        let deleted = delete_timeline_layers(
-            &storage,
-            &sync_queue,
-            sync_id,
-            SyncData {
-                retries: current_retries,
-                data: LayersDeletion {
-                    deleted_layers: HashSet::new(),
-                    layers_to_delete: HashSet::from([
-                        local_timeline_path.join("a"),
-                        local_timeline_path.join("c"),
-                        local_timeline_path.join("something_different"),
-                    ]),
-                    deletion_registered: true,
-                },
-            },
-        )
-        .await;
-        assert!(deleted, "Should be able to delete timeline files");
-
-        assert_eq!(
-            local_storage
-                .list()
-                .await?
-                .into_iter()
-                .map(|remote_path| local_storage.local_path(&remote_path).unwrap())
-                .filter_map(|local_path| { Some(local_path.file_name()?.to_str()?.to_owned()) })
-                .sorted()
-                .collect::<Vec<_>>(),
-            vec!["b".to_string(), "d".to_string()],
-            "Expect to have only non-deleted files remotely"
-        );
-
-        Ok(())
-    }
-}

pageserver/src/storage_sync/download.rs

@@ -1,695 +0,0 @@
-//! Timeline synchronization logic to fetch the layer files from remote storage into pageserver's local directory.
-
-use std::{
-    collections::{HashMap, HashSet},
-    fmt::Debug,
-    mem,
-    path::Path,
-};
-
-use anyhow::Context;
-use futures::stream::{FuturesUnordered, StreamExt};
-use remote_storage::{DownloadError, GenericRemoteStorage};
-use tokio::{
-    fs,
-    io::{self, AsyncWriteExt},
-};
-use tracing::{debug, error, info, warn};
-
-use crate::{
-    config::PageServerConf,
-    storage_sync::{index::LayerFileMetadata, SyncTask},
-    TEMP_FILE_SUFFIX,
-};
-use utils::{
-    crashsafe::path_with_suffix_extension,
-    id::{TenantId, TenantTimelineId, TimelineId},
-};
-
-use super::{
-    index::{IndexPart, RemoteTimeline},
-    LayersDownload, SyncData, SyncQueue,
-};
-
-// We collect timelines remotely available for each tenant
-// in case we failed to gather all index parts (due to an error)
-// Poisoned variant is returned.
-// When data is received succesfully without errors Present variant is used.
-pub enum TenantIndexParts {
-    Poisoned {
-        present: HashMap<TimelineId, IndexPart>,
-        missing: HashSet<TimelineId>,
-    },
-    Present(HashMap<TimelineId, IndexPart>),
-}
-
-impl TenantIndexParts {
-    fn add_poisoned(&mut self, timeline_id: TimelineId) {
-        match self {
-            TenantIndexParts::Poisoned { missing, .. } => {
-                missing.insert(timeline_id);
-            }
-            TenantIndexParts::Present(present) => {
-                *self = TenantIndexParts::Poisoned {
-                    present: mem::take(present),
-                    missing: HashSet::from([timeline_id]),
-                }
-            }
-        }
-    }
-}
-
-impl Default for TenantIndexParts {
-    fn default() -> Self {
-        TenantIndexParts::Present(HashMap::default())
-    }
-}
-
-pub async fn download_index_parts(
-    conf: &'static PageServerConf,
-    storage: &GenericRemoteStorage,
-    keys: HashSet<TenantTimelineId>,
-) -> HashMap<TenantId, TenantIndexParts> {
-    let mut index_parts: HashMap<TenantId, TenantIndexParts> = HashMap::new();
-
-    let mut part_downloads = keys
-        .into_iter()
-        .map(|id| async move { (id, download_index_part(conf, storage, id).await) })
-        .collect::<FuturesUnordered<_>>();
-
-    while let Some((id, part_upload_result)) = part_downloads.next().await {
-        match part_upload_result {
-            Ok(index_part) => {
-                debug!("Successfully fetched index part for {id}");
-                match index_parts.entry(id.tenant_id).or_default() {
-                    TenantIndexParts::Poisoned { present, .. } => {
-                        present.insert(id.timeline_id, index_part);
-                    }
-                    TenantIndexParts::Present(parts) => {
-                        parts.insert(id.timeline_id, index_part);
-                    }
-                }
-            }
-            Err(download_error) => {
-                match download_error {
-                    DownloadError::NotFound => {
-                        // thats ok because it means that we didnt upload something we have locally for example
-                    }
-                    e => {
-                        let tenant_parts = index_parts.entry(id.tenant_id).or_default();
-                        tenant_parts.add_poisoned(id.timeline_id);
-                        error!(
-                            "Failed to fetch index part for {id}: {e} poisoning tenant index parts"
-                        );
-                    }
-                }
-            }
-        }
-    }
-
-    index_parts
-}
-
-/// Note: The function is rather expensive from s3 access point of view, it will execute ceil(N/1000) + N requests.
-/// At least one request to obtain a list of tenant timelines (more requests is there are more than 1000 timelines).
-/// And then will attempt to download all index files that belong to these timelines.
-pub async fn gather_tenant_timelines_index_parts(
-    conf: &'static PageServerConf,
-    storage: &GenericRemoteStorage,
-    tenant_id: TenantId,
-) -> anyhow::Result<HashMap<TimelineId, IndexPart>> {
-    let tenant_path = conf.timelines_path(&tenant_id);
-    let timeline_sync_ids = get_timeline_sync_ids(storage, &tenant_path, tenant_id)
-        .await
-        .with_context(|| format!("Failed to list timeline sync ids for tenat {tenant_id}"))?;
-
-    match download_index_parts(conf, storage, timeline_sync_ids)
-        .await
-        .remove(&tenant_id)
-        .ok_or_else(|| anyhow::anyhow!("Missing tenant index parts. This is a bug."))?
-    {
-        TenantIndexParts::Poisoned { missing, .. } => {
-            anyhow::bail!("Failed to download index parts for all timelines. Missing {missing:?}")
-        }
-        TenantIndexParts::Present(parts) => Ok(parts),
-    }
-}
-
-/// Retrieves index data from the remote storage for a given timeline.
-async fn download_index_part(
-    conf: &'static PageServerConf,
-    storage: &GenericRemoteStorage,
-    sync_id: TenantTimelineId,
-) -> Result<IndexPart, DownloadError> {
-    let index_part_path = conf
-        .metadata_path(sync_id.timeline_id, sync_id.tenant_id)
-        .with_file_name(IndexPart::FILE_NAME);
-    let mut index_part_download = storage
-        .download_storage_object(None, &index_part_path)
-        .await?;
-
-    let mut index_part_bytes = Vec::new();
-    io::copy(
-        &mut index_part_download.download_stream,
-        &mut index_part_bytes,
-    )
-    .await
-    .with_context(|| {
-        format!(
-            "Failed to download an index part into file '{}'",
-            index_part_path.display()
-        )
-    })
-    .map_err(DownloadError::Other)?;
-
-    let index_part: IndexPart = serde_json::from_slice(&index_part_bytes)
-        .with_context(|| {
-            format!(
-                "Failed to deserialize index part file into file '{}'",
-                index_part_path.display()
-            )
-        })
-        .map_err(DownloadError::Other)?;
-
-    let missing_files = index_part.missing_files();
-    if !missing_files.is_empty() {
-        warn!("Found missing layers in index part for timeline {sync_id}: {missing_files:?}");
-    }
-
-    Ok(index_part)
-}
-
-/// Timeline download result, with extra data, needed for downloading.
-#[derive(Debug)]
-pub(super) enum DownloadedTimeline {
-    /// Remote timeline data is either absent or corrupt, no download possible.
-    Abort,
-    /// Remote timeline data is found, its latest checkpoint's metadata contents (disk_consistent_lsn) is known.
-    /// Initial download failed due to some error, the download task is rescheduled for another retry.
-    FailedAndRescheduled,
-    /// Remote timeline data is found, its latest checkpoint's metadata contents (disk_consistent_lsn) is known.
-    /// Initial download successful.
-    Successful(SyncData<LayersDownload>),
-}
-
-/// Attempts to download all given timeline's layers.
-/// Timeline files that already exist locally are skipped during the download, but the local metadata file is
-/// updated in the end, if the remote one contains a newer disk_consistent_lsn.
-///
-/// On an error, bumps the retries count and updates the files to skip with successful downloads, rescheduling the task.
-pub(super) async fn download_timeline_layers<'a>(
-    conf: &'static PageServerConf,
-    storage: &'a GenericRemoteStorage,
-    sync_queue: &'a SyncQueue,
-    remote_timeline: Option<&'a RemoteTimeline>,
-    sync_id: TenantTimelineId,
-    mut download_data: SyncData<LayersDownload>,
-) -> DownloadedTimeline {
-    let remote_timeline = match remote_timeline {
-        Some(remote_timeline) => {
-            if !remote_timeline.awaits_download {
-                error!("Timeline with sync id {sync_id} is not awaiting download");
-                return DownloadedTimeline::Abort;
-            }
-            remote_timeline
-        }
-        None => {
-            error!("Timeline with sync id {sync_id} is not present in the remote index");
-            return DownloadedTimeline::Abort;
-        }
-    };
-
-    let download = &mut download_data.data;
-
-    let layers_to_download = remote_timeline
-        .stored_files()
-        .iter()
-        .filter_map(|(layer_path, metadata)| {
-            if !download.layers_to_skip.contains(layer_path) {
-                Some((layer_path.to_owned(), metadata.to_owned()))
-            } else {
-                None
-            }
-        })
-        .collect::<Vec<_>>();
-
-    debug!("Layers to download: {layers_to_download:?}");
-    info!("Downloading {} timeline layers", layers_to_download.len());
-
-    if layers_to_download.is_empty() {
-        info!("No layers to download after filtering, skipping");
-        return DownloadedTimeline::Successful(download_data);
-    }
-
-    let mut download_tasks = layers_to_download
-        .into_iter()
-        .map(|(layer_destination_path, metadata)| async move {
-
-            match layer_destination_path.metadata() {
-                Ok(m) if m.is_file() => {
-                    // the file exists from earlier round when we failed after renaming it as
-                    // layer_destination_path
-                    let verified = if let Some(expected) = metadata.file_size() {
-                        m.len() == expected
-                    } else {
-                        // behaviour before recording metadata was to accept any existing
-                        true
-                    };
-
-                    if verified {
-                        debug!(
-                            "Layer already exists locally, skipping download: {}",
-                            layer_destination_path.display()
-                        );
-                        return Ok((layer_destination_path, LayerFileMetadata::new(m.len())))
-                    } else {
-                        // no need to remove it, it will be overwritten by fs::rename
-                        // after successful download
-                        warn!("Downloaded layer exists already but layer file metadata mismatches: {}, metadata {:?}", layer_destination_path.display(), metadata);
-                    }
-                }
-                Ok(m) => {
-                    return Err(anyhow::anyhow!("Downloaded layer destination exists but is not a file: {m:?}, target needs to be removed/archived manually: {layer_destination_path:?}"));
-                }
-                Err(_) => {
-                    // behave as the file didn't exist
-                }
-            }
-
-            // Perform a rename inspired by durable_rename from file_utils.c.
-            // The sequence:
-            //     write(tmp)
-            //     fsync(tmp)
-            //     rename(tmp, new)
-            //     fsync(new)
-            //     fsync(parent)
-            // For more context about durable_rename check this email from postgres mailing list:
-            // https://www.postgresql.org/message-id/56583BDD.9060302@2ndquadrant.com
-            // If pageserver crashes the temp file will be deleted on startup and re-downloaded.
-            let temp_file_path =
-                path_with_suffix_extension(&layer_destination_path, TEMP_FILE_SUFFIX);
-
-            // TODO: this doesn't use the cached fd for some reason?
-            let mut destination_file =
-                fs::File::create(&temp_file_path).await.with_context(|| {
-                    format!(
-                        "Failed to create a destination file for layer '{}'",
-                        temp_file_path.display()
-                    )
-                })?;
-
-            let mut layer_download = storage.download_storage_object(None, &layer_destination_path)
-                .await
-                .with_context(|| {
-                    format!(
-                        "Failed to initiate the download the layer for {sync_id} into file '{}'",
-                        temp_file_path.display()
-                    )
-                })?;
-
-            let bytes_amount = io::copy(&mut layer_download.download_stream, &mut destination_file)
-                .await
-                .with_context(|| {
-                    format!(
-                        "Failed to download the layer for {sync_id} into file '{}'",
-                        temp_file_path.display()
-                    )
-                })?;
-
-            // Tokio doc here: https://docs.rs/tokio/1.17.0/tokio/fs/struct.File.html states that:
-            // A file will not be closed immediately when it goes out of scope if there are any IO operations
-            // that have not yet completed. To ensure that a file is closed immediately when it is dropped,
-            // you should call flush before dropping it.
-            //
-            // From the tokio code I see that it waits for pending operations to complete. There shouldn't be any because
-            // we assume that `destination_file` file is fully written. I.e there is no pending .write(...).await operations.
-            // But for additional safety let's check/wait for any pending operations.
-            destination_file.flush().await.with_context(|| {
-                format!(
-                    "failed to flush source file at {}",
-                    temp_file_path.display()
-                )
-            })?;
-
-            match metadata.file_size() {
-                Some(expected) if expected != bytes_amount => {
-                    anyhow::bail!(
-                        "According to layer file metadata should had downloaded {expected} bytes but downloaded {bytes_amount} bytes into file '{}'",
-                        temp_file_path.display()
-                    );
-                },
-                Some(_) | None => {
-                    // matches, or upgrading from an earlier IndexPart version
-                }
-            }
-
-            // not using sync_data because it can lose file size update
-            destination_file.sync_all().await.with_context(|| {
-                format!(
-                    "failed to fsync source file at {}",
-                    temp_file_path.display()
-                )
-            })?;
-            drop(destination_file);
-
-            fail::fail_point!("remote-storage-download-pre-rename", |_| {
-                anyhow::bail!("remote-storage-download-pre-rename failpoint triggered")
-            });
-
-            fs::rename(&temp_file_path, &layer_destination_path).await?;
-
-            fsync_path(&layer_destination_path).await.with_context(|| {
-                format!(
-                    "Cannot fsync layer destination path {}",
-                    layer_destination_path.display(),
-                )
-            })?;
-
-            Ok::<_, anyhow::Error>((layer_destination_path, LayerFileMetadata::new(bytes_amount)))
-        })
-        .collect::<FuturesUnordered<_>>();
-
-    let mut errors_happened = false;
-    // keep files we've downloaded to remove them from layers_to_skip if directory fsync fails
-    let mut undo = HashSet::new();
-    while let Some(download_result) = download_tasks.next().await {
-        match download_result {
-            Ok((downloaded_path, metadata)) => {
-                undo.insert(downloaded_path.clone());
-                download.layers_to_skip.insert(downloaded_path.clone());
-                // what if the key existed already? ignore, because then we would had
-                // downloaded a partial file, and had to retry
-                download.gathered_metadata.insert(downloaded_path, metadata);
-            }
-            Err(e) => {
-                errors_happened = true;
-                error!("Failed to download a layer for timeline {sync_id}: {e:?}");
-            }
-        }
-    }
-
-    // fsync timeline directory which is a parent directory for downloaded files
-    let TenantTimelineId {
-        tenant_id,
-        timeline_id,
-    } = &sync_id;
-    let timeline_dir = conf.timeline_path(timeline_id, tenant_id);
-    if let Err(e) = fsync_path(&timeline_dir).await {
-        error!(
-            "Cannot fsync parent directory {} error {}",
-            timeline_dir.display(),
-            e
-        );
-        for item in undo {
-            download.layers_to_skip.remove(&item);
-            // intentionally don't clear the gathered_metadata because it exists for fsync_path
-            // failure on parent directory
-        }
-        errors_happened = true;
-    }
-
-    if errors_happened {
-        debug!("Reenqueuing failed download task for timeline {sync_id}");
-        download_data.retries += 1;
-        sync_queue.push(sync_id, SyncTask::Download(download_data));
-        DownloadedTimeline::FailedAndRescheduled
-    } else {
-        info!("Successfully downloaded all layers");
-        DownloadedTimeline::Successful(download_data)
-    }
-}
-
-async fn get_timeline_sync_ids(
-    storage: &GenericRemoteStorage,
-    tenant_path: &Path,
-    tenant_id: TenantId,
-) -> anyhow::Result<HashSet<TenantTimelineId>> {
-    let tenant_storage_path = storage.remote_object_id(tenant_path).with_context(|| {
-        format!(
-            "Failed to get tenant storage path for local path '{}'",
-            tenant_path.display()
-        )
-    })?;
-
-    let timelines = storage
-        .list_prefixes(Some(&tenant_storage_path))
-        .await
-        .with_context(|| {
-            format!(
-                "Failed to list tenant storage path {tenant_storage_path:?} to get remote timelines to download"
-            )
-        })?;
-
-    if timelines.is_empty() {
-        anyhow::bail!("no timelines found on the remote storage")
-    }
-
-    let mut sync_ids = HashSet::new();
-
-    for timeline_remote_storage_key in timelines {
-        let object_name = timeline_remote_storage_key.object_name().ok_or_else(|| {
-            anyhow::anyhow!("failed to get timeline id for remote tenant {tenant_id}")
-        })?;
-
-        let timeline_id: TimelineId = object_name.parse().with_context(|| {
-            format!("failed to parse object name into timeline id '{object_name}'")
-        })?;
-
-        sync_ids.insert(TenantTimelineId {
-            tenant_id,
-            timeline_id,
-        });
-    }
-
-    Ok(sync_ids)
-}
-
-async fn fsync_path(path: impl AsRef<Path>) -> Result<(), io::Error> {
-    fs::File::open(path).await?.sync_all().await
-}
-
-#[cfg(test)]
-mod tests {
-    use std::{
-        collections::{BTreeSet, HashSet},
-        num::NonZeroUsize,
-        path::PathBuf,
-    };
-
-    use remote_storage::{LocalFs, RemoteStorage};
-    use tempfile::tempdir;
-    use utils::lsn::Lsn;
-
-    use crate::{
-        storage_sync::{
-            index::RelativePath,
-            test_utils::{create_local_timeline, dummy_metadata},
-        },
-        tenant::harness::{TenantHarness, TIMELINE_ID},
-    };
-
-    use super::*;
-
-    #[tokio::test]
-    async fn download_timeline() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("download_timeline")?;
-        let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
-
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-        let layer_files = ["a", "b", "layer_to_skip", "layer_to_keep_locally"];
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_owned(),
-            harness.conf.workdir.clone(),
-        )?);
-        let local_storage = storage.as_local().unwrap();
-        let current_retries = 3;
-        let metadata = dummy_metadata(Lsn(0x30));
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let timeline_upload =
-            create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?;
-
-        for local_path in timeline_upload.layers_to_upload.keys() {
-            let remote_path =
-                local_storage.resolve_in_storage(&storage.remote_object_id(local_path)?)?;
-            let remote_parent_dir = remote_path.parent().unwrap();
-            if !remote_parent_dir.exists() {
-                fs::create_dir_all(&remote_parent_dir).await?;
-            }
-            fs::copy(&local_path, &remote_path).await?;
-        }
-        let mut read_dir = fs::read_dir(&local_timeline_path).await?;
-        while let Some(dir_entry) = read_dir.next_entry().await? {
-            if dir_entry.file_name().to_str() == Some("layer_to_keep_locally") {
-                continue;
-            } else {
-                fs::remove_file(dir_entry.path()).await?;
-            }
-        }
-
-        let mut remote_timeline = RemoteTimeline::new(metadata.clone());
-        remote_timeline.awaits_download = true;
-        remote_timeline.add_timeline_layers(layer_files.iter().map(|layer| {
-            let layer_path = local_timeline_path.join(layer);
-
-            // this could had also been LayerFileMetadata::default(), but since in this test we
-            // don't do the merge operation done by storage_sync::download_timeline_data, it would
-            // not be merged back to timeline.
-            let metadata_from_upload = timeline_upload
-                .layers_to_upload
-                .get(&layer_path)
-                .expect("layer must exist in previously uploaded paths")
-                .to_owned();
-            (layer_path, metadata_from_upload)
-        }));
-
-        let download_data = match download_timeline_layers(
-            harness.conf,
-            &storage,
-            &sync_queue,
-            Some(&remote_timeline),
-            sync_id,
-            SyncData::new(
-                current_retries,
-                LayersDownload::from_skipped_layers(HashSet::from([
-                    local_timeline_path.join("layer_to_skip")
-                ])),
-            ),
-        )
-        .await
-        {
-            DownloadedTimeline::Successful(data) => data,
-            wrong_result => {
-                panic!("Expected a successful download for timeline, but got: {wrong_result:?}")
-            }
-        };
-
-        assert_eq!(
-            current_retries, download_data.retries,
-            "On successful download, retries are not expected to change"
-        );
-        assert_eq!(
-            download_data
-                .data
-                .layers_to_skip
-                .into_iter()
-                .collect::<BTreeSet<_>>(),
-            layer_files
-                .iter()
-                .map(|layer| local_timeline_path.join(layer))
-                .collect(),
-            "On successful download, layers to skip should contain all downloaded files and present layers that were skipped"
-        );
-
-        let mut downloaded_files = BTreeSet::new();
-        let mut read_dir = fs::read_dir(&local_timeline_path).await?;
-        while let Some(dir_entry) = read_dir.next_entry().await? {
-            downloaded_files.insert(dir_entry.path());
-        }
-
-        assert_eq!(
-            downloaded_files,
-            layer_files
-                .iter()
-                .filter(|layer| layer != &&"layer_to_skip")
-                .map(|layer| local_timeline_path.join(layer))
-                .collect(),
-            "On successful download, all layers that were not skipped, should be downloaded"
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn download_timeline_negatives() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("download_timeline_negatives")?;
-        let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_owned(),
-            harness.conf.workdir.clone(),
-        )?);
-
-        let empty_remote_timeline_download = download_timeline_layers(
-            harness.conf,
-            &storage,
-            &sync_queue,
-            None,
-            sync_id,
-            SyncData::new(0, LayersDownload::from_skipped_layers(HashSet::new())),
-        )
-        .await;
-        assert!(
-            matches!(empty_remote_timeline_download, DownloadedTimeline::Abort),
-            "Should not allow downloading for empty remote timeline"
-        );
-
-        let not_expecting_download_remote_timeline = RemoteTimeline::new(dummy_metadata(Lsn(5)));
-        assert!(
-            !not_expecting_download_remote_timeline.awaits_download,
-            "Should not expect download for the timeline"
-        );
-        let already_downloading_remote_timeline_download = download_timeline_layers(
-            harness.conf,
-            &storage,
-            &sync_queue,
-            Some(&not_expecting_download_remote_timeline),
-            sync_id,
-            SyncData::new(0, LayersDownload::from_skipped_layers(HashSet::new())),
-        )
-        .await;
-        assert!(
-            matches!(
-                already_downloading_remote_timeline_download,
-                DownloadedTimeline::Abort,
-            ),
-            "Should not allow downloading for remote timeline that does not expect it"
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_download_index_part() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("test_download_index_part")?;
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_owned(),
-            harness.conf.workdir.clone(),
-        )?);
-        let local_storage = storage.as_local().unwrap();
-        let metadata = dummy_metadata(Lsn(0x30));
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-
-        let index_part = IndexPart::new(
-            HashSet::from([
-                RelativePath::new(&local_timeline_path, local_timeline_path.join("one"))?,
-                RelativePath::new(&local_timeline_path, local_timeline_path.join("two"))?,
-            ]),
-            HashSet::from([RelativePath::new(
-                &local_timeline_path,
-                local_timeline_path.join("three"),
-            )?]),
-            metadata.disk_consistent_lsn(),
-            metadata.to_bytes()?,
-        );
-
-        let local_index_part_path = harness
-            .conf
-            .metadata_path(sync_id.timeline_id, sync_id.tenant_id)
-            .with_file_name(IndexPart::FILE_NAME);
-        let index_part_remote_id = local_storage.remote_object_id(&local_index_part_path)?;
-        let index_part_local_path = PathBuf::from(index_part_remote_id.to_string());
-        fs::create_dir_all(index_part_local_path.parent().unwrap()).await?;
-        fs::write(&index_part_local_path, serde_json::to_vec(&index_part)?).await?;
-
-        let downloaded_index_part = download_index_part(harness.conf, &storage, sync_id).await?;
-
-        assert_eq!(
-            downloaded_index_part, index_part,
-            "Downloaded index part should be the same as the one in storage"
-        );
-
-        Ok(())
-    }
-}

pageserver/src/storage_sync/index.rs

@@ -1,712 +0,0 @@
-//! In-memory index to track the tenant files on the remote storage.
-//! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about
-//! remote timeline layers and its metadata.
-
-use std::ops::{Deref, DerefMut};
-use std::{
-    collections::{HashMap, HashSet},
-    path::{Path, PathBuf},
-    sync::Arc,
-};
-
-use anyhow::{anyhow, Context, Ok};
-use serde::{Deserialize, Serialize};
-use serde_with::{serde_as, DisplayFromStr};
-use tokio::sync::RwLock;
-use tracing::log::warn;
-
-use crate::{config::PageServerConf, tenant::metadata::TimelineMetadata};
-use utils::{
-    id::{TenantId, TenantTimelineId, TimelineId},
-    lsn::Lsn,
-};
-
-use super::download::TenantIndexParts;
-
-/// A part of the filesystem path, that needs a root to become a path again.
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
-#[serde(transparent)]
-pub struct RelativePath(String);
-
-impl RelativePath {
-    /// Attempts to strip off the base from path, producing a relative path or an error.
-    pub fn new<P: AsRef<Path>>(base: &Path, path: P) -> anyhow::Result<Self> {
-        let path = path.as_ref();
-        let relative = path.strip_prefix(base).with_context(|| {
-            format!(
-                "path '{}' is not relative to base '{}'",
-                path.display(),
-                base.display()
-            )
-        })?;
-        Ok(RelativePath(relative.to_string_lossy().to_string()))
-    }
-
-    /// Joins the relative path with the base path.
-    fn as_path(&self, base: &Path) -> PathBuf {
-        base.join(&self.0)
-    }
-}
-
-#[derive(Debug, Clone, Default)]
-pub struct TenantEntry(HashMap<TimelineId, RemoteTimeline>);
-
-impl TenantEntry {
-    pub fn has_in_progress_downloads(&self) -> bool {
-        self.values()
-            .any(|remote_timeline| remote_timeline.awaits_download)
-    }
-}
-
-impl Deref for TenantEntry {
-    type Target = HashMap<TimelineId, RemoteTimeline>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl DerefMut for TenantEntry {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
-    }
-}
-
-impl From<HashMap<TimelineId, RemoteTimeline>> for TenantEntry {
-    fn from(inner: HashMap<TimelineId, RemoteTimeline>) -> Self {
-        Self(inner)
-    }
-}
-
-/// An index to track tenant files that exist on the remote storage.
-#[derive(Debug, Clone, Default)]
-pub struct RemoteTimelineIndex {
-    entries: HashMap<TenantId, TenantEntry>,
-}
-
-/// A wrapper to synchronize the access to the index, should be created and used before dealing with any [`RemoteTimelineIndex`].
-#[derive(Default)]
-pub struct RemoteIndex(Arc<RwLock<RemoteTimelineIndex>>);
-
-impl RemoteIndex {
-    pub fn from_parts(
-        conf: &'static PageServerConf,
-        index_parts: HashMap<TenantId, TenantIndexParts>,
-    ) -> anyhow::Result<Self> {
-        let mut entries: HashMap<TenantId, TenantEntry> = HashMap::new();
-
-        for (tenant_id, index_parts) in index_parts {
-            match index_parts {
-                // TODO: should we schedule a retry so it can be recovered? otherwise we can revive it only through detach/attach or pageserver restart
-                TenantIndexParts::Poisoned { missing, ..} => warn!("skipping tenant_id set up for remote index because the index download has failed for timeline(s): {missing:?}"),
-                TenantIndexParts::Present(timelines) => {
-                    for (timeline_id, index_part) in timelines {
-                        let timeline_path = conf.timeline_path(&timeline_id, &tenant_id);
-                        let remote_timeline =
-                            RemoteTimeline::from_index_part(&timeline_path, index_part)
-                                .context("Failed to restore remote timeline data from index part")?;
-
-                        entries
-                            .entry(tenant_id)
-                            .or_default()
-                            .insert(timeline_id, remote_timeline);
-                    }
-                },
-            }
-        }
-
-        Ok(Self(Arc::new(RwLock::new(RemoteTimelineIndex { entries }))))
-    }
-
-    pub async fn read(&self) -> tokio::sync::RwLockReadGuard<'_, RemoteTimelineIndex> {
-        self.0.read().await
-    }
-
-    pub async fn write(&self) -> tokio::sync::RwLockWriteGuard<'_, RemoteTimelineIndex> {
-        self.0.write().await
-    }
-}
-
-impl Clone for RemoteIndex {
-    fn clone(&self) -> Self {
-        Self(Arc::clone(&self.0))
-    }
-}
-
-impl RemoteTimelineIndex {
-    pub fn timeline_entry(
-        &self,
-        TenantTimelineId {
-            tenant_id,
-            timeline_id,
-        }: &TenantTimelineId,
-    ) -> Option<&RemoteTimeline> {
-        self.entries.get(tenant_id)?.get(timeline_id)
-    }
-
-    pub fn timeline_entry_mut(
-        &mut self,
-        TenantTimelineId {
-            tenant_id,
-            timeline_id,
-        }: &TenantTimelineId,
-    ) -> Option<&mut RemoteTimeline> {
-        self.entries.get_mut(tenant_id)?.get_mut(timeline_id)
-    }
-
-    pub fn add_timeline_entry(
-        &mut self,
-        TenantTimelineId {
-            tenant_id,
-            timeline_id,
-        }: TenantTimelineId,
-        entry: RemoteTimeline,
-    ) {
-        self.entries
-            .entry(tenant_id)
-            .or_default()
-            .insert(timeline_id, entry);
-    }
-
-    pub fn remove_timeline_entry(
-        &mut self,
-        TenantTimelineId {
-            tenant_id,
-            timeline_id,
-        }: TenantTimelineId,
-    ) -> Option<RemoteTimeline> {
-        self.entries
-            .entry(tenant_id)
-            .or_default()
-            .remove(&timeline_id)
-    }
-
-    pub fn tenant_entry(&self, tenant_id: &TenantId) -> Option<&TenantEntry> {
-        self.entries.get(tenant_id)
-    }
-
-    pub fn tenant_entry_mut(&mut self, tenant_id: &TenantId) -> Option<&mut TenantEntry> {
-        self.entries.get_mut(tenant_id)
-    }
-
-    pub fn add_tenant_entry(&mut self, tenant_id: TenantId) -> &mut TenantEntry {
-        self.entries.entry(tenant_id).or_default()
-    }
-
-    pub fn remove_tenant_entry(&mut self, tenant_id: &TenantId) -> Option<TenantEntry> {
-        self.entries.remove(tenant_id)
-    }
-
-    pub fn set_awaits_download(
-        &mut self,
-        id: &TenantTimelineId,
-        awaits_download: bool,
-    ) -> anyhow::Result<()> {
-        self.timeline_entry_mut(id)
-            .ok_or_else(|| anyhow!("unknown timeline sync {id}"))?
-            .awaits_download = awaits_download;
-        Ok(())
-    }
-}
-
-/// Restored index part data about the timeline, stored in the remote index.
-#[derive(Debug, Clone)]
-pub struct RemoteTimeline {
-    timeline_layers: HashMap<PathBuf, LayerFileMetadata>,
-    missing_layers: HashMap<PathBuf, LayerFileMetadata>,
-
-    pub metadata: TimelineMetadata,
-    pub awaits_download: bool,
-}
-
-impl RemoteTimeline {
-    pub fn new(metadata: TimelineMetadata) -> Self {
-        Self {
-            timeline_layers: HashMap::default(),
-            missing_layers: HashMap::default(),
-            metadata,
-            awaits_download: false,
-        }
-    }
-
-    pub fn add_timeline_layers(
-        &mut self,
-        new_layers: impl IntoIterator<Item = (PathBuf, LayerFileMetadata)>,
-    ) {
-        self.timeline_layers.extend(new_layers);
-    }
-
-    pub fn add_upload_failures(
-        &mut self,
-        upload_failures: impl IntoIterator<Item = (PathBuf, LayerFileMetadata)>,
-    ) {
-        self.missing_layers.extend(upload_failures);
-    }
-
-    pub fn remove_layers(&mut self, layers_to_remove: &HashSet<PathBuf>) {
-        self.timeline_layers
-            .retain(|layer, _| !layers_to_remove.contains(layer));
-        self.missing_layers
-            .retain(|layer, _| !layers_to_remove.contains(layer));
-    }
-
-    /// Lists all layer files in the given remote timeline. Omits the metadata file.
-    pub fn stored_files(&self) -> &HashMap<PathBuf, LayerFileMetadata> {
-        &self.timeline_layers
-    }
-
-    /// Combines metadata gathered or verified during downloading needed layer files to metadata on
-    /// the [`RemoteIndex`], so it can be uploaded later.
-    pub fn merge_metadata_from_downloaded(
-        &mut self,
-        downloaded: &HashMap<PathBuf, LayerFileMetadata>,
-    ) {
-        downloaded.iter().for_each(|(path, metadata)| {
-            if let Some(upgraded) = self.timeline_layers.get_mut(path) {
-                upgraded.merge(metadata);
-            }
-        });
-    }
-
-    pub fn from_index_part(timeline_path: &Path, index_part: IndexPart) -> anyhow::Result<Self> {
-        let metadata = TimelineMetadata::from_bytes(&index_part.metadata_bytes)?;
-        let default_metadata = &IndexLayerMetadata::default();
-
-        let find_metadata = |key: &RelativePath| -> LayerFileMetadata {
-            index_part
-                .layer_metadata
-                .get(key)
-                .unwrap_or(default_metadata)
-                .into()
-        };
-
-        Ok(Self {
-            timeline_layers: index_part
-                .timeline_layers
-                .iter()
-                .map(|layer_path| (layer_path.as_path(timeline_path), find_metadata(layer_path)))
-                .collect(),
-            missing_layers: index_part
-                .missing_layers
-                .iter()
-                .map(|layer_path| (layer_path.as_path(timeline_path), find_metadata(layer_path)))
-                .collect(),
-            metadata,
-            awaits_download: false,
-        })
-    }
-}
-
-/// Metadata gathered for each of the layer files.
-///
-/// Fields have to be `Option`s because remote [`IndexPart`]'s can be from different version, which
-/// might have less or more metadata depending if upgrading or rolling back an upgrade.
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
-#[cfg_attr(test, derive(Default))]
-pub struct LayerFileMetadata {
-    file_size: Option<u64>,
-}
-
-impl From<&'_ IndexLayerMetadata> for LayerFileMetadata {
-    fn from(other: &IndexLayerMetadata) -> Self {
-        LayerFileMetadata {
-            file_size: other.file_size,
-        }
-    }
-}
-
-impl LayerFileMetadata {
-    pub fn new(file_size: u64) -> Self {
-        LayerFileMetadata {
-            file_size: Some(file_size),
-        }
-    }
-
-    pub fn file_size(&self) -> Option<u64> {
-        self.file_size
-    }
-
-    /// Metadata has holes due to version upgrades. This method is called to upgrade self with the
-    /// other value.
-    ///
-    /// This is called on the possibly outdated version.
-    pub fn merge(&mut self, other: &Self) {
-        self.file_size = other.file_size.or(self.file_size);
-    }
-}
-
-/// Part of the remote index, corresponding to a certain timeline.
-/// Contains the data about all files in the timeline, present remotely and its metadata.
-///
-/// This type needs to be backwards and forwards compatible. When changing the fields,
-/// remember to add a test case for the changed version.
-#[serde_as]
-#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
-pub struct IndexPart {
-    /// Debugging aid describing the version of this type.
-    #[serde(default)]
-    version: usize,
-
-    /// Each of the layers present on remote storage.
-    ///
-    /// Additional metadata can might exist in `layer_metadata`.
-    timeline_layers: HashSet<RelativePath>,
-
-    /// Currently is not really used in pageserver,
-    /// present to manually keep track of the layer files that pageserver might never retrieve.
-    ///
-    /// Such "holes" might appear if any upload task was evicted on an error threshold:
-    /// the this layer will only be rescheduled for upload on pageserver restart.
-    missing_layers: HashSet<RelativePath>,
-
-    /// Per layer file metadata, which can be present for a present or missing layer file.
-    ///
-    /// Older versions of `IndexPart` will not have this property or have only a part of metadata
-    /// that latest version stores.
-    #[serde(default)]
-    layer_metadata: HashMap<RelativePath, IndexLayerMetadata>,
-
-    #[serde_as(as = "DisplayFromStr")]
-    disk_consistent_lsn: Lsn,
-    metadata_bytes: Vec<u8>,
-}
-
-impl IndexPart {
-    /// When adding or modifying any parts of `IndexPart`, increment the version so that it can be
-    /// used to understand later versions.
-    ///
-    /// Version is currently informative only.
-    const LATEST_VERSION: usize = 1;
-    pub const FILE_NAME: &'static str = "index_part.json";
-
-    #[cfg(test)]
-    pub fn new(
-        timeline_layers: HashSet<RelativePath>,
-        missing_layers: HashSet<RelativePath>,
-        disk_consistent_lsn: Lsn,
-        metadata_bytes: Vec<u8>,
-    ) -> Self {
-        Self {
-            version: Self::LATEST_VERSION,
-            timeline_layers,
-            missing_layers,
-            layer_metadata: HashMap::default(),
-            disk_consistent_lsn,
-            metadata_bytes,
-        }
-    }
-
-    pub fn missing_files(&self) -> &HashSet<RelativePath> {
-        &self.missing_layers
-    }
-
-    pub fn from_remote_timeline(
-        timeline_path: &Path,
-        remote_timeline: RemoteTimeline,
-    ) -> anyhow::Result<Self> {
-        let metadata_bytes = remote_timeline.metadata.to_bytes()?;
-
-        let mut layer_metadata = HashMap::new();
-
-        let mut missing_layers = HashSet::new();
-
-        separate_paths_and_metadata(
-            timeline_path,
-            &remote_timeline.missing_layers,
-            &mut missing_layers,
-            &mut layer_metadata,
-        )
-        .context("Failed to convert missing layers' paths to relative ones")?;
-
-        let mut timeline_layers = HashSet::new();
-
-        separate_paths_and_metadata(
-            timeline_path,
-            &remote_timeline.timeline_layers,
-            &mut timeline_layers,
-            &mut layer_metadata,
-        )
-        .context("Failed to convert timeline layers' paths to relative ones")?;
-
-        Ok(Self {
-            version: Self::LATEST_VERSION,
-            timeline_layers,
-            missing_layers,
-            layer_metadata,
-            disk_consistent_lsn: remote_timeline.metadata.disk_consistent_lsn(),
-            metadata_bytes,
-        })
-    }
-}
-
-/// Serialized form of [`LayerFileMetadata`].
-#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default)]
-pub struct IndexLayerMetadata {
-    file_size: Option<u64>,
-}
-
-impl From<&'_ LayerFileMetadata> for IndexLayerMetadata {
-    fn from(other: &'_ LayerFileMetadata) -> Self {
-        IndexLayerMetadata {
-            file_size: other.file_size,
-        }
-    }
-}
-
-fn separate_paths_and_metadata(
-    timeline_path: &Path,
-    input: &HashMap<PathBuf, LayerFileMetadata>,
-    output: &mut HashSet<RelativePath>,
-    layer_metadata: &mut HashMap<RelativePath, IndexLayerMetadata>,
-) -> anyhow::Result<()> {
-    for (path, metadata) in input {
-        let rel_path = RelativePath::new(timeline_path, path)?;
-        let metadata = IndexLayerMetadata::from(metadata);
-
-        layer_metadata.insert(rel_path.clone(), metadata);
-        output.insert(rel_path);
-    }
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use std::collections::BTreeSet;
-
-    use super::*;
-    use crate::tenant::harness::{TenantHarness, TIMELINE_ID};
-    use crate::DEFAULT_PG_VERSION;
-
-    #[test]
-    fn index_part_conversion() {
-        let harness = TenantHarness::create("index_part_conversion").unwrap();
-        let timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let metadata = TimelineMetadata::new(
-            Lsn(5).align(),
-            Some(Lsn(4)),
-            None,
-            Lsn(3),
-            Lsn(2),
-            Lsn(1),
-            DEFAULT_PG_VERSION,
-        );
-        let remote_timeline = RemoteTimeline {
-            timeline_layers: HashMap::from([
-                (timeline_path.join("layer_1"), LayerFileMetadata::new(1)),
-                (timeline_path.join("layer_2"), LayerFileMetadata::new(2)),
-            ]),
-            missing_layers: HashMap::from([
-                (timeline_path.join("missing_1"), LayerFileMetadata::new(3)),
-                (timeline_path.join("missing_2"), LayerFileMetadata::new(4)),
-            ]),
-            metadata: metadata.clone(),
-            awaits_download: false,
-        };
-
-        let index_part = IndexPart::from_remote_timeline(&timeline_path, remote_timeline.clone())
-            .expect("Correct remote timeline should be convertible to index part");
-
-        assert_eq!(
-            index_part.timeline_layers.iter().collect::<BTreeSet<_>>(),
-            BTreeSet::from([
-                &RelativePath("layer_1".to_string()),
-                &RelativePath("layer_2".to_string())
-            ]),
-            "Index part should have all remote timeline layers after the conversion"
-        );
-        assert_eq!(
-            index_part.missing_layers.iter().collect::<BTreeSet<_>>(),
-            BTreeSet::from([
-                &RelativePath("missing_1".to_string()),
-                &RelativePath("missing_2".to_string())
-            ]),
-            "Index part should have all missing remote timeline layers after the conversion"
-        );
-        assert_eq!(
-            index_part.disk_consistent_lsn,
-            metadata.disk_consistent_lsn(),
-            "Index part should have disk consistent lsn from the timeline"
-        );
-        assert_eq!(
-            index_part.metadata_bytes,
-            metadata
-                .to_bytes()
-                .expect("Failed to serialize correct metadata into bytes"),
-            "Index part should have all missing remote timeline layers after the conversion"
-        );
-
-        let restored_timeline = RemoteTimeline::from_index_part(&timeline_path, index_part)
-            .expect("Correct index part should be convertible to remote timeline");
-
-        let original_metadata = &remote_timeline.metadata;
-        let restored_metadata = &restored_timeline.metadata;
-        // we have to compare the metadata this way, since its header is different after creation and restoration,
-        // but that is now consireded ok.
-        assert_eq!(
-            original_metadata.disk_consistent_lsn(),
-            restored_metadata.disk_consistent_lsn(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-        assert_eq!(
-            original_metadata.prev_record_lsn(),
-            restored_metadata.prev_record_lsn(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-        assert_eq!(
-            original_metadata.ancestor_timeline(),
-            restored_metadata.ancestor_timeline(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-        assert_eq!(
-            original_metadata.ancestor_lsn(),
-            restored_metadata.ancestor_lsn(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-        assert_eq!(
-            original_metadata.latest_gc_cutoff_lsn(),
-            restored_metadata.latest_gc_cutoff_lsn(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-        assert_eq!(
-            original_metadata.initdb_lsn(),
-            restored_metadata.initdb_lsn(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-
-        assert_eq!(
-            remote_timeline.awaits_download, restored_timeline.awaits_download,
-            "remote timeline -> index part -> remote timeline conversion should not loose download flag"
-        );
-
-        assert_eq!(
-            remote_timeline
-                .timeline_layers
-                .into_iter()
-                .collect::<BTreeSet<_>>(),
-            restored_timeline
-                .timeline_layers
-                .into_iter()
-                .collect::<BTreeSet<_>>(),
-            "remote timeline -> index part -> remote timeline conversion should not loose layer data"
-        );
-        assert_eq!(
-            remote_timeline
-                .missing_layers
-                .into_iter()
-                .collect::<BTreeSet<_>>(),
-            restored_timeline
-                .missing_layers
-                .into_iter()
-                .collect::<BTreeSet<_>>(),
-            "remote timeline -> index part -> remote timeline conversion should not loose missing file data"
-        );
-    }
-
-    #[test]
-    fn index_part_conversion_negatives() {
-        let harness = TenantHarness::create("index_part_conversion_negatives").unwrap();
-        let timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let metadata = TimelineMetadata::new(
-            Lsn(5).align(),
-            Some(Lsn(4)),
-            None,
-            Lsn(3),
-            Lsn(2),
-            Lsn(1),
-            DEFAULT_PG_VERSION,
-        );
-
-        let conversion_result = IndexPart::from_remote_timeline(
-            &timeline_path,
-            RemoteTimeline {
-                timeline_layers: HashMap::from([
-                    (PathBuf::from("bad_path"), LayerFileMetadata::new(1)),
-                    (timeline_path.join("layer_2"), LayerFileMetadata::new(2)),
-                ]),
-                missing_layers: HashMap::from([
-                    (timeline_path.join("missing_1"), LayerFileMetadata::new(3)),
-                    (timeline_path.join("missing_2"), LayerFileMetadata::new(4)),
-                ]),
-                metadata: metadata.clone(),
-                awaits_download: false,
-            },
-        );
-        assert!(conversion_result.is_err(), "Should not be able to convert metadata with layer paths that are not in the timeline directory");
-
-        let conversion_result = IndexPart::from_remote_timeline(
-            &timeline_path,
-            RemoteTimeline {
-                timeline_layers: HashMap::from([
-                    (timeline_path.join("layer_1"), LayerFileMetadata::new(1)),
-                    (timeline_path.join("layer_2"), LayerFileMetadata::new(2)),
-                ]),
-                missing_layers: HashMap::from([
-                    (PathBuf::from("bad_path"), LayerFileMetadata::new(3)),
-                    (timeline_path.join("missing_2"), LayerFileMetadata::new(4)),
-                ]),
-                metadata,
-                awaits_download: false,
-            },
-        );
-        assert!(conversion_result.is_err(), "Should not be able to convert metadata with missing layer paths that are not in the timeline directory");
-    }
-
-    #[test]
-    fn v0_indexpart_is_parsed() {
-        let example = r#"{
-            "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
-            "missing_layers":["not_a_real_layer_but_adding_coverage"],
-            "disk_consistent_lsn":"0/16960E8",
-            "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-        }"#;
-
-        let expected = IndexPart {
-            version: 0,
-            timeline_layers: [RelativePath("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".to_owned())].into_iter().collect(),
-            missing_layers: [RelativePath("not_a_real_layer_but_adding_coverage".to_owned())].into_iter().collect(),
-            layer_metadata: HashMap::default(),
-            disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
-            metadata_bytes: [113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0].to_vec(),
-        };
-
-        let part = serde_json::from_str::<IndexPart>(example).unwrap();
-        assert_eq!(part, expected);
-    }
-
-    #[test]
-    fn v1_indexpart_is_parsed() {
-        let example = r#"{
-            "version":1,
-            "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
-            "missing_layers":["not_a_real_layer_but_adding_coverage"],
-            "layer_metadata":{
-                "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
-                "not_a_real_layer_but_adding_coverage": { "file_size": 9007199254741001 }
-            },
-            "disk_consistent_lsn":"0/16960E8",
-            "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-        }"#;
-
-        let expected = IndexPart {
-            // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
-            version: 1,
-            timeline_layers: [RelativePath("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".to_owned())].into_iter().collect(),
-            missing_layers: [RelativePath("not_a_real_layer_but_adding_coverage".to_owned())].into_iter().collect(),
-            layer_metadata: HashMap::from([
-                (RelativePath("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".to_owned()), IndexLayerMetadata {
-                    file_size: Some(25600000),
-                }),
-                (RelativePath("not_a_real_layer_but_adding_coverage".to_owned()), IndexLayerMetadata {
-                    // serde_json should always parse this but this might be a double with jq for
-                    // example.
-                    file_size: Some(9007199254741001),
-                })
-            ]),
-            disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
-            metadata_bytes: [113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0].to_vec(),
-        };
-
-        let part = serde_json::from_str::<IndexPart>(example).unwrap();
-        assert_eq!(part, expected);
-    }
-}

pageserver/src/storage_sync/upload.rs

@@ -1,479 +0,0 @@
-//! Timeline synchronization logic to compress and upload to the remote storage all new timeline files from the checkpoints.
-
-use std::{fmt::Debug, path::PathBuf};
-
-use anyhow::Context;
-use futures::stream::{FuturesUnordered, StreamExt};
-use remote_storage::GenericRemoteStorage;
-use tokio::fs;
-use tracing::{debug, error, info, warn};
-
-use utils::id::TenantTimelineId;
-
-use super::{
-    index::{IndexPart, RemoteTimeline},
-    LayersUpload, SyncData, SyncQueue,
-};
-use crate::metrics::NO_LAYERS_UPLOAD;
-use crate::{config::PageServerConf, storage_sync::SyncTask};
-
-/// Serializes and uploads the given index part data to the remote storage.
-pub(super) async fn upload_index_part(
-    conf: &'static PageServerConf,
-    storage: &GenericRemoteStorage,
-    sync_id: TenantTimelineId,
-    index_part: IndexPart,
-) -> anyhow::Result<()> {
-    let index_part_bytes = serde_json::to_vec(&index_part)
-        .context("Failed to serialize index part file into bytes")?;
-    let index_part_size = index_part_bytes.len();
-    let index_part_bytes = tokio::io::BufReader::new(std::io::Cursor::new(index_part_bytes));
-
-    let index_part_path = conf
-        .metadata_path(sync_id.timeline_id, sync_id.tenant_id)
-        .with_file_name(IndexPart::FILE_NAME);
-    storage
-        .upload_storage_object(
-            Box::new(index_part_bytes),
-            index_part_size,
-            &index_part_path,
-        )
-        .await
-        .with_context(|| format!("Failed to upload index part for '{sync_id}'"))
-}
-
-/// Timeline upload result, with extra data, needed for uploading.
-#[derive(Debug)]
-pub(super) enum UploadedTimeline {
-    /// Upload failed due to some error, the upload task is rescheduled for another retry.
-    FailedAndRescheduled(anyhow::Error),
-    /// No issues happened during the upload, all task files were put into the remote storage.
-    Successful(SyncData<LayersUpload>),
-}
-
-/// Attempts to upload given layer files.
-/// No extra checks for overlapping files is made and any files that are already present remotely will be overwritten, if submitted during the upload.
-///
-/// On an error, bumps the retries count and reschedules the entire task.
-pub(super) async fn upload_timeline_layers<'a>(
-    storage: &'a GenericRemoteStorage,
-    sync_queue: &SyncQueue,
-    remote_timeline: Option<&'a RemoteTimeline>,
-    sync_id: TenantTimelineId,
-    mut upload_data: SyncData<LayersUpload>,
-) -> UploadedTimeline {
-    let upload = &mut upload_data.data;
-    let new_upload_lsn = upload
-        .metadata
-        .as_ref()
-        .map(|meta| meta.disk_consistent_lsn());
-
-    let already_uploaded_layers = remote_timeline
-        .map(|timeline| {
-            timeline
-                .stored_files()
-                .keys()
-                .cloned()
-                .collect::<std::collections::HashSet<_>>()
-        })
-        .unwrap_or_default();
-
-    let layers_to_upload = upload
-        .layers_to_upload
-        .iter()
-        .filter_map(|(k, v)| {
-            if !already_uploaded_layers.contains(k) {
-                Some((k.to_owned(), v.to_owned()))
-            } else {
-                None
-            }
-        })
-        .collect::<Vec<_>>();
-
-    if layers_to_upload.is_empty() {
-        debug!("No layers to upload after filtering, aborting");
-        NO_LAYERS_UPLOAD
-            .with_label_values(&[
-                &sync_id.tenant_id.to_string(),
-                &sync_id.timeline_id.to_string(),
-            ])
-            .inc();
-        return UploadedTimeline::Successful(upload_data);
-    }
-
-    debug!("Layers to upload: {layers_to_upload:?}");
-    info!(
-        "Uploading {} timeline layers, new lsn: {new_upload_lsn:?}",
-        layers_to_upload.len(),
-    );
-
-    let mut upload_tasks = layers_to_upload
-        .into_iter()
-        .map(|(source_path, known_metadata)| async move {
-            let source_file = match fs::File::open(&source_path).await.with_context(|| {
-                format!(
-                    "Failed to upen a source file for layer '{}'",
-                    source_path.display()
-                )
-            }) {
-                Ok(file) => file,
-                Err(e) => return Err(UploadError::MissingLocalFile(source_path, e)),
-            };
-
-            let fs_size = source_file
-                .metadata()
-                .await
-                .with_context(|| {
-                    format!(
-                        "Failed to get the source file metadata for layer '{}'",
-                        source_path.display()
-                    )
-                })
-                .map_err(UploadError::Other)?
-                .len();
-
-            // FIXME: this looks bad
-            if let Some(metadata_size) = known_metadata.file_size() {
-                if metadata_size != fs_size {
-                    return Err(UploadError::Other(anyhow::anyhow!(
-                        "File {source_path:?} has its current FS size {fs_size} diferent from initially determined {metadata_size}"
-                    )));
-                }
-            } else {
-                // this is a silly state we would like to avoid
-            }
-
-            let fs_size = usize::try_from(fs_size).with_context(|| format!("File {source_path:?} size {fs_size} could not be converted to usize"))
-                .map_err(UploadError::Other)?;
-
-            match storage
-                .upload_storage_object(Box::new(source_file), fs_size, &source_path)
-                .await
-                .with_context(|| format!("Failed to upload layer file for {sync_id}"))
-            {
-                Ok(()) => Ok(source_path),
-                Err(e) => Err(UploadError::MissingLocalFile(source_path, e)),
-            }
-        })
-        .collect::<FuturesUnordered<_>>();
-
-    let mut errors = Vec::new();
-    while let Some(upload_result) = upload_tasks.next().await {
-        match upload_result {
-            Ok(uploaded_path) => {
-                let metadata = upload
-                    .layers_to_upload
-                    .remove(&uploaded_path)
-                    .expect("metadata should always exist, assuming no double uploads");
-                upload.uploaded_layers.insert(uploaded_path, metadata);
-            }
-            Err(e) => match e {
-                UploadError::Other(e) => {
-                    error!("Failed to upload a layer for timeline {sync_id}: {e:?}");
-                    errors.push(format!("{e:#}"));
-                }
-                UploadError::MissingLocalFile(source_path, e) => {
-                    if source_path.exists() {
-                        error!("Failed to upload a layer for timeline {sync_id}: {e:?}");
-                        errors.push(format!("{e:#}"));
-                    } else {
-                        // We have run the upload sync task, but the file we wanted to upload is gone.
-                        // This is "fine" due the asynchronous nature of the sync loop: it only reacts to events and might need to
-                        // retry the upload tasks, if S3 or network is down: but during this time, pageserver might still operate and
-                        // run compaction/gc tasks, removing redundant files from disk.
-                        // It's not good to pause GC/compaction because of those and we would rather skip such uploads.
-                        //
-                        // Yet absence of such files might also mean that the timeline metadata file was updated (GC moves the Lsn forward, for instance).
-                        // We don't try to read a more recent version, since it could contain `disk_consistent_lsn` that does not have its upload finished yet.
-                        // This will create "missing" layers and make data inconsistent.
-                        // Instead, we only update the metadata when it was submitted in an upload task as a checkpoint result.
-                        upload.layers_to_upload.remove(&source_path);
-                        warn!(
-                            "Missing locally a layer file {} scheduled for upload, skipping",
-                            source_path.display()
-                        );
-                    }
-                }
-            },
-        }
-    }
-
-    if errors.is_empty() {
-        info!("Successfully uploaded all layers");
-        UploadedTimeline::Successful(upload_data)
-    } else {
-        debug!("Reenqueuing failed upload task for timeline {sync_id}");
-        upload_data.retries += 1;
-        sync_queue.push(sync_id, SyncTask::Upload(upload_data));
-        UploadedTimeline::FailedAndRescheduled(anyhow::anyhow!(
-            "Errors appeared during layer uploads: {:?}",
-            errors
-        ))
-    }
-}
-
-enum UploadError {
-    MissingLocalFile(PathBuf, anyhow::Error),
-    Other(anyhow::Error),
-}
-
-#[cfg(test)]
-mod tests {
-    use std::{
-        collections::{BTreeSet, HashSet},
-        num::NonZeroUsize,
-    };
-
-    use remote_storage::{LocalFs, RemoteStorage};
-    use tempfile::tempdir;
-    use utils::lsn::Lsn;
-
-    use crate::{
-        storage_sync::{
-            index::RelativePath,
-            test_utils::{create_local_timeline, dummy_metadata},
-        },
-        tenant::harness::{TenantHarness, TIMELINE_ID},
-    };
-
-    use super::{upload_index_part, *};
-
-    #[tokio::test]
-    async fn regular_layer_upload() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("regular_layer_upload")?;
-        let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-
-        let layer_files = ["a", "b"];
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_path_buf(),
-            harness.conf.workdir.clone(),
-        )?);
-        let local_storage = storage.as_local().unwrap();
-        let current_retries = 3;
-        let metadata = dummy_metadata(Lsn(0x30));
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let mut timeline_upload =
-            create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?;
-        timeline_upload.metadata = None;
-
-        assert!(
-            local_storage.list().await?.is_empty(),
-            "Storage should be empty before any uploads are made"
-        );
-
-        let upload_result = upload_timeline_layers(
-            &storage,
-            &sync_queue,
-            None,
-            sync_id,
-            SyncData::new(current_retries, timeline_upload.clone()),
-        )
-        .await;
-
-        let upload_data = match upload_result {
-            UploadedTimeline::Successful(upload_data) => upload_data,
-            wrong_result => {
-                panic!("Expected a successful upload for timeline, but got: {wrong_result:?}")
-            }
-        };
-
-        assert_eq!(
-            current_retries, upload_data.retries,
-            "On successful upload, retries are not expected to change"
-        );
-        let upload = &upload_data.data;
-        assert!(
-            upload.layers_to_upload.is_empty(),
-            "Successful upload should have no layers left to upload"
-        );
-        assert_eq!(
-            upload
-                .uploaded_layers
-                .keys()
-                .cloned()
-                .collect::<BTreeSet<_>>(),
-            layer_files
-                .iter()
-                .map(|layer_file| local_timeline_path.join(layer_file))
-                .collect(),
-            "Successful upload should have all layers uploaded"
-        );
-        assert_eq!(
-            upload.metadata, None,
-            "Successful upload without metadata should not have it returned either"
-        );
-
-        let storage_files = local_storage.list().await?;
-        assert_eq!(
-            storage_files.len(),
-            layer_files.len(),
-            "All layers should be uploaded"
-        );
-        assert_eq!(
-            storage_files
-                .into_iter()
-                .map(|storage_path| local_storage.local_path(&storage_path))
-                .collect::<anyhow::Result<BTreeSet<_>>>()?,
-            layer_files
-                .into_iter()
-                .map(|file| local_timeline_path.join(file))
-                .collect(),
-            "Uploaded files should match with the local ones"
-        );
-
-        Ok(())
-    }
-
-    // Currently, GC can run between upload retries, removing local layers scheduled for upload. Test this scenario.
-    #[tokio::test]
-    async fn layer_upload_after_local_fs_update() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("layer_upload_after_local_fs_update")?;
-        let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-
-        let layer_files = ["a1", "b1"];
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_owned(),
-            harness.conf.workdir.clone(),
-        )?);
-        let local_storage = storage.as_local().unwrap();
-        let current_retries = 5;
-        let metadata = dummy_metadata(Lsn(0x40));
-
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let layers_to_upload = {
-            let mut layers = layer_files.to_vec();
-            layers.push("layer_to_remove");
-            layers
-        };
-        let timeline_upload =
-            create_local_timeline(&harness, TIMELINE_ID, &layers_to_upload, metadata.clone())
-                .await?;
-        assert!(
-            local_storage.list().await?.is_empty(),
-            "Storage should be empty before any uploads are made"
-        );
-
-        fs::remove_file(local_timeline_path.join("layer_to_remove")).await?;
-
-        let upload_result = upload_timeline_layers(
-            &storage,
-            &sync_queue,
-            None,
-            sync_id,
-            SyncData::new(current_retries, timeline_upload.clone()),
-        )
-        .await;
-
-        let upload_data = match upload_result {
-            UploadedTimeline::Successful(upload_data) => upload_data,
-            wrong_result => panic!(
-                "Expected a successful after local fs upload for timeline, but got: {wrong_result:?}"
-            ),
-        };
-
-        assert_eq!(
-            current_retries, upload_data.retries,
-            "On successful upload, retries are not expected to change"
-        );
-        let upload = &upload_data.data;
-        assert!(
-                upload.layers_to_upload.is_empty(),
-                "Successful upload should have no layers left to upload, even those that were removed from the local fs"
-            );
-        assert_eq!(
-            upload
-                .uploaded_layers
-                .keys()
-                .cloned()
-                .collect::<BTreeSet<_>>(),
-            layer_files
-                .iter()
-                .map(|layer_file| local_timeline_path.join(layer_file))
-                .collect(),
-            "Successful upload should have all layers uploaded"
-        );
-        assert_eq!(
-            upload.metadata,
-            Some(metadata),
-            "Successful upload should not change its metadata"
-        );
-
-        let storage_files = local_storage.list().await?;
-        assert_eq!(
-            storage_files.len(),
-            layer_files.len(),
-            "All layers should be uploaded"
-        );
-        assert_eq!(
-            storage_files
-                .into_iter()
-                .map(|storage_path| local_storage.local_path(&storage_path))
-                .collect::<anyhow::Result<BTreeSet<_>>>()?,
-            layer_files
-                .into_iter()
-                .map(|file| local_timeline_path.join(file))
-                .collect(),
-            "Uploaded files should match with the local ones"
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_upload_index_part() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("test_upload_index_part")?;
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_owned(),
-            harness.conf.workdir.clone(),
-        )?);
-        let local_storage = storage.as_local().unwrap();
-        let metadata = dummy_metadata(Lsn(0x40));
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-
-        let index_part = IndexPart::new(
-            HashSet::from([
-                RelativePath::new(&local_timeline_path, local_timeline_path.join("one"))?,
-                RelativePath::new(&local_timeline_path, local_timeline_path.join("two"))?,
-            ]),
-            HashSet::from([RelativePath::new(
-                &local_timeline_path,
-                local_timeline_path.join("three"),
-            )?]),
-            metadata.disk_consistent_lsn(),
-            metadata.to_bytes()?,
-        );
-
-        assert!(
-            local_storage.list().await?.is_empty(),
-            "Storage should be empty before any uploads are made"
-        );
-        upload_index_part(harness.conf, &storage, sync_id, index_part.clone()).await?;
-
-        let storage_files = local_storage.list().await?;
-        assert_eq!(
-            storage_files.len(),
-            1,
-            "Should have only the index part file uploaded"
-        );
-
-        let index_part_path = storage_files.first().unwrap();
-        assert_eq!(
-            index_part_path.object_name(),
-            Some(IndexPart::FILE_NAME),
-            "Remote index part should have the correct name"
-        );
-        let remote_index_part: IndexPart = serde_json::from_slice(
-            &fs::read(local_storage.resolve_in_storage(index_part_path)?).await?,
-        )?;
-        assert_eq!(
-            index_part, remote_index_part,
-            "Remote index part should match the local one"
-        );
-
-        Ok(())
-    }
-}

pageserver/src/storage_sync2/delete.rs

@@ -0,0 +1,28 @@
+//! Helper functions to delete files from remote storage with a RemoteStorage
+use anyhow::Context;
+use std::path::Path;
+use tracing::debug;
+
+use remote_storage::GenericRemoteStorage;
+
+use crate::config::PageServerConf;
+
+pub(super) async fn delete_layer<'a>(
+    conf: &'static PageServerConf,
+    storage: &'a GenericRemoteStorage,
+    local_layer_path: &'a Path,
+) -> anyhow::Result<()> {
+    crate::fail_point!("before-delete-layer", |_| {
+        anyhow::bail!("failpoint before-delete-layer")
+    });
+    debug!("Deleting layer from remote storage: {local_layer_path:?}",);
+
+    let path_to_delete = conf.remote_path(local_layer_path)?;
+
+    // XXX: If the deletion fails because the object already didn't exist,
+    // it would be good to just issue a warning but consider it success.
+    // https://github.com/neondatabase/neon/issues/2934
+    storage.delete(&path_to_delete).await.with_context(|| {
+        format!("Failed to delete remote layer from storage at {path_to_delete:?}")
+    })
+}

pageserver/src/storage_sync2/download.rs

@@ -0,0 +1,221 @@
+//! Helper functions to download files from remote storage with a RemoteStorage
+use std::collections::HashSet;
+use std::path::Path;
+
+use anyhow::{bail, Context};
+use futures::stream::{FuturesUnordered, StreamExt};
+use tokio::fs;
+use tokio::io::AsyncWriteExt;
+use tracing::debug;
+
+use crate::config::PageServerConf;
+use crate::storage_sync::index::LayerFileMetadata;
+use remote_storage::{DownloadError, GenericRemoteStorage, RemotePath};
+use utils::crashsafe::path_with_suffix_extension;
+use utils::id::{TenantId, TimelineId};
+
+use super::index::IndexPart;
+
+async fn fsync_path(path: impl AsRef<std::path::Path>) -> Result<(), std::io::Error> {
+    fs::File::open(path).await?.sync_all().await
+}
+
+///
+/// If 'metadata' is given, we will validate that the downloaded file's size matches that
+/// in the metadata. (In the future, we might do more cross-checks, like CRC validation)
+///
+/// Returns the size of the downloaded file.
+pub async fn download_layer_file<'a>(
+    conf: &'static PageServerConf,
+    storage: &'a GenericRemoteStorage,
+    remote_path: &'a RemotePath,
+    layer_metadata: &'a LayerFileMetadata,
+) -> anyhow::Result<u64> {
+    let local_path = conf.local_path(remote_path);
+
+    // Perform a rename inspired by durable_rename from file_utils.c.
+    // The sequence:
+    //     write(tmp)
+    //     fsync(tmp)
+    //     rename(tmp, new)
+    //     fsync(new)
+    //     fsync(parent)
+    // For more context about durable_rename check this email from postgres mailing list:
+    // https://www.postgresql.org/message-id/56583BDD.9060302@2ndquadrant.com
+    // If pageserver crashes the temp file will be deleted on startup and re-downloaded.
+    let temp_file_path = path_with_suffix_extension(&local_path, TEMP_DOWNLOAD_EXTENSION);
+
+    // TODO: this doesn't use the cached fd for some reason?
+    let mut destination_file = fs::File::create(&temp_file_path).await.with_context(|| {
+        format!(
+            "Failed to create a destination file for layer '{}'",
+            temp_file_path.display()
+        )
+    })?;
+    let mut download = storage.download(remote_path).await.with_context(|| {
+        format!(
+            "Failed to open a download stream for layer with remote storage path '{remote_path:?}'"
+        )
+    })?;
+    let bytes_amount = tokio::io::copy(&mut download.download_stream, &mut destination_file).await.with_context(|| {
+        format!("Failed to download layer with remote storage path '{remote_path:?}' into file {temp_file_path:?}")
+    })?;
+
+    // Tokio doc here: https://docs.rs/tokio/1.17.0/tokio/fs/struct.File.html states that:
+    // A file will not be closed immediately when it goes out of scope if there are any IO operations
+    // that have not yet completed. To ensure that a file is closed immediately when it is dropped,
+    // you should call flush before dropping it.
+    //
+    // From the tokio code I see that it waits for pending operations to complete. There shouldt be any because
+    // we assume that `destination_file` file is fully written. I e there is no pending .write(...).await operations.
+    // But for additional safety lets check/wait for any pending operations.
+    destination_file.flush().await.with_context(|| {
+        format!(
+            "failed to flush source file at {}",
+            temp_file_path.display()
+        )
+    })?;
+
+    match layer_metadata.file_size() {
+        Some(expected) if expected != bytes_amount => {
+            anyhow::bail!(
+                "According to layer file metadata should had downloaded {expected} bytes but downloaded {bytes_amount} bytes into file '{}'",
+                temp_file_path.display()
+            );
+        }
+        Some(_) | None => {
+            // matches, or upgrading from an earlier IndexPart version
+        }
+    }
+
+    // not using sync_data because it can lose file size update
+    destination_file.sync_all().await.with_context(|| {
+        format!(
+            "failed to fsync source file at {}",
+            temp_file_path.display()
+        )
+    })?;
+    drop(destination_file);
+
+    crate::fail_point!("remote-storage-download-pre-rename", |_| {
+        bail!("remote-storage-download-pre-rename failpoint triggered")
+    });
+
+    fs::rename(&temp_file_path, &local_path).await?;
+
+    fsync_path(&local_path)
+        .await
+        .with_context(|| format!("Could not fsync layer file {}", local_path.display(),))?;
+
+    tracing::info!("download complete: {}", local_path.display());
+
+    Ok(bytes_amount)
+}
+
+const TEMP_DOWNLOAD_EXTENSION: &str = "temp_download";
+
+pub fn is_temp_download_file(path: &Path) -> bool {
+    let extension = path.extension().map(|pname| {
+        pname
+            .to_str()
+            .expect("paths passed to this function must be valid Rust strings")
+    });
+    match extension {
+        Some(TEMP_DOWNLOAD_EXTENSION) => true,
+        Some(_) => false,
+        None => false,
+    }
+}
+
+/// List timelines of given tenant in remote storage
+pub async fn list_remote_timelines<'a>(
+    storage: &'a GenericRemoteStorage,
+    conf: &'static PageServerConf,
+    tenant_id: TenantId,
+) -> anyhow::Result<Vec<(TimelineId, IndexPart)>> {
+    let tenant_path = conf.timelines_path(&tenant_id);
+    let tenant_storage_path = conf.remote_path(&tenant_path)?;
+
+    let timelines = storage
+        .list_prefixes(Some(&tenant_storage_path))
+        .await
+        .with_context(|| {
+            format!(
+                "Failed to list tenant storage path {tenant_storage_path:?} to get remote timelines to download"
+            )
+        })?;
+
+    if timelines.is_empty() {
+        anyhow::bail!("no timelines found on the remote storage")
+    }
+
+    let mut timeline_ids = HashSet::new();
+    let mut part_downloads = FuturesUnordered::new();
+
+    for timeline_remote_storage_key in timelines {
+        let object_name = timeline_remote_storage_key.object_name().ok_or_else(|| {
+            anyhow::anyhow!("failed to get timeline id for remote tenant {tenant_id}")
+        })?;
+
+        let timeline_id: TimelineId = object_name.parse().with_context(|| {
+            format!("failed to parse object name into timeline id '{object_name}'")
+        })?;
+
+        // list_prefixes returns all files with the prefix. If we haven't seen this timeline ID
+        // yet, launch a download task for it.
+        if !timeline_ids.contains(&timeline_id) {
+            timeline_ids.insert(timeline_id);
+            let storage_clone = storage.clone();
+            part_downloads.push(async move {
+                (
+                    timeline_id,
+                    download_index_part(conf, &storage_clone, tenant_id, timeline_id).await,
+                )
+            });
+        }
+    }
+
+    // Wait for all the download tasks to complete.
+    let mut timeline_parts = Vec::new();
+    while let Some((timeline_id, part_upload_result)) = part_downloads.next().await {
+        let index_part = part_upload_result
+            .with_context(|| format!("Failed to fetch index part for timeline {timeline_id}"))?;
+
+        debug!("Successfully fetched index part for timeline {timeline_id}");
+        timeline_parts.push((timeline_id, index_part));
+    }
+    Ok(timeline_parts)
+}
+
+pub async fn download_index_part(
+    conf: &'static PageServerConf,
+    storage: &GenericRemoteStorage,
+    tenant_id: TenantId,
+    timeline_id: TimelineId,
+) -> Result<IndexPart, DownloadError> {
+    let index_part_path = conf
+        .metadata_path(timeline_id, tenant_id)
+        .with_file_name(IndexPart::FILE_NAME);
+    let part_storage_path = conf
+        .remote_path(&index_part_path)
+        .map_err(DownloadError::BadInput)?;
+
+    let mut index_part_download = storage.download(&part_storage_path).await?;
+
+    let mut index_part_bytes = Vec::new();
+    tokio::io::copy(
+        &mut index_part_download.download_stream,
+        &mut index_part_bytes,
+    )
+    .await
+    .with_context(|| format!("Failed to download an index part into file {index_part_path:?}"))
+    .map_err(DownloadError::Other)?;
+
+    let index_part: IndexPart = serde_json::from_slice(&index_part_bytes)
+        .with_context(|| {
+            format!("Failed to deserialize index part file into file {index_part_path:?}")
+        })
+        .map_err(DownloadError::Other)?;
+
+    Ok(index_part)
+}

pageserver/src/storage_sync2/index.rs

@@ -0,0 +1,255 @@
+//! In-memory index to track the tenant files on the remote storage.
+//! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about
+//! remote timeline layers and its metadata.
+
+use std::collections::{HashMap, HashSet};
+
+use remote_storage::RemotePath;
+use serde::{Deserialize, Serialize};
+use serde_with::{serde_as, DisplayFromStr};
+
+use crate::tenant::metadata::TimelineMetadata;
+
+use utils::lsn::Lsn;
+
+/// Metadata gathered for each of the layer files.
+///
+/// Fields have to be `Option`s because remote [`IndexPart`]'s can be from different version, which
+/// might have less or more metadata depending if upgrading or rolling back an upgrade.
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+#[cfg_attr(test, derive(Default))]
+pub struct LayerFileMetadata {
+    file_size: Option<u64>,
+}
+
+impl From<&'_ IndexLayerMetadata> for LayerFileMetadata {
+    fn from(other: &IndexLayerMetadata) -> Self {
+        LayerFileMetadata {
+            file_size: other.file_size,
+        }
+    }
+}
+
+impl LayerFileMetadata {
+    pub fn new(file_size: u64) -> Self {
+        LayerFileMetadata {
+            file_size: Some(file_size),
+        }
+    }
+
+    /// This is used to initialize the metadata for remote layers, for which
+    /// the metadata was missing from the index part file.
+    pub const MISSING: Self = LayerFileMetadata { file_size: None };
+
+    pub fn file_size(&self) -> Option<u64> {
+        self.file_size
+    }
+
+    /// Metadata has holes due to version upgrades. This method is called to upgrade self with the
+    /// other value.
+    ///
+    /// This is called on the possibly outdated version.
+    pub fn merge(&mut self, other: &Self) {
+        self.file_size = other.file_size.or(self.file_size);
+    }
+}
+
+/// In-memory representation of an `index_part.json` file
+///
+/// Contains the data about all files in the timeline, present remotely and its metadata.
+///
+/// This type needs to be backwards and forwards compatible. When changing the fields,
+/// remember to add a test case for the changed version.
+#[serde_as]
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
+pub struct IndexPart {
+    /// Debugging aid describing the version of this type.
+    #[serde(default)]
+    version: usize,
+
+    /// Layer names, which are stored on the remote storage.
+    ///
+    /// Additional metadata can might exist in `layer_metadata`.
+    pub timeline_layers: HashSet<String>,
+
+    /// FIXME: unused field. This should be removed, but that changes the on-disk format,
+    /// so we need to make sure we're backwards-` (and maybe forwards-) compatible
+    /// First pass is to move it to Optional and the next would be its removal
+    missing_layers: Option<HashSet<String>>,
+
+    /// Per layer file name metadata, which can be present for a present or missing layer file.
+    ///
+    /// Older versions of `IndexPart` will not have this property or have only a part of metadata
+    /// that latest version stores.
+    #[serde(default)]
+    pub layer_metadata: HashMap<String, IndexLayerMetadata>,
+
+    // 'disk_consistent_lsn' is a copy of the 'disk_consistent_lsn' in the metadata.
+    // It's duplicated here for convenience.
+    #[serde_as(as = "DisplayFromStr")]
+    pub disk_consistent_lsn: Lsn,
+    metadata_bytes: Vec<u8>,
+}
+
+impl IndexPart {
+    /// When adding or modifying any parts of `IndexPart`, increment the version so that it can be
+    /// used to understand later versions.
+    ///
+    /// Version is currently informative only.
+    const LATEST_VERSION: usize = 1;
+    pub const FILE_NAME: &'static str = "index_part.json";
+
+    pub fn new(
+        layers_and_metadata: HashMap<RemotePath, LayerFileMetadata>,
+        disk_consistent_lsn: Lsn,
+        metadata_bytes: Vec<u8>,
+    ) -> Self {
+        let mut timeline_layers = HashSet::with_capacity(layers_and_metadata.len());
+        let mut layer_metadata = HashMap::with_capacity(layers_and_metadata.len());
+
+        for (remote_path, metadata) in &layers_and_metadata {
+            let metadata = IndexLayerMetadata::from(metadata);
+            match remote_path.object_name() {
+                Some(layer_name) => {
+                    timeline_layers.insert(layer_name.to_owned());
+                    layer_metadata.insert(layer_name.to_owned(), metadata);
+                }
+                // TODO move this on a type level: we know, that every layer entry does have a name
+                None => panic!("Layer {remote_path:?} has no file name, skipping"),
+            }
+        }
+
+        Self {
+            version: Self::LATEST_VERSION,
+            timeline_layers,
+            missing_layers: Some(HashSet::new()),
+            layer_metadata,
+            disk_consistent_lsn,
+            metadata_bytes,
+        }
+    }
+
+    pub fn parse_metadata(&self) -> anyhow::Result<TimelineMetadata> {
+        TimelineMetadata::from_bytes(&self.metadata_bytes)
+    }
+}
+
+/// Serialized form of [`LayerFileMetadata`].
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default)]
+pub struct IndexLayerMetadata {
+    file_size: Option<u64>,
+}
+
+impl From<&'_ LayerFileMetadata> for IndexLayerMetadata {
+    fn from(other: &'_ LayerFileMetadata) -> Self {
+        IndexLayerMetadata {
+            file_size: other.file_size,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn v0_indexpart_is_parsed() {
+        let example = r#"{
+            "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
+            "missing_layers":["not_a_real_layer_but_adding_coverage"],
+            "disk_consistent_lsn":"0/16960E8",
+            "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+        }"#;
+
+        let expected = IndexPart {
+            version: 0,
+            timeline_layers: HashSet::from([String::from("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9")]),
+            missing_layers: Some(HashSet::from([String::from("not_a_real_layer_but_adding_coverage")])),
+            layer_metadata: HashMap::default(),
+            disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
+            metadata_bytes: [113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0].to_vec(),
+        };
+
+        let part = serde_json::from_str::<IndexPart>(example).unwrap();
+        assert_eq!(part, expected);
+    }
+
+    #[test]
+    fn v1_indexpart_is_parsed() {
+        let example = r#"{
+            "version":1,
+            "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
+            "missing_layers":["not_a_real_layer_but_adding_coverage"],
+            "layer_metadata":{
+                "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
+                "not_a_real_layer_but_adding_coverage": { "file_size": 9007199254741001 }
+            },
+            "disk_consistent_lsn":"0/16960E8",
+            "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+        }"#;
+
+        let expected = IndexPart {
+            // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
+            version: 1,
+            timeline_layers: HashSet::from([String::from("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9")]),
+            missing_layers: Some(HashSet::from([String::from("not_a_real_layer_but_adding_coverage")])),
+            layer_metadata: HashMap::from([
+                (String::from("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"), IndexLayerMetadata {
+                    file_size: Some(25600000),
+                }),
+                (String::from("not_a_real_layer_but_adding_coverage"), IndexLayerMetadata {
+                    // serde_json should always parse this but this might be a double with jq for
+                    // example.
+                    file_size: Some(9007199254741001),
+                })
+            ]),
+            disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
+            metadata_bytes: [113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0].to_vec(),
+        };
+
+        let part = serde_json::from_str::<IndexPart>(example).unwrap();
+        assert_eq!(part, expected);
+    }
+
+    #[test]
+    fn v1_indexpart_is_parsed_with_optional_missing_layers() {
+        let example = r#"{
+            "version":1,
+            "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
+            "layer_metadata":{
+                "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
+                "not_a_real_layer_but_adding_coverage": { "file_size": 9007199254741001 }
+            },
+            "disk_consistent_lsn":"0/16960E8",
+            "metadata_bytes":[112,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+        }"#;
+
+        let expected = IndexPart {
+            // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
+            version: 1,
+            timeline_layers: HashSet::from(["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".to_string()]),
+            layer_metadata: HashMap::from([
+                (
+                    "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".to_string(),
+                    IndexLayerMetadata {
+                        file_size: Some(25600000),
+                    }
+                ),
+                (
+                    "not_a_real_layer_but_adding_coverage".to_string(),
+                    IndexLayerMetadata {
+                        // serde_json should always parse this but this might be a double with jq for
+                        // example.
+                        file_size: Some(9007199254741001),
+                    }
+                )
+            ]),
+            disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
+            metadata_bytes: [112,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0].to_vec(),
+            missing_layers: None,
+        };
+
+        let part = serde_json::from_str::<IndexPart>(example).unwrap();
+        assert_eq!(part, expected);
+    }
+}

pageserver/src/storage_sync2/upload.rs

@@ -0,0 +1,91 @@
+//! Helper functions to upload files to remote storage with a RemoteStorage
+
+use anyhow::{bail, Context};
+use std::path::Path;
+use tokio::fs;
+
+use super::index::IndexPart;
+use crate::config::PageServerConf;
+use crate::fail_point;
+use crate::storage_sync::LayerFileMetadata;
+use remote_storage::GenericRemoteStorage;
+use utils::id::{TenantId, TimelineId};
+
+/// Serializes and uploads the given index part data to the remote storage.
+pub(super) async fn upload_index_part<'a>(
+    conf: &'static PageServerConf,
+    storage: &'a GenericRemoteStorage,
+    tenant_id: TenantId,
+    timeline_id: TimelineId,
+    index_part: &'a IndexPart,
+) -> anyhow::Result<()> {
+    fail_point!("before-upload-index", |_| {
+        bail!("failpoint before-upload-index")
+    });
+    let index_part_bytes = serde_json::to_vec(&index_part)
+        .context("Failed to serialize index part file into bytes")?;
+    let index_part_size = index_part_bytes.len();
+    let index_part_bytes = tokio::io::BufReader::new(std::io::Cursor::new(index_part_bytes));
+
+    let index_part_path = conf
+        .metadata_path(timeline_id, tenant_id)
+        .with_file_name(IndexPart::FILE_NAME);
+    let storage_path = conf.remote_path(&index_part_path)?;
+    storage
+        .upload_storage_object(Box::new(index_part_bytes), index_part_size, &storage_path)
+        .await
+        .with_context(|| format!("Failed to upload index part for '{tenant_id} / {timeline_id}'"))
+}
+
+/// Attempts to upload given layer files.
+/// No extra checks for overlapping files is made and any files that are already present remotely will be overwritten, if submitted during the upload.
+///
+/// On an error, bumps the retries count and reschedules the entire task.
+pub(super) async fn upload_timeline_layer<'a>(
+    conf: &'static PageServerConf,
+    storage: &'a GenericRemoteStorage,
+    source_path: &'a Path,
+    known_metadata: &'a LayerFileMetadata,
+) -> anyhow::Result<()> {
+    fail_point!("before-upload-layer", |_| {
+        bail!("failpoint before-upload-layer")
+    });
+    let storage_path = conf.remote_path(source_path)?;
+
+    let source_file = fs::File::open(&source_path)
+        .await
+        .with_context(|| format!("Failed to open a source file for layer {source_path:?}"))?;
+
+    let fs_size = source_file
+        .metadata()
+        .await
+        .with_context(|| {
+            format!("Failed to get the source file metadata for layer {source_path:?}")
+        })?
+        .len();
+
+    // FIXME: this looks bad
+    if let Some(metadata_size) = known_metadata.file_size() {
+        if metadata_size != fs_size {
+            bail!("File {source_path:?} has its current FS size {fs_size} diferent from initially determined {metadata_size}");
+        }
+    } else {
+        // this is a silly state we would like to avoid
+    }
+
+    let fs_size = usize::try_from(fs_size).with_context(|| {
+        format!("File {source_path:?} size {fs_size} could not be converted to usize")
+    })?;
+
+    storage
+        .upload(Box::new(source_file), fs_size, &storage_path, None)
+        .await
+        .with_context(|| {
+            format!(
+                "Failed to upload a layer from local path '{}'",
+                source_path.display()
+            )
+        })?;
+
+    Ok(())
+}

pageserver/src/task_mgr.rs

@@ -197,8 +197,8 @@ pub enum TaskKind {
     // Task that flushes frozen in-memory layers to disk
     LayerFlushTask,
 
-    // Task that manages the remote upload queue
-    StorageSync,
+    // Task that uploads a file to remote storage
+    RemoteUploadTask,
 
     // task that handles the initial downloading of all tenants
     InitialLoad,

pageserver/src/tenant/metadata.rs

@@ -242,7 +242,8 @@ pub fn save_metadata(
     let mut file = VirtualFile::open_with_options(
         &path,
         OpenOptions::new().write(true).create_new(first_save),
-    )?;
+    )
+    .context("open_with_options")?;
 
     let metadata_bytes = data.to_bytes().context("Failed to get metadata bytes")?;
 
@@ -264,6 +265,26 @@ pub fn save_metadata(
     Ok(())
 }
 
+pub fn load_metadata(
+    conf: &'static PageServerConf,
+    timeline_id: TimelineId,
+    tenant_id: TenantId,
+) -> anyhow::Result<TimelineMetadata> {
+    let metadata_path = conf.metadata_path(timeline_id, tenant_id);
+    let metadata_bytes = std::fs::read(&metadata_path).with_context(|| {
+        format!(
+            "Failed to read metadata bytes from path {}",
+            metadata_path.display()
+        )
+    })?;
+    TimelineMetadata::from_bytes(&metadata_bytes).with_context(|| {
+        format!(
+            "Failed to parse metadata bytes from path {}",
+            metadata_path.display()
+        )
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

pageserver/src/tenant/timeline.rs

@@ -2,7 +2,6 @@
 
 use anyhow::{anyhow, bail, ensure, Context};
 use bytes::Bytes;
-use fail::fail_point;
 use itertools::Itertools;
 use once_cell::sync::OnceCell;
 use pageserver_api::models::TimelineState;
@@ -14,11 +13,14 @@ use std::cmp::{max, min, Ordering};
 use std::collections::{HashMap, HashSet};
 use std::fs;
 use std::ops::{Deref, Range};
-use std::path::PathBuf;
-use std::sync::atomic::{self, AtomicBool, AtomicI64, Ordering as AtomicOrdering};
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, AtomicI64, Ordering as AtomicOrdering};
 use std::sync::{Arc, Mutex, MutexGuard, RwLock};
 use std::time::{Duration, Instant, SystemTime};
 
+use crate::fail_point;
+use crate::storage_sync::index::IndexPart;
+use crate::storage_sync::RemoteTimelineClient;
 use crate::tenant::{
     delta_layer::{DeltaLayer, DeltaLayerWriter},
     ephemeral_file::is_ephemeral_file,
@@ -51,17 +53,14 @@ use utils::{
 
 use crate::repository::GcResult;
 use crate::repository::{Key, Value};
-use crate::task_mgr;
 use crate::task_mgr::TaskKind;
 use crate::walreceiver::{is_etcd_client_initialized, spawn_connection_manager_task};
 use crate::walredo::WalRedoManager;
 use crate::CheckpointConfig;
 use crate::METADATA_FILE_NAME;
 use crate::ZERO_PAGE;
-use crate::{
-    page_cache,
-    storage_sync::{self, index::LayerFileMetadata},
-};
+use crate::{is_temporary, task_mgr};
+use crate::{page_cache, storage_sync::index::LayerFileMetadata};
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 enum FlushLoopState {
@@ -88,6 +87,13 @@ pub struct Timeline {
     // WAL redo manager
     walredo_mgr: Arc<dyn WalRedoManager + Sync + Send>,
 
+    /// Remote storage client.
+    ///
+    /// If Some, use it to upload all newly created layers to the remote storage,
+    /// and keep remote metadata file in sync. In the future, also use it to download
+    /// layer files on-demand.
+    pub remote_client: Option<Arc<RemoteTimelineClient>>,
+
     // What page versions do we hold in the repository? If we get a
     // request > last_record_lsn, we need to wait until we receive all
     // the WAL up to the request. The SeqWait provides functions for
@@ -120,9 +126,6 @@ pub struct Timeline {
     // Metrics
     metrics: TimelineMetrics,
 
-    /// If `true`, will backup its files that appear after each checkpointing to the remote storage.
-    upload_layers: AtomicBool,
-
     /// Ensures layers aren't frozen by checkpointer between
     /// [`Timeline::get_layer_for_write`] and layer reads.
     /// Locked automatically by [`TimelineWriter`] and checkpointer.
@@ -145,7 +148,7 @@ pub struct Timeline {
     /// A lock to ensure that no layer of the timeline is removed concurrently by other tasks.
     /// This lock is acquired in [`Timeline::gc`], [`Timeline::compact`],
     /// and [`Tenant::delete_timeline`].
-    layer_removal_cs: Mutex<()>,
+    pub(super) layer_removal_cs: tokio::sync::Mutex<()>,
 
     // Needed to ensure that we can't create a branch at a point that was already garbage collected
     pub latest_gc_cutoff_lsn: Rcu<Lsn>,
@@ -409,6 +412,14 @@ impl Timeline {
         self.disk_consistent_lsn.load()
     }
 
+    pub fn get_remote_consistent_lsn(&self) -> Option<Lsn> {
+        if let Some(remote_client) = &self.remote_client {
+            remote_client.last_uploaded_consistent_lsn()
+        } else {
+            None
+        }
+    }
+
     /// Get the physical size of the timeline at the latest LSN
     pub fn get_physical_size(&self) -> u64 {
         self.metrics.current_physical_size_gauge.get()
@@ -493,15 +504,15 @@ impl Timeline {
             CheckpointConfig::Forced => {
                 self.freeze_inmem_layer(false);
                 self.flush_frozen_layers_and_wait().await?;
-                self.compact()
+                self.compact().await
             }
         }
     }
 
-    pub fn compact(&self) -> anyhow::Result<()> {
+    pub async fn compact(&self) -> anyhow::Result<()> {
         let last_record_lsn = self.get_last_record_lsn();
 
-        // Last record Lsn could be zero in case the timelie was just created
+        // Last record Lsn could be zero in case the timeline was just created
         if !last_record_lsn.is_valid() {
             warn!("Skipping compaction for potentially just initialized timeline, it has invalid last record lsn: {last_record_lsn}");
             return Ok(());
@@ -541,7 +552,12 @@ impl Timeline {
         // Below are functions compact_level0() and create_image_layers()
         // but they are a bit ad hoc and don't quite work like it's explained
         // above. Rewrite it.
-        let _layer_removal_cs = self.layer_removal_cs.lock().unwrap();
+        let _layer_removal_cs = self.layer_removal_cs.lock().await;
+        // Is the timeline being deleted?
+        let state = *self.state.borrow();
+        if state == TimelineState::Stopping {
+            anyhow::bail!("timeline is Stopping");
+        }
 
         let target_file_size = self.get_checkpoint_distance();
 
@@ -555,20 +571,15 @@ impl Timeline {
                 // 2. Create new image layers for partitions that have been modified
                 // "enough".
                 let layer_paths_to_upload = self.create_image_layers(&partitioning, lsn, false)?;
-                if !layer_paths_to_upload.is_empty()
-                    && self.upload_layers.load(atomic::Ordering::Relaxed)
-                {
-                    storage_sync::schedule_layer_upload(
-                        self.tenant_id,
-                        self.timeline_id,
-                        layer_paths_to_upload,
-                        None,
-                    );
+                if let Some(remote_client) = &self.remote_client {
+                    for (path, layer_metadata) in layer_paths_to_upload {
+                        remote_client.schedule_layer_file_upload(&path, &layer_metadata)?;
+                    }
                 }
 
                 // 3. Compact
                 let timer = self.metrics.compact_time_histo.start_timer();
-                self.compact_level0(target_file_size)?;
+                self.compact_level0(target_file_size).await?;
                 timer.stop_and_record();
             }
             Err(err) => {
@@ -657,8 +668,8 @@ impl Timeline {
             (TimelineState::Broken, _) => {
                 error!("Ignoring state update {new_state:?} for broken tenant");
             }
-            (TimelineState::Paused, TimelineState::Active) => {
-                debug!("Not activating a paused timeline");
+            (TimelineState::Stopping, TimelineState::Active) => {
+                debug!("Not activating a Stopping timeline");
             }
             (_, new_state) => {
                 self.state.send_replace(new_state);
@@ -728,7 +739,7 @@ impl Timeline {
         timeline_id: TimelineId,
         tenant_id: TenantId,
         walredo_mgr: Arc<dyn WalRedoManager + Send + Sync>,
-        upload_layers: bool,
+        remote_client: Option<RemoteTimelineClient>,
         pg_version: u32,
     ) -> Self {
         let disk_consistent_lsn = metadata.disk_consistent_lsn();
@@ -747,6 +758,8 @@ impl Timeline {
 
             walredo_mgr,
 
+            remote_client: remote_client.map(Arc::new),
+
             // initialize in-memory 'last_record_lsn' from 'disk_consistent_lsn'.
             last_record_lsn: SeqWait::new(RecordLsn {
                 last: disk_consistent_lsn,
@@ -762,15 +775,13 @@ impl Timeline {
 
             metrics: TimelineMetrics::new(&tenant_id, &timeline_id),
 
-            upload_layers: AtomicBool::new(upload_layers),
-
             flush_loop_state: Mutex::new(FlushLoopState::NotStarted),
 
             layer_flush_start_tx,
             layer_flush_done_tx,
 
             write_lock: Mutex::new(()),
-            layer_removal_cs: Mutex::new(()),
+            layer_removal_cs: Default::default(),
 
             gc_info: RwLock::new(GcInfo {
                 retain_lsns: Vec::new(),
@@ -898,6 +909,7 @@ impl Timeline {
 
         for direntry in fs::read_dir(timeline_path)? {
             let direntry = direntry?;
+            let direntry_path = direntry.path();
             let fname = direntry.file_name();
             let fname = fname.to_string_lossy();
 
@@ -909,7 +921,7 @@ impl Timeline {
                         imgfilename, self.timeline_id, disk_consistent_lsn
                     );
 
-                    rename_to_backup(direntry.path())?;
+                    rename_to_backup(&direntry_path)?;
                     continue;
                 }
 
@@ -933,7 +945,7 @@ impl Timeline {
                         deltafilename, self.timeline_id, disk_consistent_lsn
                     );
 
-                    rename_to_backup(direntry.path())?;
+                    rename_to_backup(&direntry_path)?;
                     continue;
                 }
 
@@ -946,10 +958,23 @@ impl Timeline {
                 num_layers += 1;
             } else if fname == METADATA_FILE_NAME || fname.ends_with(".old") {
                 // ignore these
+            } else if crate::storage_sync::is_temp_download_file(&direntry_path) {
+                info!(
+                    "skipping temp download file, reconcile_with_remote will resume / clean up: {}",
+                    fname
+                );
             } else if is_ephemeral_file(&fname) {
                 // Delete any old ephemeral files
                 trace!("deleting old ephemeral file in timeline dir: {}", fname);
-                fs::remove_file(direntry.path())?;
+                fs::remove_file(&direntry_path)?;
+            } else if is_temporary(&direntry_path) {
+                info!("removing temp timeline file at {}", direntry_path.display());
+                fs::remove_file(&direntry_path).with_context(|| {
+                    format!(
+                        "failed to remove temp download file at {}",
+                        direntry_path.display()
+                    )
+                })?;
             } else {
                 warn!("unrecognized filename in timeline dir: {}", fname);
             }
@@ -970,10 +995,222 @@ impl Timeline {
         Ok(())
     }
 
-    pub(super) fn layer_removal_guard(&self) -> anyhow::Result<MutexGuard<()>> {
-        self.layer_removal_cs
-            .try_lock()
-            .map_err(|e| anyhow!("cannot lock compaction critical section {e}"))
+    async fn download_missing(
+        &self,
+        index_part: &IndexPart,
+        remote_client: &RemoteTimelineClient,
+        local_layers: HashSet<PathBuf>,
+        up_to_date_disk_consistent_lsn: Lsn,
+    ) -> anyhow::Result<HashSet<PathBuf>> {
+        // Are we missing some files that are present in remote storage?
+        // Download them now.
+        // TODO Downloading many files this way is not efficient.
+        //     Better to use FuturesUnordered. Maybe keep as is because:
+        //    a) inplace download is a throw-away code, on-demand patch doesnt need that
+        //    b) typical case now is that there is nothing to sync, this downloads a lot
+        //       1) if there was another pageserver that came and generated new files
+        //       2) during attach of a timeline with big history which we currently do not do
+        let mut local_only_layers = local_layers;
+        let timeline_dir = self.conf.timeline_path(&self.timeline_id, &self.tenant_id);
+        for remote_layer_name in &index_part.timeline_layers {
+            let local_layer_path = timeline_dir.join(remote_layer_name);
+            local_only_layers.remove(&local_layer_path);
+
+            let remote_layer_metadata = index_part
+                .layer_metadata
+                .get(remote_layer_name)
+                .map(LayerFileMetadata::from)
+                .unwrap_or(LayerFileMetadata::MISSING);
+
+            let remote_layer_path = self
+                .conf
+                .remote_path(&local_layer_path)
+                .expect("local_layer_path received from the same conf that provided a workdir");
+
+            if local_layer_path.exists() {
+                let mut already_downloaded = true;
+                // Are there any local files that exist, with a size that doesn't match
+                // with the size stored in the remote index file?
+                // If so, rename_to_backup those files so that we re-download them later.
+                if let Some(remote_size) = remote_layer_metadata.file_size() {
+                    match local_layer_path.metadata() {
+                        Ok(metadata) => {
+                            let local_size = metadata.len();
+
+                            if local_size != remote_size {
+                                warn!("removing local file {local_layer_path:?} because it has unexpected length {local_size}; length in remote index is {remote_size}");
+                                if let Err(err) = rename_to_backup(&local_layer_path) {
+                                    error!("could not rename file {local_layer_path:?}: {err:?}");
+                                } else {
+                                    self.metrics.current_physical_size_gauge.sub(local_size);
+                                    already_downloaded = false;
+                                }
+                            }
+                        }
+                        Err(err) => {
+                            error!("could not get size of local file {local_layer_path:?}: {err:?}")
+                        }
+                    }
+                }
+
+                if already_downloaded {
+                    continue;
+                }
+            } else {
+                info!("remote layer {remote_layer_path:?} does not exist locally");
+            }
+
+            let layer_name = local_layer_path
+                .file_name()
+                .and_then(|os_str| os_str.to_str())
+                .with_context(|| {
+                    format!("Layer file {local_layer_path:?} has no name in unicode")
+                })?;
+            if let Some(imgfilename) = ImageFileName::parse_str(layer_name) {
+                if imgfilename.lsn > up_to_date_disk_consistent_lsn {
+                    warn!(
+                        "found future image layer {} on timeline {} remote_consistent_lsn is {}",
+                        imgfilename, self.timeline_id, up_to_date_disk_consistent_lsn
+                    );
+                    continue;
+                }
+
+                trace!("downloading image file: {remote_layer_path:?}");
+                let downloaded_size = remote_client
+                    .download_layer_file(&remote_layer_path, &remote_layer_metadata)
+                    .await
+                    .with_context(|| {
+                        format!("failed to download image layer from path {remote_layer_path:?}")
+                    })?;
+                trace!("done");
+
+                let image_layer =
+                    ImageLayer::new(self.conf, self.timeline_id, self.tenant_id, &imgfilename);
+
+                self.layers
+                    .write()
+                    .unwrap()
+                    .insert_historic(Arc::new(image_layer));
+                self.metrics
+                    .current_physical_size_gauge
+                    .add(downloaded_size);
+            } else if let Some(deltafilename) = DeltaFileName::parse_str(layer_name) {
+                // Create a DeltaLayer struct for each delta file.
+                // The end-LSN is exclusive, while disk_consistent_lsn is
+                // inclusive. For example, if disk_consistent_lsn is 100, it is
+                // OK for a delta layer to have end LSN 101, but if the end LSN
+                // is 102, then it might not have been fully flushed to disk
+                // before crash.
+                if deltafilename.lsn_range.end > up_to_date_disk_consistent_lsn + 1 {
+                    warn!(
+                        "found future delta layer {} on timeline {} remote_consistent_lsn is {}",
+                        deltafilename, self.timeline_id, up_to_date_disk_consistent_lsn
+                    );
+                    continue;
+                }
+
+                trace!("downloading delta file: {remote_layer_path:?}");
+                let sz = remote_client
+                    .download_layer_file(&remote_layer_path, &remote_layer_metadata)
+                    .await
+                    .with_context(|| {
+                        format!("failed to download delta layer from path {remote_layer_path:?}")
+                    })?;
+                trace!("done");
+
+                let delta_layer =
+                    DeltaLayer::new(self.conf, self.timeline_id, self.tenant_id, &deltafilename);
+
+                self.layers
+                    .write()
+                    .unwrap()
+                    .insert_historic(Arc::new(delta_layer));
+                self.metrics.current_physical_size_gauge.add(sz);
+            } else {
+                bail!("unexpected layer filename {layer_name} in remote storage path: {remote_layer_path:?}");
+            }
+        }
+
+        Ok(local_only_layers)
+    }
+
+    ///
+    /// This function will synchronize local data with what we have in remote storage.
+    /// 1. It will download missing layer files.
+    /// 2. It will update local metadata if remote one has greater `disk_consistent_lsn`.
+    /// 3. It will upload files that are missing on the remote
+    /// 4. It will update index file on the remote accordingly
+    /// TODO may be a bit cleaner to do things based on populated remote client,
+    ///     and then do things based on its upload_queue.latest_files
+    ///
+    /// This is used during tenant attach. The layer map must have been loaded
+    /// with local filesystem contents already.
+    ///
+    /// The caller should provide IndexPart if it exists on the remote storage. If it's None,
+    /// we assume that it is missing on the remote storage, which means that we initialized
+    /// a timeline and then restarted before successful upload was performed
+    ///
+    #[instrument(skip(self, index_part, up_to_date_metadata))]
+    pub async fn reconcile_with_remote(
+        &self,
+        up_to_date_metadata: &TimelineMetadata,
+        index_part: Option<&IndexPart>,
+    ) -> anyhow::Result<()> {
+        info!("starting");
+        let remote_client = self
+            .remote_client
+            .as_ref()
+            .ok_or_else(|| anyhow!("cannot download without remote storage"))?;
+
+        let disk_consistent_lsn = up_to_date_metadata.disk_consistent_lsn();
+
+        // Build a map of local layers for quick lookups
+        let local_layers = self
+            .layers
+            .read()
+            .unwrap()
+            .iter_historic_layers()
+            .map(|historic_layer| {
+                historic_layer
+                    .local_path()
+                    .expect("Historic layers should have a path")
+            })
+            .collect::<HashSet<_>>();
+
+        let local_only_layers = match index_part {
+            Some(index_part) => {
+                info!(
+                    "initializing upload queue from remote index with {} layer files",
+                    index_part.timeline_layers.len()
+                );
+                remote_client.init_upload_queue(index_part)?;
+                self.download_missing(index_part, remote_client, local_layers, disk_consistent_lsn)
+                    .await?
+            }
+            None => {
+                info!("initializing upload queue as empty");
+                remote_client.init_upload_queue_for_empty_remote(up_to_date_metadata)?;
+                local_layers
+            }
+        };
+
+        // Are there local files that don't exist remotely? Schedule uploads for them
+        for layer_path in &local_only_layers {
+            let layer_size = layer_path
+                .metadata()
+                .with_context(|| format!("failed to get file {layer_path:?} metadata"))?
+                .len();
+            info!("scheduling {layer_path:?} for upload");
+            remote_client
+                .schedule_layer_file_upload(layer_path, &LayerFileMetadata::new(layer_size))?;
+        }
+        if !local_only_layers.is_empty() {
+            remote_client.schedule_index_upload(up_to_date_metadata)?;
+        }
+
+        info!("Done");
+
+        Ok(())
     }
 
     fn try_spawn_size_init_task(self: &Arc<Self>, init_lsn: Lsn) {
@@ -1014,7 +1251,7 @@ impl Timeline {
                                         match new_state {
                                             // we're running this job for active timelines only
                                             TimelineState::Active => continue,
-                                            TimelineState::Broken | TimelineState::Paused | TimelineState::Suspended => return Some(new_state),
+                                            TimelineState::Broken | TimelineState::Stopping | TimelineState::Suspended => return Some(new_state),
                                         }
                                     }
                                     Err(_sender_dropped_error) => return None,
@@ -1475,7 +1712,8 @@ impl Timeline {
         // After crash, we will restart WAL streaming and processing from that point.
         if disk_consistent_lsn != old_disk_consistent_lsn {
             assert!(disk_consistent_lsn > old_disk_consistent_lsn);
-            self.update_metadata_file(disk_consistent_lsn, layer_paths_to_upload)?;
+            self.update_metadata_file(disk_consistent_lsn, layer_paths_to_upload)
+                .context("update_metadata_file")?;
             // Also update the in-memory copy
             self.disk_consistent_lsn.store(disk_consistent_lsn);
         }
@@ -1530,15 +1768,18 @@ impl Timeline {
             self.tenant_id,
             &metadata,
             false,
-        )?;
+        )
+        .context("save_metadata")?;
 
-        if self.can_upload_layers() {
-            storage_sync::schedule_layer_upload(
-                self.tenant_id,
-                self.timeline_id,
-                layer_paths_to_upload,
-                Some(metadata),
-            );
+        if let Some(remote_client) = &self.remote_client {
+            for (path, layer_metadata) in layer_paths_to_upload {
+                remote_client
+                    .schedule_layer_file_upload(&path, &layer_metadata)
+                    .context("schedule_layer_file_upload")?;
+            }
+            remote_client
+                .schedule_index_upload(&metadata)
+                .context("schedule_layer_file_upload")?;
         }
 
         Ok(())
@@ -1734,19 +1975,25 @@ impl Timeline {
 
         Ok(layer_paths_to_upload)
     }
+}
+#[derive(Default)]
+struct CompactLevel0Phase1Result {
+    new_layers: Vec<DeltaLayer>,
+    deltas_to_compact: Vec<Arc<dyn Layer>>,
+}
 
-    ///
-    /// Collect a bunch of Level 0 layer files, and compact and reshuffle them as
-    /// as Level 1 files.
-    ///
-    fn compact_level0(&self, target_file_size: u64) -> anyhow::Result<()> {
+impl Timeline {
+    async fn compact_level0_phase1(
+        &self,
+        target_file_size: u64,
+    ) -> anyhow::Result<CompactLevel0Phase1Result> {
         let layers = self.layers.read().unwrap();
         let mut level0_deltas = layers.get_level0_deltas()?;
         drop(layers);
 
         // Only compact if enough layers have accumulated.
         if level0_deltas.is_empty() || level0_deltas.len() < self.get_compaction_threshold() {
-            return Ok(());
+            return Ok(Default::default());
         }
 
         // Gather the files to compact in this iteration.
@@ -1981,6 +2228,35 @@ impl Timeline {
             layer_paths.pop().unwrap();
         }
 
+        drop(all_keys_iter); // So that deltas_to_compact is no longer borrowed
+
+        Ok(CompactLevel0Phase1Result {
+            new_layers,
+            deltas_to_compact,
+        })
+    }
+
+    ///
+    /// Collect a bunch of Level 0 layer files, and compact and reshuffle them as
+    /// as Level 1 files.
+    ///
+    async fn compact_level0(&self, target_file_size: u64) -> anyhow::Result<()> {
+        let CompactLevel0Phase1Result {
+            new_layers,
+            deltas_to_compact,
+        } = self.compact_level0_phase1(target_file_size).await?;
+
+        // Before deleting any layers, we need to wait for their upload ops to finish.
+        // See storage_sync module level comment on consistency.
+        // Do it here because we don't want to hold self.layers.write() while waiting.
+        if let Some(remote_client) = &self.remote_client {
+            info!("waiting for upload ops to complete");
+            remote_client
+                .wait_completion()
+                .await
+                .context("wait for layer upload ops to complete")?;
+        }
+
         let mut layers = self.layers.write().unwrap();
         let mut new_layer_paths = HashMap::with_capacity(new_layers.len());
         for l in new_layers {
@@ -1988,6 +2264,13 @@ impl Timeline {
 
             let metadata = new_delta_path.metadata()?;
 
+            if let Some(remote_client) = &self.remote_client {
+                remote_client.schedule_layer_file_upload(
+                    &new_delta_path,
+                    &LayerFileMetadata::new(metadata.len()),
+                )?;
+            }
+
             // update the timeline's physical size
             self.metrics.current_physical_size_gauge.add(metadata.len());
 
@@ -1997,32 +2280,22 @@ impl Timeline {
 
         // Now that we have reshuffled the data to set of new delta layers, we can
         // delete the old ones
-        let mut layer_paths_do_delete = HashSet::with_capacity(deltas_to_compact.len());
-        drop(all_keys_iter);
+        let mut layer_paths_to_delete = Vec::with_capacity(deltas_to_compact.len());
         for l in deltas_to_compact {
             if let Some(path) = l.local_path() {
                 self.metrics
                     .current_physical_size_gauge
                     .sub(path.metadata()?.len());
-                layer_paths_do_delete.insert(path);
+                layer_paths_to_delete.push(path);
             }
             l.delete()?;
             layers.remove_historic(l);
         }
         drop(layers);
 
-        if self.can_upload_layers() {
-            storage_sync::schedule_layer_upload(
-                self.tenant_id,
-                self.timeline_id,
-                new_layer_paths,
-                None,
-            );
-            storage_sync::schedule_layer_delete(
-                self.tenant_id,
-                self.timeline_id,
-                layer_paths_do_delete,
-            );
+        // Also schedule the deletions in remote storage
+        if let Some(remote_client) = &self.remote_client {
+            remote_client.schedule_layer_file_deletion(&layer_paths_to_delete)?;
         }
 
         Ok(())
@@ -2114,25 +2387,43 @@ impl Timeline {
     /// within a layer file. We can only remove the whole file if it's fully
     /// obsolete.
     ///
-    pub(super) fn gc(&self) -> anyhow::Result<GcResult> {
-        let mut result: GcResult = GcResult::default();
-        let now = SystemTime::now();
-
+    pub(super) async fn gc(&self) -> anyhow::Result<GcResult> {
         fail_point!("before-timeline-gc");
 
-        let _layer_removal_cs = self.layer_removal_cs.lock().unwrap();
+        let _layer_removal_cs = self.layer_removal_cs.lock().await;
+        // Is the timeline being deleted?
+        let state = *self.state.borrow();
+        if state == TimelineState::Stopping {
+            anyhow::bail!("timeline is Stopping");
+        }
 
-        let gc_info = self.gc_info.read().unwrap();
+        let (horizon_cutoff, pitr_cutoff, retain_lsns) = {
+            let gc_info = self.gc_info.read().unwrap();
 
-        let horizon_cutoff = min(gc_info.horizon_cutoff, self.get_disk_consistent_lsn());
-        let pitr_cutoff = gc_info.pitr_cutoff;
-        let retain_lsns = &gc_info.retain_lsns;
+            let horizon_cutoff = min(gc_info.horizon_cutoff, self.get_disk_consistent_lsn());
+            let pitr_cutoff = gc_info.pitr_cutoff;
+            let retain_lsns = gc_info.retain_lsns.clone();
+            (horizon_cutoff, pitr_cutoff, retain_lsns)
+        };
 
         let new_gc_cutoff = Lsn::min(horizon_cutoff, pitr_cutoff);
 
-        let _enter =
-            info_span!("gc_timeline", timeline = %self.timeline_id, cutoff = %new_gc_cutoff)
-                .entered();
+        self.gc_timeline(horizon_cutoff, pitr_cutoff, retain_lsns, new_gc_cutoff)
+            .instrument(
+                info_span!("gc_timeline", timeline = %self.timeline_id, cutoff = %new_gc_cutoff),
+            )
+            .await
+    }
+
+    async fn gc_timeline(
+        &self,
+        horizon_cutoff: Lsn,
+        pitr_cutoff: Lsn,
+        retain_lsns: Vec<Lsn>,
+        new_gc_cutoff: Lsn,
+    ) -> anyhow::Result<GcResult> {
+        let now = SystemTime::now();
+        let mut result: GcResult = GcResult::default();
 
         // Nothing to GC. Return early.
         let latest_gc_cutoff = *self.get_latest_gc_cutoff_lsn();
@@ -2166,6 +2457,17 @@ impl Timeline {
 
         debug!("retain_lsns: {:?}", retain_lsns);
 
+        // Before deleting any layers, we need to wait for their upload ops to finish.
+        // See storage_sync module level comment on consistency.
+        // Do it here because we don't want to hold self.layers.write() while waiting.
+        if let Some(remote_client) = &self.remote_client {
+            info!("waiting for upload ops to complete");
+            remote_client
+                .wait_completion()
+                .await
+                .context("wait for layer upload ops to complete")?;
+        }
+
         let mut layers_to_remove = Vec::new();
 
         // Scan all on-disk layers in the timeline.
@@ -2217,7 +2519,7 @@ impl Timeline {
             // might be referenced by child branches forever.
             // We can track this in child timeline GC and delete parent layers when
             // they are no longer needed. This might be complicated with long inheritance chains.
-            for retain_lsn in retain_lsns {
+            for retain_lsn in &retain_lsns {
                 // start_lsn is inclusive
                 if &l.get_lsn_range().start <= retain_lsn {
                     debug!(
@@ -2273,13 +2575,13 @@ impl Timeline {
         // Actually delete the layers from disk and remove them from the map.
         // (couldn't do this in the loop above, because you cannot modify a collection
         // while iterating it. BTreeMap::retain() would be another option)
-        let mut layer_paths_to_delete = HashSet::with_capacity(layers_to_remove.len());
+        let mut layer_paths_to_delete = Vec::with_capacity(layers_to_remove.len());
         for doomed_layer in layers_to_remove {
             if let Some(path) = doomed_layer.local_path() {
                 self.metrics
                     .current_physical_size_gauge
                     .sub(path.metadata()?.len());
-                layer_paths_to_delete.insert(path);
+                layer_paths_to_delete.push(path);
             }
             doomed_layer.delete()?;
             layers.remove_historic(doomed_layer);
@@ -2295,12 +2597,8 @@ impl Timeline {
             fail_point!("after-timeline-gc-removed-layers");
         }
 
-        if self.can_upload_layers() {
-            storage_sync::schedule_layer_delete(
-                self.tenant_id,
-                self.timeline_id,
-                layer_paths_to_delete,
-            );
+        if let Some(remote_client) = &self.remote_client {
+            remote_client.schedule_layer_file_deletion(&layer_paths_to_delete)?;
         }
 
         result.elapsed = now.elapsed()?;
@@ -2344,24 +2642,22 @@ impl Timeline {
                     data.records.len()
                 );
             } else {
-                let base_img = if let Some((_lsn, img)) = data.img {
+                if data.img.is_some() {
                     trace!(
                         "found {} WAL records and a base image for {} at {}, performing WAL redo",
                         data.records.len(),
                         key,
                         request_lsn
                     );
-                    Some(img)
                 } else {
                     trace!("found {} WAL records that will init the page for {} at {}, performing WAL redo", data.records.len(), key, request_lsn);
-                    None
                 };
 
                 let last_rec_lsn = data.records.last().unwrap().0;
 
                 let img = self
                     .walredo_mgr
-                    .request_redo(key, request_lsn, base_img, data.records, self.pg_version)
+                    .request_redo(key, request_lsn, data.img, data.records, self.pg_version)
                     .context("Failed to reconstruct a page image:")?;
 
                 if img.len() == page_cache::PAGE_SZ {
@@ -2381,11 +2677,6 @@ impl Timeline {
             }
         }
     }
-
-    fn can_upload_layers(&self) -> bool {
-        self.upload_layers.load(atomic::Ordering::Relaxed)
-            && self.current_state() != TimelineState::Broken
-    }
 }
 
 /// Helper function for get_reconstruct_data() to add the path of layers traversed
@@ -2463,12 +2754,12 @@ impl<'a> TimelineWriter<'a> {
 
 /// Add a suffix to a layer file's name: .{num}.old
 /// Uses the first available num (starts at 0)
-fn rename_to_backup(path: PathBuf) -> anyhow::Result<()> {
+fn rename_to_backup(path: &Path) -> anyhow::Result<()> {
     let filename = path
         .file_name()
         .ok_or_else(|| anyhow!("Path {} don't have a file name", path.display()))?
         .to_string_lossy();
-    let mut new_path = path.clone();
+    let mut new_path = path.to_owned();
 
     for i in 0u32.. {
         new_path.set_file_name(format!("{}.{}.old", filename, i));

pageserver/src/tenant_config.rs

@@ -51,6 +51,7 @@ pub struct TenantConf {
     // This parameter determines L1 layer file size.
     pub compaction_target_size: u64,
     // How often to check if there's compaction work to be done.
+    // Duration::ZERO means automatic compaction is disabled.
     #[serde(with = "humantime_serde")]
     pub compaction_period: Duration,
     // Level0 delta layer threshold for compaction.
@@ -61,6 +62,7 @@ pub struct TenantConf {
     // Page versions older than this are garbage collected away.
     pub gc_horizon: u64,
     // Interval at which garbage collection is triggered.
+    // Duration::ZERO means automatic GC is disabled
     #[serde(with = "humantime_serde")]
     pub gc_period: Duration,
     // Delta layer churn threshold to create L1 image layers.
@@ -183,6 +185,9 @@ impl TenantConfOpt {
         if let Some(max_lsn_wal_lag) = other.max_lsn_wal_lag {
             self.max_lsn_wal_lag = Some(max_lsn_wal_lag);
         }
+        if let Some(trace_read_requests) = other.trace_read_requests {
+            self.trace_read_requests = Some(trace_read_requests);
+        }
     }
 }
 

pageserver/src/tenant_tasks.rs

@@ -6,7 +6,8 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use crate::metrics::TENANT_TASK_EVENTS;
-use crate::task_mgr::{self, TaskKind, BACKGROUND_RUNTIME};
+use crate::task_mgr;
+use crate::task_mgr::{TaskKind, BACKGROUND_RUNTIME};
 use crate::tenant::{Tenant, TenantState};
 use crate::tenant_mgr;
 use tracing::*;
@@ -65,13 +66,17 @@ async fn compaction_loop(tenant_id: TenantId) {
                 },
             };
 
-            // Run blocking part of the task
-
-            // Run compaction
             let mut sleep_duration = tenant.get_compaction_period();
-            if let Err(e) = tenant.compaction_iteration() {
-                sleep_duration = wait_duration;
-                error!("Compaction failed, retrying in {:?}: {e:?}", sleep_duration);
+            if sleep_duration == Duration::ZERO {
+                info!("automatic compaction is disabled");
+                // check again in 10 seconds, in case it's been enabled again.
+                sleep_duration = Duration::from_secs(10);
+            } else {
+                // Run compaction
+                if let Err(e) = tenant.compaction_iteration().await {
+                    sleep_duration = wait_duration;
+                    error!("Compaction failed, retrying in {:?}: {e:?}", sleep_duration);
+                }
             }
 
             // Sleep
@@ -112,15 +117,21 @@ async fn gc_loop(tenant_id: TenantId) {
                 },
             };
 
-            // Run gc
             let gc_period = tenant.get_gc_period();
             let gc_horizon = tenant.get_gc_horizon();
             let mut sleep_duration = gc_period;
-            if gc_horizon > 0 {
-                if let Err(e) = tenant.gc_iteration(None, gc_horizon, tenant.get_pitr_interval(), false).await
-                {
-                    sleep_duration = wait_duration;
-                    error!("Gc failed, retrying in {:?}: {e:?}", sleep_duration);
+            if sleep_duration == Duration::ZERO {
+                info!("automatic GC is disabled");
+                // check again in 10 seconds, in case it's been enabled again.
+                sleep_duration = Duration::from_secs(10);
+            } else {
+                // Run gc
+                if gc_horizon > 0 {
+                    if let Err(e) = tenant.gc_iteration(None, gc_horizon, tenant.get_pitr_interval(), false).await
+                    {
+                        sleep_duration = wait_duration;
+                        error!("Gc failed, retrying in {:?}: {e:?}", sleep_duration);
+                    }
                 }
             }
 
@@ -144,7 +155,7 @@ async fn wait_for_active_tenant(
     wait: Duration,
 ) -> ControlFlow<(), Arc<Tenant>> {
     let tenant = loop {
-        match tenant_mgr::get_tenant(tenant_id, false) {
+        match tenant_mgr::get_tenant(tenant_id, false).await {
             Ok(tenant) => break tenant,
             Err(e) => {
                 error!("Failed to get a tenant {tenant_id}: {e:#}");
@@ -154,7 +165,7 @@ async fn wait_for_active_tenant(
     };
 
     // if the tenant has a proper status already, no need to wait for anything
-    if tenant.should_run_tasks() {
+    if tenant.current_state() == TenantState::Active {
         ControlFlow::Continue(tenant)
     } else {
         let mut tenant_state_updates = tenant.subscribe_for_state_updates();
@@ -163,14 +174,12 @@ async fn wait_for_active_tenant(
                 Ok(()) => {
                     let new_state = *tenant_state_updates.borrow();
                     match new_state {
-                        TenantState::Active {
-                            background_jobs_running: true,
-                        } => {
-                            debug!("Tenant state changed to active with background jobs enabled, continuing the task loop");
+                        TenantState::Active => {
+                            debug!("Tenant state changed to active, continuing the task loop");
                             return ControlFlow::Continue(tenant);
                         }
                         state => {
-                            debug!("Not running the task loop, tenant is not active with background jobs enabled: {state:?}");
+                            debug!("Not running the task loop, tenant is not active: {state:?}");
                             continue;
                         }
                     }

pageserver/src/walingest.rs

@@ -1091,9 +1091,9 @@ mod tests {
         Ok(walingest)
     }
 
-    #[test]
-    fn test_relsize() -> Result<()> {
-        let tenant = TenantHarness::create("test_relsize")?.load();
+    #[tokio::test]
+    async fn test_relsize() -> Result<()> {
+        let tenant = TenantHarness::create("test_relsize")?.load().await;
         let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?;
         let mut walingest = init_walingest_test(&*tline)?;
 
@@ -1219,9 +1219,9 @@ mod tests {
 
     // Test what happens if we dropped a relation
     // and then created it again within the same layer.
-    #[test]
-    fn test_drop_extend() -> Result<()> {
-        let tenant = TenantHarness::create("test_drop_extend")?.load();
+    #[tokio::test]
+    async fn test_drop_extend() -> Result<()> {
+        let tenant = TenantHarness::create("test_drop_extend")?.load().await;
         let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?;
         let mut walingest = init_walingest_test(&*tline)?;
 
@@ -1259,9 +1259,9 @@ mod tests {
     // Test what happens if we truncated a relation
     // so that one of its segments was dropped
     // and then extended it again within the same layer.
-    #[test]
-    fn test_truncate_extend() -> Result<()> {
-        let tenant = TenantHarness::create("test_truncate_extend")?.load();
+    #[tokio::test]
+    async fn test_truncate_extend() -> Result<()> {
+        let tenant = TenantHarness::create("test_truncate_extend")?.load().await;
         let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?;
         let mut walingest = init_walingest_test(&*tline)?;
 
@@ -1347,9 +1347,9 @@ mod tests {
 
     /// Test get_relsize() and truncation with a file larger than 1 GB, so that it's
     /// split into multiple 1 GB segments in Postgres.
-    #[test]
-    fn test_large_rel() -> Result<()> {
-        let tenant = TenantHarness::create("test_large_rel")?.load();
+    #[tokio::test]
+    async fn test_large_rel() -> Result<()> {
+        let tenant = TenantHarness::create("test_large_rel")?.load().await;
         let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?;
         let mut walingest = init_walingest_test(&*tline)?;
 

pageserver/src/walreceiver/connection_manager.rs

@@ -214,7 +214,7 @@ async fn connection_manager_loop_step(
                             match new_state {
                                 // we're already active as walreceiver, no need to reactivate
                                 TimelineState::Active => continue,
-                                TimelineState::Broken | TimelineState::Paused | TimelineState::Suspended => return ControlFlow::Continue(new_state),
+                                TimelineState::Broken | TimelineState::Stopping | TimelineState::Suspended => return ControlFlow::Continue(new_state),
                             }
                         }
                         Err(_sender_dropped_error) => return ControlFlow::Break(()),
@@ -870,10 +870,10 @@ mod tests {
     use crate::tenant::harness::{TenantHarness, TIMELINE_ID};
     use url::Host;
 
-    #[test]
-    fn no_connection_no_candidate() -> anyhow::Result<()> {
+    #[tokio::test]
+    async fn no_connection_no_candidate() -> anyhow::Result<()> {
         let harness = TenantHarness::create("no_connection_no_candidate")?;
-        let mut state = dummy_state(&harness);
+        let mut state = dummy_state(&harness).await;
         let now = Utc::now().naive_utc();
 
         let lagging_wal_timeout = chrono::Duration::from_std(state.lagging_wal_timeout)?;
@@ -964,7 +964,7 @@ mod tests {
     #[tokio::test]
     async fn connection_no_candidate() -> anyhow::Result<()> {
         let harness = TenantHarness::create("connection_no_candidate")?;
-        let mut state = dummy_state(&harness);
+        let mut state = dummy_state(&harness).await;
         let now = Utc::now().naive_utc();
 
         let connected_sk_id = NodeId(0);
@@ -1058,10 +1058,10 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn no_connection_candidate() -> anyhow::Result<()> {
+    #[tokio::test]
+    async fn no_connection_candidate() -> anyhow::Result<()> {
         let harness = TenantHarness::create("no_connection_candidate")?;
-        let mut state = dummy_state(&harness);
+        let mut state = dummy_state(&harness).await;
         let now = Utc::now().naive_utc();
 
         state.wal_connection = None;
@@ -1176,7 +1176,7 @@ mod tests {
     #[tokio::test]
     async fn candidate_with_many_connection_failures() -> anyhow::Result<()> {
         let harness = TenantHarness::create("candidate_with_many_connection_failures")?;
-        let mut state = dummy_state(&harness);
+        let mut state = dummy_state(&harness).await;
         let now = Utc::now().naive_utc();
 
         let current_lsn = Lsn(100_000).align();
@@ -1244,7 +1244,7 @@ mod tests {
     #[tokio::test]
     async fn lsn_wal_over_threshhold_current_candidate() -> anyhow::Result<()> {
         let harness = TenantHarness::create("lsn_wal_over_threshcurrent_candidate")?;
-        let mut state = dummy_state(&harness);
+        let mut state = dummy_state(&harness).await;
         let current_lsn = Lsn(100_000).align();
         let now = Utc::now().naive_utc();
 
@@ -1336,7 +1336,7 @@ mod tests {
     #[tokio::test]
     async fn timeout_connection_threshhold_current_candidate() -> anyhow::Result<()> {
         let harness = TenantHarness::create("timeout_connection_threshhold_current_candidate")?;
-        let mut state = dummy_state(&harness);
+        let mut state = dummy_state(&harness).await;
         let current_lsn = Lsn(100_000).align();
         let now = Utc::now().naive_utc();
 
@@ -1411,7 +1411,7 @@ mod tests {
     #[tokio::test]
     async fn timeout_wal_over_threshhold_current_candidate() -> anyhow::Result<()> {
         let harness = TenantHarness::create("timeout_wal_over_threshhold_current_candidate")?;
-        let mut state = dummy_state(&harness);
+        let mut state = dummy_state(&harness).await;
         let current_lsn = Lsn(100_000).align();
         let new_lsn = Lsn(100_100).align();
         let now = Utc::now().naive_utc();
@@ -1490,7 +1490,7 @@ mod tests {
 
     const DUMMY_SAFEKEEPER_HOST: &str = "safekeeper_connstr";
 
-    fn dummy_state(harness: &TenantHarness<'_>) -> WalreceiverState {
+    async fn dummy_state(harness: &TenantHarness<'_>) -> WalreceiverState {
         WalreceiverState {
             id: TenantTimelineId {
                 tenant_id: harness.tenant_id,
@@ -1498,6 +1498,7 @@ mod tests {
             },
             timeline: harness
                 .load()
+                .await
                 .create_empty_timeline(TIMELINE_ID, Lsn(0), crate::DEFAULT_PG_VERSION)
                 .expect("Failed to create an empty timeline for dummy wal connection manager")
                 .initialize()

pageserver/src/walreceiver/walreceiver_connection.rs

@@ -9,7 +9,6 @@ use std::{
 use anyhow::{bail, ensure, Context};
 use bytes::BytesMut;
 use chrono::{NaiveDateTime, Utc};
-use fail::fail_point;
 use futures::StreamExt;
 use postgres::{SimpleQueryMessage, SimpleQueryRow};
 use postgres_ffi::v14::xlog_utils::normalize_lsn;
@@ -20,20 +19,20 @@ use tokio::{pin, select, sync::watch, time};
 use tokio_postgres::{replication::ReplicationStream, Client};
 use tracing::{debug, error, info, trace, warn};
 
+use crate::fail_point;
 use crate::{metrics::LIVE_CONNECTIONS_COUNT, walreceiver::TaskStateUpdate};
 use crate::{
     task_mgr,
     task_mgr::TaskKind,
     task_mgr::WALRECEIVER_RUNTIME,
     tenant::{Timeline, WalReceiverInfo},
-    tenant_mgr,
     walingest::WalIngest,
     walrecord::DecodedWALRecord,
 };
 use postgres_connection::PgConnectionConfig;
 use postgres_ffi::waldecoder::WalStreamDecoder;
 use pq_proto::ReplicationFeedback;
-use utils::{id::TenantTimelineId, lsn::Lsn};
+use utils::lsn::Lsn;
 
 /// Status of the connection.
 #[derive(Debug, Clone)]
@@ -141,10 +140,6 @@ pub async fn handle_walreceiver_connection(
         return Ok(());
     }
 
-    let tenant_id = timeline.tenant_id;
-    let timeline_id = timeline.timeline_id;
-    let tenant = tenant_mgr::get_tenant(tenant_id, true)?;
-
     //
     // Start streaming the WAL, from where we left off previously.
     //
@@ -293,19 +288,8 @@ pub async fn handle_walreceiver_connection(
         })?;
 
         if let Some(last_lsn) = status_update {
-            let remote_index = tenant.get_remote_index();
-            let timeline_remote_consistent_lsn = remote_index
-                .read()
-                .await
-                // here we either do not have this timeline in remote index
-                // or there were no checkpoints for it yet
-                .timeline_entry(&TenantTimelineId {
-                    tenant_id,
-                    timeline_id,
-                })
-                .map(|remote_timeline| remote_timeline.metadata.disk_consistent_lsn())
-                // no checkpoint was uploaded
-                .unwrap_or(Lsn(0));
+            let timeline_remote_consistent_lsn =
+                timeline.get_remote_consistent_lsn().unwrap_or(Lsn(0));
 
             // The last LSN we processed. It is not guaranteed to survive pageserver crash.
             let write_lsn = u64::from(last_lsn);

pageserver/src/walredo.rs

@@ -84,7 +84,7 @@ pub trait WalRedoManager: Send + Sync {
         &self,
         key: Key,
         lsn: Lsn,
-        base_img: Option<Bytes>,
+        base_img: Option<(Lsn, Bytes)>,
         records: Vec<(Lsn, NeonWalRecord)>,
         pg_version: u32,
     ) -> Result<Bytes, WalRedoError>;
@@ -147,7 +147,7 @@ impl WalRedoManager for PostgresRedoManager {
         &self,
         key: Key,
         lsn: Lsn,
-        base_img: Option<Bytes>,
+        base_img: Option<(Lsn, Bytes)>,
         records: Vec<(Lsn, NeonWalRecord)>,
         pg_version: u32,
     ) -> Result<Bytes, WalRedoError> {
@@ -156,7 +156,8 @@ impl WalRedoManager for PostgresRedoManager {
             return Err(WalRedoError::InvalidRequest);
         }
 
-        let mut img: Option<Bytes> = base_img;
+        let base_img_lsn = base_img.as_ref().map(|p| p.0).unwrap_or(Lsn::INVALID);
+        let mut img = base_img.map(|p| p.1);
         let mut batch_neon = can_apply_in_neon(&records[0].1);
         let mut batch_start = 0;
         for i in 1..records.len() {
@@ -170,6 +171,7 @@ impl WalRedoManager for PostgresRedoManager {
                         key,
                         lsn,
                         img,
+                        base_img_lsn,
                         &records[batch_start..i],
                         self.conf.wal_redo_timeout,
                         pg_version,
@@ -189,6 +191,7 @@ impl WalRedoManager for PostgresRedoManager {
                 key,
                 lsn,
                 img,
+                base_img_lsn,
                 &records[batch_start..],
                 self.conf.wal_redo_timeout,
                 pg_version,
@@ -223,11 +226,13 @@ impl PostgresRedoManager {
     ///
     /// Process one request for WAL redo using wal-redo postgres
     ///
+    #[allow(clippy::too_many_arguments)]
     fn apply_batch_postgres(
         &self,
         key: Key,
         lsn: Lsn,
         base_img: Option<Bytes>,
+        base_img_lsn: Lsn,
         records: &[(Lsn, NeonWalRecord)],
         wal_redo_timeout: Duration,
         pg_version: u32,
@@ -282,9 +287,12 @@ impl PostgresRedoManager {
         // next request will launch a new one.
         if result.is_err() {
             error!(
-                "error applying {} WAL records ({} bytes) to reconstruct page image at LSN {}",
+                "error applying {} WAL records {}..{} ({} bytes) to base image with LSN {} to reconstruct page image at LSN {}",
                 records.len(),
+				records.first().map(|p| p.0).unwrap_or(Lsn(0)),
+				records.last().map(|p| p.0).unwrap_or(Lsn(0)),
                 nbytes,
+				base_img_lsn,
                 lsn
             );
             let process = process_guard.take().unwrap();
@@ -922,8 +930,7 @@ impl NoLeakChild {
 
         match child.wait() {
             Ok(exit_status) => {
-                // log at error level since .kill() is something we only do on errors ATM
-                error!(exit_status = %exit_status, "wait successful");
+                info!(exit_status = %exit_status, "wait successful");
             }
             Err(e) => {
                 error!(error = %e, "wait error; might leak the child process; it will show as zombie (defunct)");

pgxn/neon/libpagestore.c

@@ -464,12 +464,12 @@ pg_init_libpagestore(void)
 							NULL, NULL, NULL);
 	DefineCustomIntVariable("neon.readahead_buffer_size",
 							"number of prefetches to buffer",
-							"This buffer is used to store prefetched data; so "
-							"it is important that this buffer is at least as "
-							"large as the configured value of all tablespaces' "
-							"effective_io_concurrency and maintenance_io_concurrency, "
-							"your sessions' values of these, and the value for "
-							"seqscan_prefetch_buffers.",
+							"This buffer is used to hold and manage prefetched "
+							"data; so it is important that this buffer is at "
+							"least as large as the configured value of all "
+							"tablespaces' effective_io_concurrency and "
+							"maintenance_io_concurrency, and your sessions' "
+							"values for these settings.",
 							&readahead_buffer_size,
 							128, 16, 1024,
 							PGC_USERSET,

pgxn/neon/pagestore_smgr.c

@@ -242,6 +242,14 @@ PrefetchState *MyPState;
 	) \
 )
 
+#define ReceiveBufferNeedsCompaction() (\
+	(MyPState->n_responses_buffered / 8) < ( \
+		MyPState->ring_receive - \
+			MyPState->ring_last - \
+			MyPState->n_responses_buffered \
+	) \
+)
+
 int			n_prefetch_hits = 0;
 int			n_prefetch_misses = 0;
 int			n_prefetch_missed_caches = 0;
@@ -249,17 +257,99 @@ int			n_prefetch_dupes = 0;
 
 XLogRecPtr	prefetch_lsn = 0;
 
+static bool compact_prefetch_buffers(void);
 static void consume_prefetch_responses(void);
 static uint64 prefetch_register_buffer(BufferTag tag, bool *force_latest, XLogRecPtr *force_lsn);
 static bool prefetch_read(PrefetchRequest *slot);
 static void prefetch_do_request(PrefetchRequest *slot, bool *force_latest, XLogRecPtr *force_lsn);
 static bool prefetch_wait_for(uint64 ring_index);
-static void prefetch_cleanup(void);
+static void prefetch_cleanup_trailing_unused(void);
 static inline void prefetch_set_unused(uint64 ring_index);
 
 static XLogRecPtr neon_get_request_lsn(bool *latest, RelFileNode rnode,
 									   ForkNumber forknum, BlockNumber blkno);
 
+static bool
+compact_prefetch_buffers(void)
+{
+	uint64	empty_ring_index = MyPState->ring_last;
+	uint64	search_ring_index = MyPState->ring_receive;
+	int n_moved = 0;
+	
+	if (MyPState->ring_receive == MyPState->ring_last)
+		return false;
+
+	while (search_ring_index > MyPState->ring_last)
+	{
+		search_ring_index--;
+		if (GetPrfSlot(search_ring_index)->status == PRFS_UNUSED)
+		{
+			empty_ring_index = search_ring_index;
+			break;
+		}
+	}
+
+	/*
+	 * Here we have established:
+	 * slots < search_ring_index may be unused (not scanned)
+	 * slots >= search_ring_index and <= empty_ring_index are unused
+	 * slots > empty_ring_index are in use, or outside our buffer's range.
+	 * 
+	 * Therefore, there is a gap of at least one unused items between
+	 * search_ring_index and empty_ring_index, which grows as we hit
+	 * more unused items while moving backwards through the array.
+	 */
+
+	while (search_ring_index > MyPState->ring_last)
+	{
+		PrefetchRequest *source_slot;
+		PrefetchRequest *target_slot;
+		bool		found;
+
+		search_ring_index--;
+
+		source_slot = GetPrfSlot(search_ring_index);
+
+		if (source_slot->status == PRFS_UNUSED)
+			continue;
+
+		target_slot = GetPrfSlot(empty_ring_index);
+
+		Assert(source_slot->status == PRFS_RECEIVED);
+		Assert(target_slot->status == PRFS_UNUSED);
+
+		target_slot->buftag = source_slot->buftag;
+		target_slot->status = source_slot->status;
+		target_slot->response = source_slot->response;
+		target_slot->effective_request_lsn = source_slot->effective_request_lsn;
+		target_slot->my_ring_index = empty_ring_index;
+
+		prfh_delete(MyPState->prf_hash, source_slot);
+		prfh_insert(MyPState->prf_hash, target_slot, &found);
+
+		Assert(!found);
+
+		/* Adjust the location of our known-empty slot */
+		empty_ring_index--;
+
+		source_slot->status = PRFS_UNUSED;
+		source_slot->buftag = (BufferTag) {0};
+		source_slot->response = NULL;
+		source_slot->my_ring_index = 0;
+		source_slot->effective_request_lsn = 0;
+
+		n_moved++;
+	}
+
+	if (MyPState->ring_last != empty_ring_index)
+	{
+		MyPState->ring_last = empty_ring_index;
+		return true;
+	}
+
+	return false;
+}
+
 void
 readahead_buffer_resize(int newsize, void *extra)
 {
@@ -267,7 +357,7 @@ readahead_buffer_resize(int newsize, void *extra)
 				nfree = newsize;
 	PrefetchState *newPState;
 	Size 		newprfs_size = offsetof(PrefetchState, prf_buffer) + (
-		sizeof(PrefetchRequest) * readahead_buffer_size
+		sizeof(PrefetchRequest) * newsize
 	);
 	
 	/* don't try to re-initialize if we haven't initialized yet */
@@ -323,7 +413,7 @@ readahead_buffer_resize(int newsize, void *extra)
 		prfh_insert(newPState->prf_hash, newslot, &found);
 
 		Assert(!found);
-		
+
 		switch (newslot->status)
 		{
 			case PRFS_UNUSED:
@@ -370,7 +460,7 @@ consume_prefetch_responses(void)
 }
 
 static void
-prefetch_cleanup(void)
+prefetch_cleanup_trailing_unused(void)
 {
 	uint64	ring_index;
 	PrefetchRequest *slot;
@@ -531,7 +621,10 @@ prefetch_set_unused(uint64 ring_index)
 
 	/* run cleanup if we're holding back ring_last */
 	if (MyPState->ring_last == ring_index)
-		prefetch_cleanup();
+		prefetch_cleanup_trailing_unused();
+	/* ... and try to store the buffered responses more compactly if > 12.5% of the buffer is gaps */
+	else if (ReceiveBufferNeedsCompaction())
+		compact_prefetch_buffers();
 }
 
 static void
@@ -702,20 +795,31 @@ prefetch_register_buffer(BufferTag tag, bool *force_latest, XLogRecPtr *force_ls
 
 		Assert(slot->status != PRFS_UNUSED);
 
-		/* We have the slot for ring_last, so that must still be in progress */
-		switch (slot->status)
+		/*
+		 * If there is good reason to run compaction on the prefetch buffers,
+		 * try to do that.
+		 */
+		if (ReceiveBufferNeedsCompaction() && compact_prefetch_buffers())
 		{
-			case PRFS_REQUESTED:
-				Assert(MyPState->ring_receive == cleanup_index);
-				prefetch_wait_for(cleanup_index);
-				prefetch_set_unused(cleanup_index);
-				break;
-			case PRFS_RECEIVED:
-			case PRFS_TAG_REMAINS:
-				prefetch_set_unused(cleanup_index);
-				break;
-			default:
-				pg_unreachable();
+			Assert(slot->status == PRFS_UNUSED);
+		}
+		else
+		{
+			/* We have the slot for ring_last, so that must still be in progress */
+			switch (slot->status)
+			{
+				case PRFS_REQUESTED:
+					Assert(MyPState->ring_receive == cleanup_index);
+					prefetch_wait_for(cleanup_index);
+					prefetch_set_unused(cleanup_index);
+					break;
+				case PRFS_RECEIVED:
+				case PRFS_TAG_REMAINS:
+					prefetch_set_unused(cleanup_index);
+					break;
+				default:
+					pg_unreachable();
+			}
 		}
 	}
 
@@ -1102,7 +1206,7 @@ PageIsEmptyHeapPage(char *buffer)
 }
 
 static void
-neon_wallog_page(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer)
+neon_wallog_page(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer, bool force)
 {
 	XLogRecPtr	lsn = PageGetLSN(buffer);
 
@@ -1116,7 +1220,7 @@ neon_wallog_page(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, ch
 	 * correctness, the non-logged updates are not critical. But we want to
 	 * have a reasonably up-to-date VM and FSM in the page server.
 	 */
-	if (forknum == FSM_FORKNUM && !RecoveryInProgress())
+	if ((force || forknum == FSM_FORKNUM || forknum == VISIBILITYMAP_FORKNUM) && !RecoveryInProgress())
 	{
 		/* FSM is never WAL-logged and we don't care. */
 		XLogRecPtr	recptr;
@@ -1125,30 +1229,7 @@ neon_wallog_page(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, ch
 		XLogFlush(recptr);
 		lsn = recptr;
 		ereport(SmgrTrace,
-				(errmsg("FSM page %u of relation %u/%u/%u.%u was force logged. Evicted at lsn=%X/%X",
-						blocknum,
-						reln->smgr_rnode.node.spcNode,
-						reln->smgr_rnode.node.dbNode,
-						reln->smgr_rnode.node.relNode,
-						forknum, LSN_FORMAT_ARGS(lsn))));
-	}
-	else if (forknum == VISIBILITYMAP_FORKNUM && !RecoveryInProgress())
-	{
-		/*
-		 * Always WAL-log vm. We should never miss clearing visibility map
-		 * bits.
-		 *
-		 * TODO Is it too bad for performance? Hopefully we do not evict
-		 * actively used vm too often.
-		 */
-		XLogRecPtr	recptr;
-
-		recptr = log_newpage_copy(&reln->smgr_rnode.node, forknum, blocknum, buffer, false);
-		XLogFlush(recptr);
-		lsn = recptr;
-
-		ereport(SmgrTrace,
-				(errmsg("Visibilitymap page %u of relation %u/%u/%u.%u was force logged at lsn=%X/%X",
+				(errmsg("Page %u of relation %u/%u/%u.%u was force logged. Evicted at lsn=%X/%X",
 						blocknum,
 						reln->smgr_rnode.node.spcNode,
 						reln->smgr_rnode.node.dbNode,
@@ -1543,6 +1624,7 @@ neon_extend(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno,
 			char *buffer, bool skipFsync)
 {
 	XLogRecPtr	lsn;
+	BlockNumber	n_blocks = 0;
 
 	switch (reln->smgr_relpersistence)
 	{
@@ -1582,7 +1664,16 @@ neon_extend(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno,
 					 errhint("This limit is defined by neon.max_cluster_size GUC")));
 	}
 
-	neon_wallog_page(reln, forkNum, blkno, buffer);
+	/*
+	 * Usually Postgres doesn't extend relation on more than one page
+	 * (leaving holes). But this rule is violated in PG-15 where CreateAndCopyRelationData
+	 * call smgrextend for destination relation n using size of source relation
+	 */
+	get_cached_relsize(reln->smgr_rnode.node, forkNum, &n_blocks);
+	while (n_blocks < blkno)
+		neon_wallog_page(reln, forkNum, n_blocks++, buffer, true);
+
+	neon_wallog_page(reln, forkNum, blkno, buffer, false);
 	set_cached_relsize(reln->smgr_rnode.node, forkNum, blkno + 1);
 
 	lsn = PageGetLSN(buffer);
@@ -1780,6 +1871,17 @@ neon_read_at_lsn(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno,
 												  &request_lsn);
 			slot = GetPrfSlot(ring_index);
 		}
+		else
+		{
+			/*
+			 * Empty our reference to the prefetch buffer's hash entry.
+			 * When we wait for prefetches, the entry reference is invalidated by 
+			 * potential updates to the hash, and when we reconnect to the 
+			 * pageserver the prefetch we're waiting for may be dropped,
+			 * in which case we need to retry and take the branch above.
+			 */
+			entry = NULL;
+		}
 
 		Assert(slot->my_ring_index == ring_index);
 		Assert(MyPState->ring_last <= ring_index &&
@@ -1818,7 +1920,7 @@ neon_read_at_lsn(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno,
 
 	/* buffer was used, clean up for later reuse */
 	prefetch_set_unused(ring_index);
-	prefetch_cleanup();
+	prefetch_cleanup_trailing_unused();
 }
 
 /*
@@ -1999,7 +2101,7 @@ neon_write(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum,
 			elog(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence);
 	}
 
-	neon_wallog_page(reln, forknum, blocknum, buffer);
+	neon_wallog_page(reln, forknum, blocknum, buffer, false);
 
 	lsn = PageGetLSN(buffer);
 	elog(SmgrTrace, "smgrwrite called for %u/%u/%u.%u blk %u, page LSN: %X/%08X",

proxy/src/main.rs

@@ -28,6 +28,7 @@ use std::{borrow::Cow, future::Future, net::SocketAddr};
 use tokio::{net::TcpListener, task::JoinError};
 use tracing::info;
 use utils::project_git_version;
+use utils::sentry_init::{init_sentry, release_name};
 
 project_git_version!(GIT_VERSION);
 
@@ -45,6 +46,9 @@ async fn main() -> anyhow::Result<()> {
         .with_target(false)
         .init();
 
+    // initialize sentry if SENTRY_DSN is provided
+    let _sentry_guard = init_sentry(release_name!(), &[]);
+
     let arg_matches = cli().get_matches();
 
     let tls_config = match (

run_clippy.sh

@@ -13,7 +13,7 @@
 # avoid running regular linting script that checks every feature.
 if [[ "$OSTYPE" == "darwin"* ]]; then
     # no extra features to test currently, add more here when needed
-    cargo clippy --locked --all --all-targets --features testing -- -A unknown_lints -D warnings
+    cargo clippy --locked --all --all-targets -- -A unknown_lints -D warnings
 else
     # * `-A unknown_lints` – do not warn about unknown lint suppressions
     #                        that people with newer toolchains might use

safekeeper/src/bin/safekeeper.rs

@@ -4,7 +4,6 @@
 use anyhow::{bail, Context, Result};
 use clap::{value_parser, Arg, ArgAction, Command};
 use const_format::formatcp;
-use nix::unistd::Pid;
 use remote_storage::RemoteStorageConfig;
 use std::fs::{self, File};
 use std::io::{ErrorKind, Write};
@@ -15,7 +14,7 @@ use tokio::sync::mpsc;
 use toml_edit::Document;
 use tracing::*;
 use url::{ParseError, Url};
-use utils::lock_file;
+use utils::pid_file;
 
 use metrics::set_build_info_metric;
 use safekeeper::broker;
@@ -35,11 +34,14 @@ use utils::{
     http::endpoint,
     id::NodeId,
     logging::{self, LogFormat},
-    project_git_version, signals, tcp_listener,
+    project_git_version,
+    sentry_init::{init_sentry, release_name},
+    signals, tcp_listener,
 };
 
 const PID_FILE_NAME: &str = "safekeeper.pid";
 const ID_FILE_NAME: &str = "safekeeper.id";
+
 project_git_version!(GIT_VERSION);
 
 fn main() -> anyhow::Result<()> {
@@ -133,6 +135,8 @@ fn main() -> anyhow::Result<()> {
         conf.log_format = LogFormat::from_config(log_format)?;
     }
 
+    // initialize sentry if SENTRY_DSN is provided
+    let _sentry_guard = init_sentry(release_name!(), &[("node_id", &conf.my_id.to_string())]);
     start_safekeeper(conf, given_id, arg_matches.get_flag("init"))
 }
 
@@ -142,28 +146,13 @@ fn start_safekeeper(mut conf: SafeKeeperConf, given_id: Option<NodeId>, init: bo
 
     // Prevent running multiple safekeepers on the same directory
     let lock_file_path = conf.workdir.join(PID_FILE_NAME);
-    let lock_file = match lock_file::create_lock_file(&lock_file_path, Pid::this().to_string()) {
-        lock_file::LockCreationResult::Created {
-            new_lock_contents,
-            file,
-        } => {
-            info!("Created lock file at {lock_file_path:?} with contenst {new_lock_contents}");
-            file
-        }
-        lock_file::LockCreationResult::AlreadyLocked {
-            existing_lock_contents,
-        } => anyhow::bail!(
-            "Could not lock pid file; safekeeper is already running in {:?} with PID {}",
-            conf.workdir,
-            existing_lock_contents
-        ),
-        lock_file::LockCreationResult::CreationFailed(e) => {
-            return Err(e.context(format!("Failed to create lock file at {lock_file_path:?}")))
-        }
-    };
+    let lock_file =
+        pid_file::claim_for_current_process(&lock_file_path).context("claim pid file")?;
+    info!("Claimed pid file at {lock_file_path:?}");
+
     // ensure that the lock file is held even if the main thread of the process is panics
     // we need to release the lock file only when the current process is gone
-    let _ = Box::leak(Box::new(lock_file));
+    std::mem::forget(lock_file);
 
     // Set or read our ID.
     set_id(&mut conf, given_id)?;

safekeeper/src/send_wal.rs

@@ -226,6 +226,7 @@ impl ReplicationConn {
             let mut end_pos = stop_pos.unwrap_or(inmem_state.commit_lsn);
 
             let mut wal_reader = WalReader::new(
+                spg.conf.workdir.clone(),
                 spg.conf.timeline_dir(&tli.ttid),
                 &persisted_state,
                 start_pos,

safekeeper/src/wal_backup.rs

@@ -13,7 +13,7 @@ use std::time::Duration;
 use postgres_ffi::v14::xlog_utils::XLogSegNoOffsetToRecPtr;
 use postgres_ffi::XLogFileName;
 use postgres_ffi::{XLogSegNo, PG_TLI};
-use remote_storage::GenericRemoteStorage;
+use remote_storage::{GenericRemoteStorage, RemotePath};
 use tokio::fs::File;
 use tokio::runtime::Builder;
 
@@ -151,7 +151,7 @@ async fn update_task(
             let timeline_dir = conf.timeline_dir(&ttid);
 
             let handle = tokio::spawn(
-                backup_task_main(ttid, timeline_dir, shutdown_rx)
+                backup_task_main(ttid, timeline_dir, conf.workdir.clone(), shutdown_rx)
                     .instrument(info_span!("WAL backup task", ttid = %ttid)),
             );
 
@@ -182,10 +182,10 @@ async fn wal_backup_launcher_main_loop(
 
     let conf_ = conf.clone();
     REMOTE_STORAGE.get_or_init(|| {
-        conf_.remote_storage.as_ref().map(|c| {
-            GenericRemoteStorage::from_config(conf_.workdir, c)
-                .expect("failed to create remote storage")
-        })
+        conf_
+            .remote_storage
+            .as_ref()
+            .map(|c| GenericRemoteStorage::from_config(c).expect("failed to create remote storage"))
     });
 
     // Presense in this map means launcher is aware s3 offloading is needed for
@@ -234,6 +234,7 @@ async fn wal_backup_launcher_main_loop(
 struct WalBackupTask {
     timeline: Arc<Timeline>,
     timeline_dir: PathBuf,
+    workspace_dir: PathBuf,
     wal_seg_size: usize,
     commit_lsn_watch_rx: watch::Receiver<Lsn>,
 }
@@ -242,6 +243,7 @@ struct WalBackupTask {
 async fn backup_task_main(
     ttid: TenantTimelineId,
     timeline_dir: PathBuf,
+    workspace_dir: PathBuf,
     mut shutdown_rx: Receiver<()>,
 ) {
     info!("started");
@@ -257,6 +259,7 @@ async fn backup_task_main(
         commit_lsn_watch_rx: tli.get_commit_lsn_watch_rx(),
         timeline: tli,
         timeline_dir,
+        workspace_dir,
     };
 
     // task is spinned up only when wal_seg_size already initialized
@@ -321,6 +324,7 @@ impl WalBackupTask {
                 commit_lsn,
                 self.wal_seg_size,
                 &self.timeline_dir,
+                &self.workspace_dir,
             )
             .await
             {
@@ -353,11 +357,12 @@ pub async fn backup_lsn_range(
     end_lsn: Lsn,
     wal_seg_size: usize,
     timeline_dir: &Path,
+    workspace_dir: &Path,
 ) -> Result<Lsn> {
     let mut res = start_lsn;
     let segments = get_segments(start_lsn, end_lsn, wal_seg_size);
     for s in &segments {
-        backup_single_segment(s, timeline_dir)
+        backup_single_segment(s, timeline_dir, workspace_dir)
             .await
             .with_context(|| format!("offloading segno {}", s.seg_no))?;
 
@@ -372,11 +377,24 @@ pub async fn backup_lsn_range(
     Ok(res)
 }
 
-async fn backup_single_segment(seg: &Segment, timeline_dir: &Path) -> Result<()> {
-    let segment_file_name = seg.file_path(timeline_dir)?;
+async fn backup_single_segment(
+    seg: &Segment,
+    timeline_dir: &Path,
+    workspace_dir: &Path,
+) -> Result<()> {
+    let segment_file_path = seg.file_path(timeline_dir)?;
+    let remote_segment_path = segment_file_path
+        .strip_prefix(&workspace_dir)
+        .context("Failed to strip workspace dir prefix")
+        .and_then(RemotePath::new)
+        .with_context(|| {
+            format!(
+                "Failed to resolve remote part of path {segment_file_path:?} for base {workspace_dir:?}",
+            )
+        })?;
 
-    backup_object(&segment_file_name, seg.size()).await?;
-    debug!("Backup of {} done", segment_file_name.display());
+    backup_object(&segment_file_path, &remote_segment_path, seg.size()).await?;
+    debug!("Backup of {} done", segment_file_path.display());
 
     Ok(())
 }
@@ -426,7 +444,7 @@ fn get_segments(start: Lsn, end: Lsn, seg_size: usize) -> Vec<Segment> {
 
 static REMOTE_STORAGE: OnceCell<Option<GenericRemoteStorage>> = OnceCell::new();
 
-async fn backup_object(source_file: &Path, size: usize) -> Result<()> {
+async fn backup_object(source_file: &Path, target_file: &RemotePath, size: usize) -> Result<()> {
     let storage = REMOTE_STORAGE
         .get()
         .expect("failed to get remote storage")
@@ -441,12 +459,12 @@ async fn backup_object(source_file: &Path, size: usize) -> Result<()> {
     })?);
 
     storage
-        .upload_storage_object(Box::new(file), size, source_file)
+        .upload_storage_object(Box::new(file), size, target_file)
         .await
 }
 
 pub async fn read_object(
-    file_path: PathBuf,
+    file_path: &RemotePath,
     offset: u64,
 ) -> anyhow::Result<Pin<Box<dyn tokio::io::AsyncRead>>> {
     let storage = REMOTE_STORAGE
@@ -455,19 +473,13 @@ pub async fn read_object(
         .as_ref()
         .context("No remote storage configured")?;
 
-    info!(
-        "segment download about to start for local path {} at offset {}",
-        file_path.display(),
-        offset
-    );
+    info!("segment download about to start from remote path {file_path:?} at offset {offset}");
+
     let download = storage
-        .download_storage_object(Some((offset, None)), &file_path)
+        .download_storage_object(Some((offset, None)), file_path)
         .await
         .with_context(|| {
-            format!(
-                "Failed to open WAL segment download stream for local path {}",
-                file_path.display()
-            )
+            format!("Failed to open WAL segment download stream for remote path {file_path:?}")
         })?;
 
     Ok(download.download_stream)

safekeeper/src/wal_storage.rs

@@ -8,6 +8,7 @@
 //! Note that last file has `.partial` suffix, that's different from postgres.
 
 use anyhow::{bail, Context, Result};
+use remote_storage::RemotePath;
 
 use std::io::{self, Seek, SeekFrom};
 use std::pin::Pin;
@@ -445,6 +446,7 @@ fn remove_segments_from_disk(
 }
 
 pub struct WalReader {
+    workdir: PathBuf,
     timeline_dir: PathBuf,
     wal_seg_size: usize,
     pos: Lsn,
@@ -459,6 +461,7 @@ pub struct WalReader {
 
 impl WalReader {
     pub fn new(
+        workdir: PathBuf,
         timeline_dir: PathBuf,
         state: &SafeKeeperState,
         start_pos: Lsn,
@@ -478,6 +481,7 @@ impl WalReader {
         }
 
         Ok(Self {
+            workdir,
             timeline_dir,
             wal_seg_size: state.server.wal_seg_size as usize,
             pos: start_pos,
@@ -545,7 +549,17 @@ impl WalReader {
 
         // Try to open remote file, if remote reads are enabled
         if self.enable_remote_read {
-            return read_object(wal_file_path, xlogoff as u64).await;
+            let remote_wal_file_path = wal_file_path
+                .strip_prefix(&self.workdir)
+                .context("Failed to strip workdir prefix")
+                .and_then(RemotePath::new)
+                .with_context(|| {
+                    format!(
+                        "Failed to resolve remote part of path {:?} for base {:?}",
+                        wal_file_path, self.workdir,
+                    )
+                })?;
+            return read_object(&remote_wal_file_path, xlogoff as u64).await;
         }
 
         bail!("WAL segment is not found")

storage_broker/build.rs

@@ -1,7 +1,11 @@
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Generate code to deterministic location to make finding it easier.
-    tonic_build::configure()
-        .out_dir("proto/") // put generated code to proto/
-        .compile(&["proto/broker.proto"], &["proto/"])?;
+    // Generate rust code from .proto protobuf.
+    //
+    // Note: we previously tried to use deterministic location at proto/ for
+    // easy location, but apparently interference with cachepot sometimes fails
+    // the build then. Anyway, per cargo docs build script shouldn't output to
+    // anywhere but $OUT_DIR.
+    tonic_build::compile_protos("proto/broker.proto")
+        .unwrap_or_else(|e| panic!("failed to compile protos {:?}", e));
     Ok(())
 }

storage_broker/src/lib.rs

@@ -11,7 +11,7 @@ use proto::{
 
 // Code generated by protobuf.
 pub mod proto {
-    include!("../proto/storage_broker.rs");
+    tonic::include_proto!("storage_broker");
 }
 
 pub mod metrics;

test_runner/README.md

@@ -6,9 +6,6 @@ Prerequisites:
 - Correctly configured Python, see [`/docs/sourcetree.md`](/docs/sourcetree.md#using-python)
 - Neon and Postgres binaries
     - See the root [README.md](/README.md) for build directions
-      If you want to test tests with test-only APIs, you would need to add `--features testing` to Rust code build commands.
-      For convenience, repository cargo config contains `build_testing` alias, that serves as a subcommand, adding the required feature flags.
-      Usage example: `cargo build_testing --release` is equivalent to `cargo build --features testing --release`
     - Tests can be run from the git tree; or see the environment variables
       below to run from other directories.
 - The neon git repo, including the postgres submodule

test_runner/fixtures/neon_fixtures.py

@@ -33,7 +33,7 @@ from _pytest.config import Config
 from _pytest.fixtures import FixtureRequest
 from fixtures.log_helper import log
 from fixtures.types import Lsn, TenantId, TimelineId
-from fixtures.utils import Fn, allure_attach_from_dir, etcd_path, get_self_dir, subprocess_capture
+from fixtures.utils import allure_attach_from_dir, etcd_path, get_self_dir, subprocess_capture
 
 # Type-related stuff
 from psycopg2.extensions import connection as PgConnection
@@ -587,6 +587,7 @@ class NeonEnvBuilder:
         auth_enabled: bool = False,
         rust_log_override: Optional[str] = None,
         default_branch_name: str = DEFAULT_BRANCH_NAME,
+        testing_mode: bool = True,
     ):
         self.repo_dir = repo_dir
         self.rust_log_override = rust_log_override
@@ -608,6 +609,7 @@ class NeonEnvBuilder:
         self.neon_binpath = neon_binpath
         self.pg_distrib_dir = pg_distrib_dir
         self.pg_version = pg_version
+        self.testing_mode = testing_mode
 
     def init(self) -> NeonEnv:
         # Cannot create more than one environment from one builder
@@ -858,6 +860,7 @@ class NeonEnv:
             http=self.port_distributor.get_port(),
         )
         pageserver_auth_type = "NeonJWT" if config.auth_enabled else "Trust"
+        pageserver_testing_mode = "true" if config.testing_mode else "false"
 
         toml += textwrap.dedent(
             f"""
@@ -866,6 +869,7 @@ class NeonEnv:
             listen_pg_addr = 'localhost:{pageserver_port.pg}'
             listen_http_addr = 'localhost:{pageserver_port.http}'
             auth_type = '{pageserver_auth_type}'
+            testing_mode = {pageserver_testing_mode}
         """
         )
 
@@ -978,6 +982,10 @@ def _shared_simple_env(
         pg_distrib_dir=pg_distrib_dir,
         pg_version=pg_version,
         run_id=run_id,
+        # Disable failpoint support. Failpoints could have unexpected consequences
+        # when the pageserver is shared by concurrent tests. Also, it might affect
+        # performance, and we use the shared simple env in performance tests.
+        testing_mode=False,
     ) as builder:
         env = builder.init_start()
 
@@ -1048,11 +1056,10 @@ class PageserverApiException(Exception):
 
 
 class PageserverHttpClient(requests.Session):
-    def __init__(self, port: int, is_testing_enabled_or_skip: Fn, auth_token: Optional[str] = None):
+    def __init__(self, port: int, auth_token: Optional[str] = None):
         super().__init__()
         self.port = port
         self.auth_token = auth_token
-        self.is_testing_enabled_or_skip = is_testing_enabled_or_skip
 
         if auth_token is not None:
             self.headers["Authorization"] = f"Bearer {auth_token}"
@@ -1071,8 +1078,6 @@ class PageserverHttpClient(requests.Session):
         self.get(f"http://localhost:{self.port}/v1/status").raise_for_status()
 
     def configure_failpoints(self, config_strings: Tuple[str, str] | List[Tuple[str, str]]):
-        self.is_testing_enabled_or_skip()
-
         if isinstance(config_strings, tuple):
             pairs = [config_strings]
         else:
@@ -1119,6 +1124,14 @@ class PageserverHttpClient(requests.Session):
         res = self.post(f"http://localhost:{self.port}/v1/tenant/{tenant_id}/detach")
         self.verbose_error(res)
 
+    def tenant_load(self, tenant_id: TenantId):
+        res = self.post(f"http://localhost:{self.port}/v1/tenant/{tenant_id}/load")
+        self.verbose_error(res)
+
+    def tenant_ignore(self, tenant_id: TenantId):
+        res = self.post(f"http://localhost:{self.port}/v1/tenant/{tenant_id}/ignore")
+        self.verbose_error(res)
+
     def tenant_status(self, tenant_id: TenantId) -> Dict[Any, Any]:
         res = self.get(f"http://localhost:{self.port}/v1/tenant/{tenant_id}")
         self.verbose_error(res)
@@ -1204,8 +1217,6 @@ class PageserverHttpClient(requests.Session):
     def timeline_gc(
         self, tenant_id: TenantId, timeline_id: TimelineId, gc_horizon: Optional[int]
     ) -> dict[str, Any]:
-        self.is_testing_enabled_or_skip()
-
         log.info(
             f"Requesting GC: tenant {tenant_id}, timeline {timeline_id}, gc_horizon {repr(gc_horizon)}"
         )
@@ -1221,8 +1232,6 @@ class PageserverHttpClient(requests.Session):
         return res_json
 
     def timeline_compact(self, tenant_id: TenantId, timeline_id: TimelineId):
-        self.is_testing_enabled_or_skip()
-
         log.info(f"Requesting compact: tenant {tenant_id}, timeline {timeline_id}")
         res = self.put(
             f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/compact"
@@ -1246,8 +1255,6 @@ class PageserverHttpClient(requests.Session):
         return res_json
 
     def timeline_checkpoint(self, tenant_id: TenantId, timeline_id: TimelineId):
-        self.is_testing_enabled_or_skip()
-
         log.info(f"Requesting checkpoint: tenant {tenant_id}, timeline {timeline_id}")
         res = self.put(
             f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/checkpoint"
@@ -1761,6 +1768,13 @@ class NeonPageserver(PgProtocol):
             ".*Removing intermediate uninit mark file.*",
             # FIXME: known race condition in TaskHandle: https://github.com/neondatabase/neon/issues/2885
             ".*sender is dropped while join handle is still alive.*",
+            # Tenant::delete_timeline() can cause any of the four following errors.
+            # FIXME: we shouldn't be considering it an error: https://github.com/neondatabase/neon/issues/2946
+            ".*could not flush frozen layer.*queue is in state Stopped",  # when schedule layer upload fails because queued got closed before compaction got killed
+            ".*wait for layer upload ops to complete.*",  # .*Caused by:.*wait_completion aborted because upload queue was stopped
+            ".*gc_loop.*Gc failed, retrying in.*timeline is Stopping",  # When gc checks timeline state after acquiring layer_removal_cs
+            ".*compaction_loop.*Compaction failed, retrying in.*timeline is Stopping",  # When compaction checks timeline state after acquiring layer_removal_cs
+            ".*query handler for 'pagestream.*failed: Timeline .* was not found",  # postgres reconnects while timeline_delete doesn't hold the tenant's timelines.lock()
         ]
 
     def start(
@@ -1800,10 +1814,6 @@ class NeonPageserver(PgProtocol):
     ):
         self.stop(immediate=True)
 
-    def is_testing_enabled_or_skip(self):
-        if '"testing"' not in self.version:
-            pytest.skip("pageserver was built without 'testing' feature")
-
     def is_profiling_enabled_or_skip(self):
         if '"profiling"' not in self.version:
             pytest.skip("pageserver was built without 'profiling' feature")
@@ -1812,7 +1822,6 @@ class NeonPageserver(PgProtocol):
         return PageserverHttpClient(
             port=self.service_port.http,
             auth_token=auth_token,
-            is_testing_enabled_or_skip=self.is_testing_enabled_or_skip,
         )
 
     def assert_no_errors(self):
@@ -2528,6 +2537,7 @@ class SafekeeperTimelineStatus:
     acceptor_epoch: int
     pg_version: int
     flush_lsn: Lsn
+    commit_lsn: Lsn
     timeline_start_lsn: Lsn
     backup_lsn: Lsn
     remote_consistent_lsn: Lsn
@@ -2577,6 +2587,7 @@ class SafekeeperHttpClient(requests.Session):
             acceptor_epoch=resj["acceptor_state"]["epoch"],
             pg_version=resj["pg_info"]["pg_version"],
             flush_lsn=Lsn(resj["flush_lsn"]),
+            commit_lsn=Lsn(resj["commit_lsn"]),
             timeline_start_lsn=Lsn(resj["timeline_start_lsn"]),
             backup_lsn=Lsn(resj["backup_lsn"]),
             remote_consistent_lsn=Lsn(resj["remote_consistent_lsn"]),

test_runner/performance/README.md

@@ -3,7 +3,7 @@
 First make a release build. The profiling flag is optional, used only for tests that
 generate flame graphs. The `-s` flag just silences a lot of output, and makes it
 easier to see if you have compile errors without scrolling up.
-`BUILD_TYPE=release CARGO_BUILD_FLAGS="--features=testing,profiling" make -s -j8`
+`BUILD_TYPE=release CARGO_BUILD_FLAGS="--features=profiling" make -s -j8`
 
 NOTE: the `profiling` flag only works on linux because we use linux-specific
 libc APIs like `libc::timer_t`.

test_runner/performance/test_bulk_update.py

@@ -42,7 +42,8 @@ def test_bulk_update(neon_env_builder: NeonEnvBuilder, zenbenchmark, fillfactor)
 
     cur.execute("drop table t")
     cur.execute("set enable_seqscan_prefetch=on")
-    cur.execute("set seqscan_prefetch_buffers=100")
+    cur.execute("set effective_io_concurrency=32")
+    cur.execute("set maintenance_io_concurrency=32")
 
     cur.execute(f"create table t2(x integer) WITH (fillfactor={fillfactor})")
 

test_runner/performance/test_layer_map.py

@@ -12,12 +12,12 @@ def test_layer_map(neon_env_builder: NeonEnvBuilder, zenbenchmark):
     n_iters = 10
     n_records = 100000
 
-    # We want to have a lot of lot of layer files to exercise the layer map. Make
-    # gc_horizon and checkpoint_distance very small, so that we get a lot of small layer files.
+    # We want to have a lot of lot of layer files to exercise the layer map. Disable
+    # GC, and make checkpoint_distance very small, so that we get a lot of small layer
+    # files.
     tenant, _ = env.neon_cli.create_tenant(
         conf={
-            "gc_period": "100 m",
-            "gc_horizon": "1048576",
+            "gc_period": "0s",
             "checkpoint_distance": "8192",
             "compaction_period": "1 s",
             "compaction_threshold": "1",

test_runner/performance/test_perf_olap.py

@@ -0,0 +1,111 @@
+from dataclasses import dataclass
+from typing import Dict, Tuple
+
+import pytest
+from fixtures.compare_fixtures import RemoteCompare
+from fixtures.log_helper import log
+
+
+@dataclass
+class LabelledQuery:
+    """An SQL query with a label for the test report."""
+
+    label: str
+    query: str
+
+
+# A list of queries to run.
+# Please do not alter the label for the query, as it is used to identify it.
+# Labels for ClickBench queries match the labels in ClickBench reports
+# on https://benchmark.clickhouse.com/ (the DB size may differ).
+QUERIES: Tuple[LabelledQuery, ...] = (
+    # Disable `black` formatting for the list of queries so that it's easier to read
+    # fmt: off
+    ### ClickBench queries:
+    LabelledQuery("Q0",  r"SELECT COUNT(*) FROM hits;"),
+    LabelledQuery("Q1",  r"SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;"),
+    LabelledQuery("Q2",  r"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;"),
+    LabelledQuery("Q3",  r"SELECT AVG(UserID) FROM hits;"),
+    LabelledQuery("Q4",  r"SELECT COUNT(DISTINCT UserID) FROM hits;"),
+    LabelledQuery("Q5",  r"SELECT COUNT(DISTINCT SearchPhrase) FROM hits;"),
+    LabelledQuery("Q6",  r"SELECT MIN(EventDate), MAX(EventDate) FROM hits;"),
+    LabelledQuery("Q7",  r"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;"),
+    LabelledQuery("Q8",  r"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;"),
+    LabelledQuery("Q9",  r"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;"),
+    LabelledQuery("Q10", r"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;"),
+    LabelledQuery("Q11", r"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;"),
+    LabelledQuery("Q12", r"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;"),
+    LabelledQuery("Q13", r"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;"),
+    LabelledQuery("Q14", r"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;"),
+    LabelledQuery("Q15", r"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;"),
+    LabelledQuery("Q16", r"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;"),
+    LabelledQuery("Q17", r"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;"),
+    LabelledQuery("Q18", r"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;"),
+    LabelledQuery("Q19", r"SELECT UserID FROM hits WHERE UserID = 435090932899640449;"),
+    LabelledQuery("Q20", r"SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';"),
+    LabelledQuery("Q21", r"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;"),
+    LabelledQuery("Q22", r"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;"),
+    LabelledQuery("Q23", r"SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;"),
+    LabelledQuery("Q24", r"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;"),
+    LabelledQuery("Q25", r"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;"),
+    LabelledQuery("Q26", r"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;"),
+    LabelledQuery("Q27", r"SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;"),
+    LabelledQuery("Q28", r"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;"),
+    LabelledQuery("Q29", r"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;"),
+    LabelledQuery("Q30", r"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;"),
+    LabelledQuery("Q31", r"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;"),
+    LabelledQuery("Q32", r"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;"),
+    LabelledQuery("Q33", r"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;"),
+    LabelledQuery("Q34", r"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;"),
+    LabelledQuery("Q35", r"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;"),
+    LabelledQuery("Q36", r"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;"),
+    LabelledQuery("Q37", r"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;"),
+    LabelledQuery("Q38", r"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;"),
+    LabelledQuery("Q39", r"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;"),
+    LabelledQuery("Q40", r"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;"),
+    LabelledQuery("Q41", r"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;"),
+    LabelledQuery("Q42", r"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;"),
+    ### Custom Neon queries:
+    # I suggest using the NQ prefix (which stands for Neon Query) instead of Q
+    # to not intersect with the original ClickBench queries if their list is extended.
+    #
+    # LabelledQuery("NQ0", r"..."),
+    # LabelledQuery("NQ1", r"..."),
+    # ...
+    # fmt: on
+)
+
+
+def run_psql(env: RemoteCompare, labelled_query: LabelledQuery, times: int) -> None:
+    # prepare connstr:
+    # - cut out password from connstr to pass it via env
+    # - add options to connstr
+    password = env.pg.default_options.get("password", None)
+    options = f"-cstatement_timeout=0 {env.pg.default_options.get('options', '')}"
+    connstr = env.pg.connstr(password=None, options=options)
+
+    environ: Dict[str, str] = {}
+    if password is not None:
+        environ["PGPASSWORD"] = password
+
+    label, query = labelled_query.label, labelled_query.query
+
+    log.info(f"Running query {label} {times} times")
+    for i in range(times):
+        run = i + 1
+        log.info(f"Run {run}/{times}")
+        with env.zenbenchmark.record_duration(f"{label}/{run}"):
+            env.pg_bin.run_capture(["psql", connstr, "-c", query], env=environ)
+
+
+@pytest.mark.parametrize("query", QUERIES)
+@pytest.mark.remote_cluster
+def test_clickbench(query: LabelledQuery, remote_compare: RemoteCompare):
+    """
+    An OLAP-style ClickHouse benchmark
+
+    Based on https://github.com/ClickHouse/ClickBench/tree/c00135ca5b6a0d86fedcdbf998fdaa8ed85c1c3b/aurora-postgresql
+    The DB prepared manually in advance
+    """
+
+    run_psql(remote_compare, query, times=3)

test_runner/regress/test_ancestor_branch.py

@@ -11,16 +11,12 @@ def test_ancestor_branch(neon_env_builder: NeonEnvBuilder):
     env = neon_env_builder.init_start()
     pageserver_http = env.pageserver.http_client()
 
-    # Override defaults, 1M gc_horizon and 4M checkpoint_distance.
-    # Extend compaction_period and gc_period to disable background compaction and gc.
+    # Override defaults: 4M checkpoint_distance, disable background compaction and gc.
     tenant, _ = env.neon_cli.create_tenant(
         conf={
-            "gc_period": "10 m",
-            "gc_horizon": "1048576",
             "checkpoint_distance": "4194304",
-            "compaction_period": "10 m",
-            "compaction_threshold": "2",
-            "compaction_target_size": "4194304",
+            "gc_period": "0s",
+            "compaction_period": "0s",
         }
     )
 

test_runner/regress/test_branch_and_gc.py

@@ -52,8 +52,7 @@ def test_branch_and_gc(neon_simple_env: NeonEnv):
     tenant, _ = env.neon_cli.create_tenant(
         conf={
             # disable background GC
-            "gc_period": "10 m",
-            "gc_horizon": f"{10 * 1024 ** 3}",
+            "gc_period": "0s",
             # small checkpoint distance to create more delta layer files
             "checkpoint_distance": f"{1024 ** 2}",
             # set the target size to be large to allow the image layer to cover the whole key space
@@ -127,8 +126,7 @@ def test_branch_creation_before_gc(neon_simple_env: NeonEnv):
     tenant, _ = env.neon_cli.create_tenant(
         conf={
             # disable background GC
-            "gc_period": "10 m",
-            "gc_horizon": f"{10 * 1024 ** 3}",
+            "gc_period": "0s",
             # small checkpoint distance to create more delta layer files
             "checkpoint_distance": f"{1024 ** 2}",
             # set the target size to be large to allow the image layer to cover the whole key space

test_runner/regress/test_broken_timeline.py

@@ -15,10 +15,12 @@ def test_broken_timeline(neon_env_builder: NeonEnvBuilder):
 
     env.pageserver.allowed_errors.extend(
         [
-            ".*No timelines to attach received.*",
-            ".*Failed to process timeline dir contents.*",
             ".*Failed to load delta layer.*",
-            ".*Timeline .* was not found.*",
+            ".*could not find data for key.*",
+            ".*is not active. Current state: Broken.*",
+            ".*will not become active. Current state: Broken.*",
+            ".*failed to load metadata.*",
+            ".*could not load tenant.*load local timeline.*",
         ]
     )
 
@@ -77,16 +79,20 @@ def test_broken_timeline(neon_env_builder: NeonEnvBuilder):
     # But all others are broken
 
     # First timeline would not get loaded into pageserver due to corrupt metadata file
-    with pytest.raises(Exception, match=f"Timeline {tenant1}/{timeline1} was not found") as err:
+    with pytest.raises(
+        Exception, match=f"Tenant {tenant1} will not become active. Current state: Broken"
+    ) as err:
         pg1.start()
     log.info(
         f"As expected, compute startup failed eagerly for timeline with corrupt metadata: {err}"
     )
 
-    # Second timeline has no ancestors, only the metadata file and no layer files
-    # We don't have the remote storage enabled, which means timeline is in an incorrect state,
-    # it's not loaded at all
-    with pytest.raises(Exception, match=f"Timeline {tenant2}/{timeline2} was not found") as err:
+    # Second timeline has no ancestors, only the metadata file and no layer files.
+    # That is checked explicitly in the pageserver, and causes the tenant to be marked
+    # as broken.
+    with pytest.raises(
+        Exception, match=f"Tenant {tenant2} will not become active. Current state: Broken"
+    ) as err:
         pg2.start()
     log.info(f"As expected, compute startup failed for timeline with missing layers: {err}")
 

test_runner/regress/test_compatibility.py

@@ -327,7 +327,6 @@ def check_neon_works(
     auth_token = snapshot_config["pageserver"]["auth_token"]
     pageserver_http = PageserverHttpClient(
         port=pageserver_port,
-        is_testing_enabled_or_skip=lambda: True,  # TODO: check if testing really enabled
         auth_token=auth_token,
     )
 

test_runner/regress/test_import.py

@@ -120,6 +120,12 @@ def test_import_from_vanilla(test_output_dir, pg_bin, vanilla_pg, neon_env_build
             ]
         )
 
+    # Importing empty file fails
+    empty_file = os.path.join(test_output_dir, "empty_file")
+    with open(empty_file, "w") as _:
+        with pytest.raises(Exception):
+            import_tar(empty_file, empty_file)
+
     # Importing corrupt backup fails
     with pytest.raises(Exception):
         import_tar(corrupt_base_tar, wal_tar)

test_runner/regress/test_pageserver_restart.py

@@ -56,9 +56,24 @@ def test_pageserver_restart(neon_env_builder: NeonEnvBuilder):
     cur.execute("SELECT count(*) FROM foo")
     assert cur.fetchone() == (100000,)
 
-    # Stop the page server by force, and restart it
+    # Restart the server again, but delay the loading of tenants, and test what the
+    # pageserver does if a compute node connects and sends a request for the tenant
+    # while it's still in Loading state. (It waits for the loading to finish, and then
+    # processes the request.)
     env.pageserver.stop()
-    env.pageserver.start()
+    env.pageserver.start(extra_env_vars={"FAILPOINTS": "before-loading-tenant=return(5000)"})
+
+    # Check that it's in Loading state
+    client = env.pageserver.http_client()
+    tenant_status = client.tenant_status(env.initial_tenant)
+    log.info("Tenant status : %s", tenant_status)
+    assert tenant_status["state"] == "Loading"
+
+    # Try to read. This waits until the loading finishes, and then return normally.
+    pg_conn = pg.connect()
+    cur = pg_conn.cursor()
+    cur.execute("SELECT count(*) FROM foo")
+    assert cur.fetchone() == (100000,)
 
 
 # Test that repeatedly kills and restarts the page server, while the